diff --git a/.coderabbit.yaml b/.coderabbit.yaml deleted file mode 100644 index 46a4f0b9..00000000 --- a/.coderabbit.yaml +++ /dev/null @@ -1,5 +0,0 @@ -reviews: - auto_review: - enabled: true - auto_incremental_review: true - drafts: false diff --git a/.gemini/config.yaml b/.gemini/config.yaml deleted file mode 100644 index 3b5e413e..00000000 --- a/.gemini/config.yaml +++ /dev/null @@ -1,23 +0,0 @@ -have_fun: false - -ignore_patterns: - - "**/charts/**" - - "**/vendor/**" - - "**/zz_generated.*.go" - - "**/pkg/generated/**" - - "**/_out/**" - - "**/*.tgz" - - "**/dashboards/**/*.json" - - "**/*.patch" - - "**/*.diff" - - "**/images/*.json" - -code_review: - disable: false - comment_severity_threshold: LOW - max_review_comments: 50 - pull_request_opened: - help: false - summary: true - code_review: true - include_drafts: false diff --git a/.gemini/styleguide.md b/.gemini/styleguide.md deleted file mode 100644 index 7185a625..00000000 --- a/.gemini/styleguide.md +++ /dev/null @@ -1,110 +0,0 @@ -# Cozystack Review Guidelines - -## Project Architecture - -Cozystack is a Kubernetes-based PaaS built on Helm umbrella charts and FluxCD. -Packages live in `packages/{core,system,apps,extra,library,tests}/`. The `library/` group holds reusable helper charts; the `tests/` group exists because library charts are not directly testable. -Each package wraps one or more upstream Helm charts. -Go code in `cmd/`, `internal/`, `pkg/` implements Kubernetes controllers and API server. -Static CRDs are generated by `hack/update-codegen.sh` from types under `api/v1alpha1/`, `api/backups/`, and `api/dashboard/`. -App Kinds (like `Postgres`, `Kafka`) live in `api/apps/v1alpha1/` but are registered dynamically at runtime from `ApplicationDefinition` resources — they are not static CRDs. - -## Vendored Code — Critical Rules - -**Never suggest editing files inside any `charts/` directory under `packages/`.** -Those are upstream Helm charts vendored via `make update` (which runs `helm pull`). -Any direct edit is overwritten on the next update and provides zero value. - -If you find an issue that appears to live in vendored chart code: - -- For configuration-level changes: suggest overrides in the package root `values.yaml`. -- For structural changes: suggest a patch file in `packages//patches/` applied by the Makefile. -- For source-code changes in images: suggest a patch in `packages//images//patches/`. -- For true upstream bugs: point to the upstream repository and suggest an upstream issue/PR. -- Do NOT suggest creating `charts/patches/` — patches never live inside `charts/`. - -Similarly, never propose edits to: - -- `vendor/` — Go dependencies. Changes go through `go get` and `go mod tidy`. -- `zz_generated.*.go` — regenerated by `make generate`. -- `pkg/generated/` — auto-generated Kubernetes client code. -- Image digest values in `values.yaml` — set by CI via `make image`, not by humans. -- `go.mod` / `go.sum` by hand — use `go get` and `go mod tidy`. - -## Commit and PR Requirements - -Each commit must follow [Conventional Commits](https://www.conventionalcommits.org/) format: `type(scope): brief description`. - -Valid types: `feat`, `fix`, `docs`, `style`, `refactor`, `perf`, `test`, `build`, `ci`, `chore`. - -Valid scopes: - -- System: `dashboard`, `platform`, `cilium`, `kube-ovn`, `linstor`, `fluxcd`, `cluster-api` -- Apps: `postgres`, `mariadb`, `redis`, `kafka`, `clickhouse`, `kubernetes`, `virtual-machine` -- Meta: `api`, `hack`, `tests`, `ci`, `docs` -- Package-specific: any `` matching a directory under `packages/` - -Breaking changes: append `!` after type/scope (`feat(api)!: ...`) or add a `BREAKING CHANGE:` footer. - -Each commit must have a `Signed-off-by:` trailer (produced by `git commit --signoff`). - -PR body must contain a release note block: - -````text -```release-note -type(scope): human-readable changelog entry -``` -```` - -Flag any PR whose commits lack the Conventional Commits format or signoff, or whose body has no release-note block. - -## Helm Chart Conventions - -Packages follow an umbrella chart pattern: - -- `charts/` — vendored upstream, read-only -- `templates/` — Cozystack-specific extra manifests -- `values.yaml` — override values for the upstream chart -- `values.schema.json` — JSON Schema for dashboard UI and input validation - -When reviewing `values.schema.json`: - -- `make generate` regenerates this file and strips `title`, `description`, and `x-*` custom annotations. Do not suggest adding fields that will be stripped on the next regeneration. -- Focus on type correctness, `required` fields, `enum` values, and default values. -- Flag breaking changes: removing a field, changing its type, or narrowing its enum. - -## Sensitive Components - -**`packages/core/platform/`**: the platform chart that deploys everything. Changes here can require updates to the migration flow — the Helm hook in `packages/core/platform/templates/migration-hook.yaml` and the runner image with scripts in `packages/core/platform/images/migrations/`. Flag any change to this package that lacks a corresponding migration update or an explicit note that backward compatibility is preserved. - -**`api/apps/v1alpha1/`**: app CRD Kinds are registered at runtime from `ApplicationDefinition` resources and the matching `values.schema.json`. Suggest changes to the relevant package schema rather than hand-editing generated types. - -**RBAC, ServiceAccounts, and SecurityContext**: flag overly broad RBAC (`*` on resources or verbs without justification), missing `securityContext`, containers running as root without an explicit reason, and `hostPath`/`hostNetwork` usage without clear rationale. - -## Go Code Standards - -- Use controller-runtime patterns for reconcilers. -- Use structured logging via `logr` — flag `fmt.Print*` and `log.Print*` in controller code. -- Handle errors explicitly. Discarding meaningful errors with `_` is a bug. -- Propagate `context.Context` through call chains. Flag `context.Background()` created inside a reconciler or request handler. -- Prefer `ctrl.Result{RequeueAfter: ...}` over empty requeue for predictable reconciliation loops. -- Tests live beside the code (`*_test.go`). New behavior without tests is worth flagging. - -## What to Review Carefully - -- Logic errors, off-by-one bugs, nil dereferences. -- Missing error handling, especially in reconcilers and API handlers. -- Helm template correctness: missing `quote`, incorrect indentation, wrong scope in `with`/`range`. -- Security: permissive RBAC, privileged containers, secrets in environment variables, hardcoded credentials. -- Missing resource requests/limits on new workloads. -- Breaking changes in `values.schema.json`: removed fields, tightened types, narrower enums. - -## Anti-patterns — Do Not Flag These - -- Large JSON files in `dashboards/` — imported from upstream Grafana sources, not hand-written. -- Files under any `charts/` directory — vendored upstream, left as-is intentionally. -- Whitespace or formatting in `*.patch` / `*.diff` files — machine-applied, not authored. -- Missing comments on generated code. -- `go.sum` changes accompanying `go.mod` changes — expected and correct. -- Fork relationships for vendored tooling images — intentional (e.g., `cozystack/kilo` fork is expected). -- Absence of unit tests for vendored chart overrides — covered by E2E tests in `hack/e2e-apps/`. diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2482712d..2045ec5a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -* @kvaps @lllamnyp @lexfrei @androndo @IvanHunters @sircthulhu @myasnikovdaniil +* @kvaps @lllamnyp @lexfrei @androndo @IvanHunters @sircthulhu diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index fefdab82..1a119259 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,7 +1,7 @@ --- name: Bug report about: Create a report to help us improve -labels: 'kind/bug' +labels: 'bug' assignees: '' --- diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 044dd6a0..9a52457c 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,11 +1,8 @@ - ### Release note ```release-note - +[] ``` \ No newline at end of file diff --git a/.github/labels.yml b/.github/labels.yml deleted file mode 100644 index 2ce04130..00000000 --- a/.github/labels.yml +++ /dev/null @@ -1,371 +0,0 @@ -# Cozystack repository labels -# -# Label conventions follow the Kubernetes scheme: -# https://github.com/kubernetes/test-infra/blob/master/label_sync/labels.md -# -# Synced into the repository by .github/workflows/labels.yaml -# (EndBug/label-sync@v2). Edit this file via pull request — UI changes -# will be overwritten on the next sync. -# -# Constraints (enforced by the validate job in labels.yaml): -# - description ≤ 100 characters (GitHub REST API limit) -# - color is a 6-character hex string (no leading #) -# - label names are unique -# - aliases do not collide with top-level names -# -# Categories: -# kind/ issue or PR type -# priority/ urgency -# triage/ review state -# lifecycle/ issue or PR lifecycle -# area/ subsystem; extensible — add when 3+ open issues exist -# do-not-merge/ PR merge blockers -# security/ security-finding severity and status (Cozystack-specific) -# size: PR size (auto-applied) -# -# `aliases:` lets EndBug/label-sync rename existing labels without losing -# references on already-tagged issues and PRs. -# -# GitHub-default labels not migrated here (`wontfix`, `invalid`) currently -# carry zero issues/PRs in this repo and will be removed in a follow-up -# cleanup PR rather than aliased to a different namespace. - -# ────────────────────────────────────────────── -# kind/ — issue or PR type -# ────────────────────────────────────────────── - -- name: kind/bug - color: 'd73a4a' - description: Categorizes issue or PR as related to a bug - aliases: ['bug'] - -- name: kind/feature - color: 'a2eeef' - description: Categorizes issue or PR as related to a new feature - aliases: ['enhancement'] - -- name: kind/documentation - color: '0075ca' - description: Categorizes issue or PR as related to documentation - aliases: ['documentation'] - -- name: kind/support - color: 'd876e3' - description: Categorizes issue as a support question - aliases: ['question'] - -- name: kind/cleanup - color: 'c7def8' - description: Categorizes issue or PR as related to cleanup of code, process, or technical debt - -- name: kind/regression - color: 'e11d21' - description: Categorizes issue or PR as related to a regression from a prior release - -- name: kind/flake - color: 'f7c6c7' - description: Categorizes issue or PR as related to a flaky test - -- name: kind/failing-test - color: 'e11d21' - description: Categorizes issue or PR as related to a consistently or frequently failing test - -- name: kind/api-change - color: 'c7def8' - description: Categorizes issue or PR as related to adding, removing, or otherwise changing an API - -- name: kind/breaking-change - color: 'e11d21' - description: Indicates the change introduces a breaking API or behaviour change - -# ────────────────────────────────────────────── -# priority/ — urgency -# ────────────────────────────────────────────── - -- name: priority/critical-urgent - color: 'e11d21' - description: Highest priority. Must be actively worked on as someone's top priority right now - -- name: priority/important-soon - color: 'eb6420' - description: Must be staffed and worked on either currently, or very soon, ideally in time for the next release - -- name: priority/important-longterm - color: 'fbca04' - description: Important over the long term, but may not be staffed and/or may need multiple releases to complete - -- name: priority/backlog - color: 'fef2c0' - description: General backlog priority. Lower than priority/important-longterm - -# ────────────────────────────────────────────── -# triage/ — review state -# ────────────────────────────────────────────── - -- name: triage/needs-triage - color: 'ededed' - description: Indicates an issue needs triage by a maintainer - -- name: triage/accepted - color: '0e8a16' - description: Indicates an issue is ready to be actively worked on - -- name: triage/needs-information - color: 'fbca04' - description: Indicates an issue needs more information in order to work on it - -- name: triage/not-reproducible - color: 'fbca04' - description: Indicates an issue can not be reproduced as described - -- name: triage/duplicate - color: 'cfd3d7' - description: Indicates an issue is a duplicate of another issue - aliases: ['duplicate'] - -- name: triage/unresolved - color: 'cfd3d7' - description: Indicates an issue that can not or will not be resolved - -# ────────────────────────────────────────────── -# lifecycle/ — issue or PR lifecycle -# ────────────────────────────────────────────── - -- name: lifecycle/active - color: '1d76db' - description: Indicates that an issue or PR is actively being worked on by a contributor - -- name: lifecycle/frozen - color: 'db5dd6' - description: Indicates that an issue or PR should not be auto-closed due to staleness - aliases: ['frozen'] - -- name: lifecycle/stale - color: 'dadada' - description: Denotes an issue or PR has remained open with no activity and has become stale - aliases: ['stale'] - -- name: lifecycle/rotten - color: '795548' - description: Denotes an issue or PR that has aged beyond stale and will be auto-closed - -# ────────────────────────────────────────────── -# area/ — subsystem (extensible) -# Add a new area/* when there are 3+ open issues on the topic. -# ────────────────────────────────────────────── - -- name: area/api - color: 'bfd4f2' - description: Issues or PRs related to the cozystack-api aggregated API server - -- name: area/ai - color: 'bfd4f2' - description: Issues or PRs related to AI agent guides, AGENTS.md, docs/agents/ - -- name: area/build - color: 'bfd4f2' - description: Issues or PRs related to image build infrastructure, multi-arch support - -- name: area/ci - color: 'bfd4f2' - description: Issues or PRs related to CI workflows, GitHub Actions, automation - -- name: area/dashboard - color: 'bfd4f2' - description: Issues or PRs related to the dashboard / UI - -- name: area/extra - color: 'bfd4f2' - description: Issues or PRs related to tenant-specific modules (packages/extra/) - -- name: area/database - color: 'bfd4f2' - description: Issues or PRs related to managed databases (postgres, mariadb, redis, etcd, kafka, clickhouse) - -- name: area/kubernetes - color: 'bfd4f2' - description: Issues or PRs related to the tenant Kubernetes app - -- name: area/monitoring - color: 'bfd4f2' - description: Issues or PRs related to the monitoring stack (vlogs, vmstack, grafana, workloadmonitor) - -- name: area/networking - color: 'bfd4f2' - description: Issues or PRs related to networking (ingress, gateway, vpn, metallb, cilium, kube-ovn) - -- name: area/platform - color: 'bfd4f2' - description: Issues or PRs related to platform infrastructure (bundle, flux, talos, installer) - -- name: area/release - color: 'bfd4f2' - description: Issues or PRs related to release tooling (changelog, backport, release pipeline) - -- name: area/storage - color: 'bfd4f2' - description: Issues or PRs related to storage (linstor, seaweedfs, bucket, velero, harbor) - -- name: area/testing - color: 'bfd4f2' - description: Issues or PRs related to testing (e2e, bats, unit tests) - -- name: area/virtualization - color: 'bfd4f2' - description: Issues or PRs related to virtualization (kubevirt, cdi, vmi, vm-import) - -- name: area/uncategorized - color: 'fbca04' - description: PR auto-labeler could not map title scope to a known area/*; please review - -# ────────────────────────────────────────────── -# do-not-merge/ — PR merge blockers (Prow convention) -# ────────────────────────────────────────────── - -- name: do-not-merge/work-in-progress - color: 'e11d21' - description: Indicates that a PR should not merge because it is a work in progress - # Both legacy spellings collapse here. EndBug processes aliases sequentially; - # the second rename hits a name collision and logs a warning — the legacy - # label survives and gets cleaned up in the follow-up dedup PR. - aliases: ['do-not-merge', 'do not merge'] - -- name: do-not-merge/hold - color: 'e11d21' - description: Indicates that a PR should not merge because someone has issued /hold - -# ────────────────────────────────────────────── -# Cozystack-specific (preserved) -# ────────────────────────────────────────────── - -- name: epic - color: 'a335ee' - description: A large development increment that brings definite value to Cozystack users - -- name: community - color: '97458a' - description: Community contributions are welcome in this issue - -- name: help wanted - color: '008672' - description: Extra attention is needed - -- name: good first issue - color: '7057ff' - description: Good for newcomers - -- name: quality-of-life - color: 'aaaaaa' - description: QoL improvements - -- name: upstream-issue - color: 'aaaaaa' - description: Requires resolving an issue in an upstream project - -- name: backport - color: 'fbca04' - description: Should change be backported on previous release - -- name: backport-previous - color: 'fbd876' - description: Backport target — previous release line - -- name: release - color: 'aaaaaa' - description: Releasing a new Cozystack version - -- name: automated - color: 'ededed' - description: Created by automation - -- name: debug - color: '704479' - description: Debugging in progress - -- name: sponsored - color: '00ff00' - description: Sponsored work - -- name: lgtm - color: '238636' - description: This PR has been approved by a maintainer - -- name: ok-to-test - color: '00ff00' - description: Indicates a non-member PR is safe to run CI on - -# ────────────────────────────────────────────── -# size: — PR size (auto-applied by sizing bot) -# ────────────────────────────────────────────── - -- name: 'size:XS' - color: '00ff00' - description: This PR changes 0-9 lines, ignoring generated files - -- name: 'size:S' - color: '77b800' - description: This PR changes 10-29 lines, ignoring generated files - -- name: 'size:M' - color: 'ebb800' - description: This PR changes 30-99 lines, ignoring generated files - -- name: 'size:L' - color: 'eb9500' - description: This PR changes 100-499 lines, ignoring generated files - -- name: 'size:XL' - color: 'ff823f' - description: This PR changes 500-999 lines, ignoring generated files - -- name: 'size:XXL' - color: 'ffb8b8' - description: This PR changes 1000+ lines, ignoring generated files - -# ────────────────────────────────────────────── -# security/ — security-finding severity and status -# ────────────────────────────────────────────── - -- name: security - color: 'aaaaaa' - description: Security-related issues and features - -- name: security/critical - color: 'd73a4a' - description: Critical security vulnerability - -- name: security/high - color: 'e99695' - description: High severity security finding - -- name: security/medium - color: 'f9c513' - description: Medium severity security finding - -- name: security/low - color: '0e8a16' - description: Low severity security finding - -- name: security/triage-needed - color: 'fbca04' - description: Needs security triage - -- name: security/confirmed - color: '1d76db' - description: Confirmed vulnerability - -- name: security/false-positive - color: 'c5def5' - description: Triaged as false positive - -- name: security/accepted-risk - color: 'bfd4f2' - description: Risk accepted with justification - -- name: security/in-progress - color: '0075ca' - description: Fix in progress - -- name: security/fixed - color: '0e8a16' - description: Fix released diff --git a/.github/workflows/auto-release.yaml b/.github/workflows/auto-release.yaml index 44ca8ca6..19d4b460 100644 --- a/.github/workflows/auto-release.yaml +++ b/.github/workflows/auto-release.yaml @@ -20,14 +20,6 @@ jobs: pull-requests: read steps: - - name: Generate GitHub App token - id: app-token - uses: actions/create-github-app-token@v1 - with: - app-id: ${{ secrets.COZYSTACK_CI_APP_ID }} - private-key: ${{ secrets.COZYSTACK_CI_PRIVATE_KEY }} - owner: cozystack - - name: Checkout code uses: actions/checkout@v4 with: @@ -36,27 +28,27 @@ jobs: - name: Configure git env: - APP_TOKEN: ${{ steps.app-token.outputs.token }} + GH_PAT: ${{ secrets.GH_PAT }} run: | - git config user.name "cozystack-ci[bot]" - git config user.email "274107086+cozystack-ci[bot]@users.noreply.github.com" - git remote set-url origin https://x-access-token:${APP_TOKEN}@github.com/${GITHUB_REPOSITORY} + git config user.name "cozystack-bot" + git config user.email "217169706+cozystack-bot@users.noreply.github.com" + git remote set-url origin https://cozystack-bot:${GH_PAT}@github.com/${GITHUB_REPOSITORY} git config --unset-all http.https://github.com/.extraheader || true - name: Process release branches uses: actions/github-script@v7 env: - APP_TOKEN: ${{ steps.app-token.outputs.token }} + GH_PAT: ${{ secrets.GH_PAT }} with: - github-token: ${{ steps.app-token.outputs.token }} + github-token: ${{ secrets.GH_PAT }} script: | const { execSync } = require('child_process'); - // Configure git to use GitHub App token for authentication - execSync('git config user.name "cozystack-ci[bot]"', { encoding: 'utf8' }); - execSync('git config user.email "274107086+cozystack-ci[bot]@users.noreply.github.com"', { encoding: 'utf8' }); - execSync(`git remote set-url origin https://x-access-token:${process.env.APP_TOKEN}@github.com/${process.env.GITHUB_REPOSITORY}`, { encoding: 'utf8' }); - // Remove GITHUB_TOKEN extraheader to ensure App token is used (needed to trigger other workflows) + // Configure git to use PAT for authentication + execSync('git config user.name "cozystack-bot"', { encoding: 'utf8' }); + execSync('git config user.email "217169706+cozystack-bot@users.noreply.github.com"', { encoding: 'utf8' }); + execSync(`git remote set-url origin https://cozystack-bot:${process.env.GH_PAT}@github.com/${process.env.GITHUB_REPOSITORY}`, { encoding: 'utf8' }); + // Remove GITHUB_TOKEN extraheader to ensure PAT is used (needed to trigger other workflows) execSync('git config --unset-all http.https://github.com/.extraheader || true', { encoding: 'utf8' }); // Get all release-X.Y branches diff --git a/.github/workflows/codegen-drift.yml b/.github/workflows/codegen-drift.yml deleted file mode 100644 index a26caf5e..00000000 --- a/.github/workflows/codegen-drift.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: Codegen Drift Check - -on: - pull_request: - types: [opened, synchronize, reopened] - paths: - - 'api/**' - - 'pkg/apis/**' - - 'pkg/generated/**' - - 'internal/crdinstall/manifests/**' - - 'packages/system/cozystack-controller/definitions/**' - - 'packages/system/application-definition-crd/definition/**' - - 'packages/system/backup-controller/definitions/**' - - 'packages/system/backupstrategy-controller/definitions/**' - - 'hack/update-codegen.sh' - - 'hack/boilerplate.go.txt' - - 'Makefile' - - 'go.mod' - - 'go.sum' - - '.github/workflows/codegen-drift.yml' - -concurrency: - group: codegen-drift-${{ github.workflow }}-${{ github.event.pull_request.number }} - cancel-in-progress: true - -jobs: - codegen-drift: - name: Verify generated code is up to date - runs-on: ubuntu-22.04 - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version-file: go.mod - cache: true - - - name: Pre-fetch k8s.io/code-generator module - # hack/update-codegen.sh sources kube_codegen.sh from the Go module cache. - # The module is not declared in go.mod, so fetch it explicitly at the - # version pinned in the script. - run: | - version=$(grep -oP 'code-generator@\Kv[0-9.]+' hack/update-codegen.sh) - tmpdir=$(mktemp -d) - cd "$tmpdir" - go mod init codegen-fetch - go get "k8s.io/code-generator@${version}" - - - name: Run make generate - run: make generate - - - name: Fail on drift - run: | - if [ -n "$(git status --porcelain)" ]; then - echo "::error::'make generate' produced changes. Run 'make generate' locally and commit the result." - git status --short - git diff --color=always - exit 1 - fi diff --git a/.github/workflows/labels.yaml b/.github/workflows/labels.yaml deleted file mode 100644 index 69c18329..00000000 --- a/.github/workflows/labels.yaml +++ /dev/null @@ -1,84 +0,0 @@ -name: Labels - -on: - pull_request: - paths: - - .github/labels.yml - - .github/workflows/labels.yaml - push: - branches: [main] - paths: - - .github/labels.yml - - .github/workflows/labels.yaml - workflow_dispatch: - schedule: - - cron: '17 4 * * 1' # Mondays at 04:17 UTC - -permissions: - contents: read - -concurrency: - group: labels-sync - cancel-in-progress: false - -jobs: - validate: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Validate labels.yml schema - run: | - python3 - <<'PY' - import re, sys, yaml - - path = '.github/labels.yml' - data = yaml.safe_load(open(path)) - errors = [] - - # 1. description ≤ 100 chars (GitHub REST API limit) - for label in data: - desc = label.get('description', '') or '' - if len(desc) > 100: - errors.append(f"{label['name']}: description {len(desc)} chars (max 100)") - - # 2. color is 6-char hex without leading # - for label in data: - color = label.get('color', '') or '' - if not re.match(r'^[0-9A-Fa-f]{6}$', color): - errors.append(f"{label['name']}: bad color {color!r} (must be 6-char hex without #)") - - # 3. unique top-level names - names = [label['name'] for label in data] - dups = sorted({n for n in names if names.count(n) > 1}) - for n in dups: - errors.append(f"duplicate name: {n}") - - # 4. aliases do not collide with any top-level name - name_set = set(names) - for label in data: - for alias in (label.get('aliases') or []): - if alias in name_set: - errors.append(f"alias {alias!r} (under {label['name']!r}) collides with a top-level name") - - if errors: - for err in errors: - print(f"::error::{err}") - sys.exit(1) - - print(f"labels.yml schema OK ({len(data)} labels)") - PY - - sync: - needs: validate - if: github.event_name != 'pull_request' - runs-on: ubuntu-latest - permissions: - issues: write - pull-requests: write - steps: - - uses: actions/checkout@v4 - - uses: EndBug/label-sync@v2 - with: - config-file: .github/labels.yml - delete-other-labels: false diff --git a/.github/workflows/pr-labeler.yaml b/.github/workflows/pr-labeler.yaml deleted file mode 100644 index 6e6f3017..00000000 --- a/.github/workflows/pr-labeler.yaml +++ /dev/null @@ -1,206 +0,0 @@ -name: PR Auto-Label - -on: - pull_request_target: - types: [opened, edited, reopened, synchronize] - -permissions: - contents: read - pull-requests: write - -jobs: - label: - runs-on: ubuntu-latest - steps: - - name: Apply labels from PR title - uses: actions/github-script@v7 - with: - script: | - // Conventional Commits types accepted by Cozystack (per docs/agents/contributing.md): - // feat, fix, docs, style, refactor, perf, test, build, ci, chore, revert - // Mapping below maps a subset to kind/* — types not listed do not produce a kind/*. - const typeToKind = { - feat: 'kind/feature', - fix: 'kind/bug', - docs: 'kind/documentation', - chore: 'kind/cleanup', - refactor: 'kind/cleanup', - // style, perf, test, build, ci, revert — no kind mapping - }; - - // scope -> area/* mapping. Keys are the scopes observed in cozystack issues - // and PRs. Add new entries when a scope recurs (3+ times). - const scopeToArea = { - // area/api - 'api': 'area/api', - 'cozystack-api': 'area/api', - - // area/ai - 'agents': 'area/ai', - 'ai': 'area/ai', - - // area/build - 'build': 'area/build', - - // area/ci - 'ci': 'area/ci', - - // area/dashboard - 'dashboard': 'area/dashboard', - - // area/database - 'postgres': 'area/database', - 'postgres-operator': 'area/database', - 'mariadb': 'area/database', - 'mariadb-operator': 'area/database', - 'redis': 'area/database', - 'etcd': 'area/database', - 'kafka': 'area/database', - 'clickhouse': 'area/database', - - // area/extra - 'extra': 'area/extra', - - // area/kubernetes - 'kubernetes': 'area/kubernetes', - 'apps/kubernetes': 'area/kubernetes', - - // area/monitoring - 'monitoring': 'area/monitoring', - 'vlogs': 'area/monitoring', - 'vmstack': 'area/monitoring', - 'grafana': 'area/monitoring', - 'workloadmonitor': 'area/monitoring', - - // area/networking - 'ingress': 'area/networking', - 'ingress-nginx': 'area/networking', - 'gateway': 'area/networking', - 'vpn': 'area/networking', - 'metallb': 'area/networking', - 'cilium': 'area/networking', - 'kube-ovn': 'area/networking', - 'tcp-balancer': 'area/networking', - 'securitygroups': 'area/networking', - 'cozy-proxy': 'area/networking', - - // area/platform - 'platform': 'area/platform', - 'bundle': 'area/platform', - 'flux': 'area/platform', - 'fluxcd': 'area/platform', - 'cluster-api': 'area/platform', - 'talos': 'area/platform', - 'installer': 'area/platform', - 'cozyctl': 'area/platform', - 'cozystack-engine': 'area/platform', - 'cozy-lib': 'area/platform', - - // area/release - 'backport': 'area/release', - 'release': 'area/release', - - // area/storage - 'seaweedfs': 'area/storage', - 'seaweedfs-cosi-driver': 'area/storage', - 'bucket': 'area/storage', - 'linstor': 'area/storage', - 'velero': 'area/storage', - 'harbor': 'area/storage', - 'backups': 'area/storage', - - // area/testing - 'tests': 'area/testing', - 'e2e': 'area/testing', - - // area/virtualization - 'kubevirt': 'area/virtualization', - 'cdi': 'area/virtualization', - 'vmi': 'area/virtualization', - 'vm-import': 'area/virtualization', - 'virtual-machine': 'area/virtualization', - 'hami': 'area/virtualization', - 'gpu-operator': 'area/virtualization', - }; - - const pr = context.payload.pull_request; - const title = pr.title || ''; - const body = pr.body || ''; - const existing = new Set((pr.labels || []).map(l => l.name)); - const toAdd = new Set(); - - // 1. Strip "[Backport release-1.x]" prefix if present. - const backportMatch = title.match(/^\[Backport ([^\]]+)\]\s+(.+)$/); - const cleanTitle = backportMatch ? backportMatch[2] : title; - if (backportMatch) { - toAdd.add('area/release'); - toAdd.add('backport'); - } - - // 2. Try Conventional Commits form: type(scope)?(!)?: description - const conv = cleanTitle.match(/^([a-z]+)(?:\(([^)]+)\))?(!)?:\s*.+$/); - // 3. Fall back to bracket form: [scope] description - const bracket = !conv && cleanTitle.match(/^\[([^\]]+)\]\s+.+$/); - - let type = null, scopeStr = null, breaking = false; - if (conv) { - type = conv[1]; - scopeStr = conv[2] || null; - breaking = !!conv[3]; - } else if (bracket) { - scopeStr = bracket[1]; - } - - // 4. Detect BREAKING CHANGE: or BREAKING-CHANGE: footer in body. - // Conventional Commits 1.0 spec item 16 treats them as synonymous. - if (/^BREAKING[ -]CHANGE:/m.test(body)) { - breaking = true; - } - - // 5. Apply kind/* from type. - if (type) { - if (typeToKind[type]) { - toAdd.add(typeToKind[type]); - } else { - core.warning(`type "${type}" has no kind/* mapping — typo or new type? See .github/workflows/pr-labeler.yaml typeToKind`); - } - } - - // 6. Apply area/* from scope. Composite scopes split on comma. - const scopes = (scopeStr || '') - .split(/,\s*/) - .map(s => s.trim()) - .filter(Boolean); - for (const s of scopes) { - if (scopeToArea[s]) { - toAdd.add(scopeToArea[s]); - } else { - core.warning(`scope "${s}" has no area/* mapping — consider extending scopeToArea in .github/workflows/pr-labeler.yaml if it recurs`); - } - } - - // 7. kind/breaking-change. - if (breaking) { - toAdd.add('kind/breaking-change'); - } - - // 8. Fallback: no area/* applied -> area/uncategorized. - const hasArea = [...toAdd].some(l => l.startsWith('area/')); - if (!hasArea) { - toAdd.add('area/uncategorized'); - } - - // 9. Additive only — never remove existing labels. - const newLabels = [...toAdd].filter(l => !existing.has(l)); - if (newLabels.length === 0) { - core.info('No new labels to apply'); - return; - } - - await github.rest.issues.addLabels({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: pr.number, - labels: newLabels, - }); - core.info(`Applied labels: ${newLabels.join(', ')}`); diff --git a/.github/workflows/pull-requests-release.yaml b/.github/workflows/pull-requests-release.yaml index bb7f5da5..72f31b54 100644 --- a/.github/workflows/pull-requests-release.yaml +++ b/.github/workflows/pull-requests-release.yaml @@ -23,14 +23,6 @@ jobs: contains(github.event.pull_request.labels.*.name, 'release') steps: - - name: Generate GitHub App token - id: app-token - uses: actions/create-github-app-token@v1 - with: - app-id: ${{ secrets.COZYSTACK_CI_APP_ID }} - private-key: ${{ secrets.COZYSTACK_CI_PRIVATE_KEY }} - owner: cozystack - # Extract tag from branch name (branch = release-X.Y.Z*) - name: Extract tag from branch name id: get_tag @@ -55,11 +47,11 @@ jobs: - name: Create tag on merge commit env: - APP_TOKEN: ${{ steps.app-token.outputs.token }} + GH_PAT: ${{ secrets.GH_PAT }} run: | - git config user.name "cozystack-ci[bot]" - git config user.email "274107086+cozystack-ci[bot]@users.noreply.github.com" - git remote set-url origin https://x-access-token:${APP_TOKEN}@github.com/${GITHUB_REPOSITORY} + git config user.name "cozystack-bot" + git config user.email "217169706+cozystack-bot@users.noreply.github.com" + git remote set-url origin https://cozystack-bot:${GH_PAT}@github.com/${GITHUB_REPOSITORY} git tag -f ${{ steps.get_tag.outputs.tag }} ${{ github.sha }} git push -f origin ${{ steps.get_tag.outputs.tag }} @@ -67,7 +59,7 @@ jobs: - name: Ensure maintenance branch release-X.Y uses: actions/github-script@v7 with: - github-token: ${{ steps.app-token.outputs.token }} + github-token: ${{ secrets.GH_PAT }} script: | const tag = '${{ steps.get_tag.outputs.tag }}'; // e.g. v0.1.3 or v0.1.3-rc3 const match = tag.match(/^v(\d+)\.(\d+)\.\d+(?:[-\w\.]+)?$/); diff --git a/.github/workflows/pull-requests.yaml b/.github/workflows/pull-requests.yaml index d9898616..76dbfc17 100644 --- a/.github/workflows/pull-requests.yaml +++ b/.github/workflows/pull-requests.yaml @@ -99,14 +99,6 @@ jobs: disk_id: ${{ steps.fetch_assets.outputs.disk_id }} steps: - - name: Generate GitHub App token - id: app-token - uses: actions/create-github-app-token@v1 - with: - app-id: ${{ secrets.COZYSTACK_CI_APP_ID }} - private-key: ${{ secrets.COZYSTACK_CI_PRIVATE_KEY }} - owner: cozystack - - name: Checkout code if: contains(github.event.pull_request.labels.*.name, 'release') uses: actions/checkout@v4 @@ -133,7 +125,7 @@ jobs: id: fetch_assets uses: actions/github-script@v7 with: - github-token: ${{ steps.app-token.outputs.token }} + github-token: ${{ secrets.GH_PAT }} script: | const tag = '${{ steps.get_tag.outputs.tag }}'; const releases = await github.rest.repos.listReleases({ @@ -167,15 +159,6 @@ jobs: if: ${{ always() && (needs.build.result == 'success' || needs.resolve_assets.result == 'success') }} steps: - - name: Generate GitHub App token - if: contains(github.event.pull_request.labels.*.name, 'release') - id: app-token - uses: actions/create-github-app-token@v1 - with: - app-id: ${{ secrets.COZYSTACK_CI_APP_ID }} - private-key: ${{ secrets.COZYSTACK_CI_PRIVATE_KEY }} - owner: cozystack - # ▸ Checkout and prepare the codebase - name: Checkout code uses: actions/checkout@v4 @@ -205,11 +188,11 @@ jobs: if: contains(github.event.pull_request.labels.*.name, 'release') run: | mkdir -p _out/assets - curl -sSL -H "Authorization: token ${APP_TOKEN}" -H "Accept: application/octet-stream" \ + curl -sSL -H "Authorization: token ${GH_PAT}" -H "Accept: application/octet-stream" \ -o _out/assets/nocloud-amd64.raw.xz \ "https://api.github.com/repos/${GITHUB_REPOSITORY}/releases/assets/${{ needs.resolve_assets.outputs.disk_id }}" env: - APP_TOKEN: ${{ steps.app-token.outputs.token }} + GH_PAT: ${{ secrets.GH_PAT }} - name: Set sandbox ID run: echo "SANDBOX_NAME=cozy-e2e-sandbox-$(echo "${GITHUB_REPOSITORY}:${GITHUB_WORKFLOW}:${GITHUB_REF}" | sha256sum | cut -c1-10)" >> $GITHUB_ENV diff --git a/.github/workflows/tags.yaml b/.github/workflows/tags.yaml index 1b68ee32..2bbdb7a6 100644 --- a/.github/workflows/tags.yaml +++ b/.github/workflows/tags.yaml @@ -25,14 +25,6 @@ jobs: actions: write steps: - - name: Generate GitHub App token - id: app-token - uses: actions/create-github-app-token@v1 - with: - app-id: ${{ secrets.COZYSTACK_CI_APP_ID }} - private-key: ${{ secrets.COZYSTACK_CI_PRIVATE_KEY }} - owner: cozystack - # Check if a non-draft release with this tag already exists - name: Check if release already exists id: check_release @@ -123,11 +115,11 @@ jobs: - name: Commit release artifacts if: steps.check_release.outputs.skip == 'false' env: - APP_TOKEN: ${{ steps.app-token.outputs.token }} + GH_PAT: ${{ secrets.GH_PAT }} run: | - git config user.name "cozystack-ci[bot]" - git config user.email "274107086+cozystack-ci[bot]@users.noreply.github.com" - git remote set-url origin https://x-access-token:${APP_TOKEN}@github.com/${GITHUB_REPOSITORY} + git config user.name "cozystack-bot" + git config user.email "217169706+cozystack-bot@users.noreply.github.com" + git remote set-url origin https://cozystack-bot:${GH_PAT}@github.com/${GITHUB_REPOSITORY} git config --unset-all http.https://github.com/.extraheader || true git add . git commit -m "Prepare release ${GITHUB_REF#refs/tags/}" -s || echo "No changes to commit" @@ -136,13 +128,13 @@ jobs: - name: Tag API submodule if: steps.check_release.outputs.skip == 'false' env: - APP_TOKEN: ${{ steps.app-token.outputs.token }} + GH_PAT: ${{ secrets.GH_PAT }} run: | VTAG="${{ steps.tag.outputs.tag }}" SUBTAG="api/apps/v1alpha1/${VTAG}" - git config user.name "cozystack-ci[bot]" - git config user.email "274107086+cozystack-ci[bot]@users.noreply.github.com" - git remote set-url origin https://x-access-token:${APP_TOKEN}@github.com/${GITHUB_REPOSITORY} + git config user.name "cozystack-bot" + git config user.email "217169706+cozystack-bot@users.noreply.github.com" + git remote set-url origin https://cozystack-bot:${GH_PAT}@github.com/${GITHUB_REPOSITORY} TARGET="$(git rev-parse "${VTAG}^{}")" git tag -f "${SUBTAG}" "$TARGET" git push -f origin "refs/tags/${SUBTAG}" @@ -191,11 +183,11 @@ jobs: - name: Create release branch if: steps.check_release.outputs.skip == 'false' env: - APP_TOKEN: ${{ steps.app-token.outputs.token }} + GH_PAT: ${{ secrets.GH_PAT }} run: | - git config user.name "cozystack-ci[bot]" - git config user.email "274107086+cozystack-ci[bot]@users.noreply.github.com" - git remote set-url origin https://x-access-token:${APP_TOKEN}@github.com/${GITHUB_REPOSITORY} + git config user.name "cozystack-bot" + git config user.email "217169706+cozystack-bot@users.noreply.github.com" + git remote set-url origin https://cozystack-bot:${GH_PAT}@github.com/${GITHUB_REPOSITORY} BRANCH="release-${GITHUB_REF#refs/tags/v}" git branch -f "$BRANCH" git push -f origin "$BRANCH" @@ -205,7 +197,7 @@ jobs: if: steps.check_release.outputs.skip == 'false' uses: actions/github-script@v7 with: - github-token: ${{ steps.app-token.outputs.token }} + github-token: ${{ secrets.GH_PAT }} script: | const version = context.ref.replace('refs/tags/v', ''); const base = '${{ steps.get_base.outputs.branch }}'; @@ -223,7 +215,7 @@ jobs: repo: context.repo.repo, head, base, - title: `chore(release): cut v${version}`, + title: `Release v${version}`, body: `This PR prepares the release \`v${version}\`.`, draft: false }); @@ -247,29 +239,6 @@ jobs: pull-requests: write if: needs.prepare-release.result == 'success' steps: - - name: Generate GitHub App token - id: app-token - uses: actions/create-github-app-token@v1 - with: - app-id: ${{ secrets.COZYSTACK_CI_APP_ID }} - private-key: ${{ secrets.COZYSTACK_CI_PRIVATE_KEY }} - owner: cozystack - - # Read-only token for the AI step. Minting a separate scoped token - # means the Generate changelog using AI step cannot push branches, - # open PRs, or mutate any repository even with --allow-all-tools, - # regardless of whether the agent follows the prompt's instructions. - - name: Generate read-only GitHub App token - id: app-token-read - uses: actions/create-github-app-token@v1 - with: - app-id: ${{ secrets.COZYSTACK_CI_APP_ID }} - private-key: ${{ secrets.COZYSTACK_CI_PRIVATE_KEY }} - owner: cozystack - permission-contents: read - permission-pull-requests: read - permission-metadata: read - - name: Parse tag id: tag uses: actions/github-script@v7 @@ -292,7 +261,7 @@ jobs: ref: main fetch-depth: 0 fetch-tags: true - token: ${{ steps.app-token.outputs.token }} + token: ${{ secrets.GH_PAT }} - name: Check if changelog already exists id: check_changelog @@ -318,65 +287,56 @@ jobs: - name: Generate changelog using AI if: steps.check_changelog.outputs.exists == 'false' - timeout-minutes: 30 env: COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} - GH_TOKEN: ${{ steps.app-token-read.outputs.token }} - VERSION: ${{ steps.tag.outputs.version }} + GH_TOKEN: ${{ secrets.GH_PAT }} run: | - copilot \ - --prompt "Generate the release changelog for tag v${VERSION}. Follow the instructions in @docs/agents/changelog.md exactly, including the 'Scope and boundaries' section at the top. Your deliverable is the single file docs/changelogs/v${VERSION}.md — write it and exit; this workflow handles branching, committing, pushing, and opening the PR." \ + copilot --prompt "prepare changelog file for tagged release v${{ steps.tag.outputs.version }}, use @docs/agents/changelog.md for it. Create the changelog file at docs/changelogs/v${{ steps.tag.outputs.version }}.md" \ --allow-all-tools --allow-all-paths < /dev/null - name: Create changelog branch and commit if: steps.check_changelog.outputs.exists == 'false' env: - APP_TOKEN: ${{ steps.app-token.outputs.token }} - VERSION: ${{ steps.tag.outputs.version }} + GH_PAT: ${{ secrets.GH_PAT }} run: | - set -euo pipefail - - CHANGELOG_FILE="docs/changelogs/v${VERSION}.md" - CHANGELOG_BRANCH="changelog-v${VERSION}" - - if [ ! -f "$CHANGELOG_FILE" ]; then - echo "::error::Changelog file $CHANGELOG_FILE was not produced by the Generate changelog using AI step" + git config user.name "cozystack-bot" + git config user.email "217169706+cozystack-bot@users.noreply.github.com" + git remote set-url origin https://cozystack-bot:${GH_PAT}@github.com/${GITHUB_REPOSITORY} + + CHANGELOG_FILE="docs/changelogs/v${{ steps.tag.outputs.version }}.md" + CHANGELOG_BRANCH="changelog-v${{ steps.tag.outputs.version }}" + + if [ -f "$CHANGELOG_FILE" ]; then + # Fetch latest main branch + git fetch origin main + + # Delete local branch if it exists + git branch -D "$CHANGELOG_BRANCH" 2>/dev/null || true + + # Create and checkout new branch from main + git checkout -b "$CHANGELOG_BRANCH" origin/main + + # Add and commit changelog + git add "$CHANGELOG_FILE" + if git diff --staged --quiet; then + echo "⚠️ No changes to commit (file may already be committed)" + else + git commit -m "docs: add changelog for v${{ steps.tag.outputs.version }}" -s + echo "✅ Changelog committed to branch $CHANGELOG_BRANCH" + fi + + # Push the branch (force push to update if it exists) + git push -f origin "$CHANGELOG_BRANCH" + else + echo "⚠️ Changelog file was not generated" exit 1 fi - if [ ! -s "$CHANGELOG_FILE" ]; then - echo "::error::Changelog file $CHANGELOG_FILE is empty" - exit 1 - fi - - # Snapshot the file across the branch switch — the checkout below - # resets tracked files to match origin/main. - TEMP_FILE="$(mktemp)" - trap 'rm -f "$TEMP_FILE"' EXIT - cp "$CHANGELOG_FILE" "$TEMP_FILE" - - git config user.name "cozystack-ci[bot]" - git config user.email "274107086+cozystack-ci[bot]@users.noreply.github.com" - git remote set-url origin "https://x-access-token:${APP_TOKEN}@github.com/${GITHUB_REPOSITORY}" - - git fetch origin main - git branch -D "$CHANGELOG_BRANCH" 2>/dev/null || true - git checkout -b "$CHANGELOG_BRANCH" origin/main - - mkdir -p "$(dirname "$CHANGELOG_FILE")" - cp "$TEMP_FILE" "$CHANGELOG_FILE" - - # The `check_changelog` step gated this job on the file being absent - # from origin/main, so `git add` + `git commit` must produce a diff. - # If they don't, something is wrong (e.g. empty file) — fail loud. - git add "$CHANGELOG_FILE" - git commit -m "docs: add changelog for v${VERSION}" -s - git push -f origin "$CHANGELOG_BRANCH" - name: Create PR for changelog if: steps.check_changelog.outputs.exists == 'false' uses: actions/github-script@v7 with: - github-token: ${{ steps.app-token.outputs.token }} + github-token: ${{ secrets.GH_PAT }} script: | const version = '${{ steps.tag.outputs.version }}'; const changelogBranch = `changelog-v${version}`; @@ -411,7 +371,7 @@ jobs: repo: context.repo.repo, head: changelogBranch, base: baseBranch, - title: `docs(release): add changelog for v${version}`, + title: `docs: add changelog for v${version}`, body: `This PR adds the changelog for release \`v${version}\`.\n\n✅ Changelog has been automatically generated in \`docs/changelogs/v${version}.md\`.`, draft: false }); @@ -421,7 +381,7 @@ jobs: owner: context.repo.owner, repo: context.repo.repo, issue_number: pr.data.number, - labels: ['kind/documentation', 'automated'] + labels: ['documentation', 'automated'] }); console.log(`Created PR #${pr.data.number} for changelog`); @@ -435,14 +395,6 @@ jobs: permissions: contents: read steps: - - name: Generate GitHub App token - id: app-token - uses: actions/create-github-app-token@v1 - with: - app-id: ${{ secrets.COZYSTACK_CI_APP_ID }} - private-key: ${{ secrets.COZYSTACK_CI_PRIVATE_KEY }} - owner: cozystack - - name: Parse tag id: tag uses: actions/github-script@v7 @@ -462,61 +414,20 @@ jobs: uses: actions/checkout@v4 with: repository: cozystack/website - token: ${{ steps.app-token.outputs.token }} + token: ${{ secrets.GH_PAT }} ref: main - # Decide whether this release promotes the `next/` trunk to a new released - # version directory. Per the website repo's contract (see website#495): - # - `make release-next` runs only for new minor/major final releases - # (tag matches `vX.Y.Z` with no prerelease suffix AND `vX.Y/` does - # not yet exist on disk). - # - Prereleases and patch releases skip this step and let - # `make update-all` handle routing on its own. - - name: Determine if this release promotes next/ - id: promote - env: - TAG: ${{ steps.tag.outputs.tag }} - run: | - if [[ ! "$TAG" =~ ^v([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then - echo "promote=false" >> "$GITHUB_OUTPUT" - echo "Prerelease tag '$TAG' — skipping release-next." - exit 0 - fi - MAJOR="${BASH_REMATCH[1]}" - MINOR="${BASH_REMATCH[2]}" - if [[ "$MAJOR" == "0" ]]; then - DOC_VERSION="v0" - else - DOC_VERSION="v${MAJOR}.${MINOR}" - fi - if [[ -d "content/en/docs/${DOC_VERSION}" ]]; then - echo "promote=false" >> "$GITHUB_OUTPUT" - echo "content/en/docs/${DOC_VERSION}/ already exists — patch release, skipping release-next." - else - echo "promote=true" >> "$GITHUB_OUTPUT" - echo "New minor/major release ${DOC_VERSION} — will promote next/ → ${DOC_VERSION}/." - fi - - - name: Promote next/ to released version - if: steps.promote.outputs.promote == 'true' - env: - GH_TOKEN: ${{ steps.app-token.outputs.token }} - TAG: ${{ steps.tag.outputs.tag }} - run: make release-next RELEASE_TAG="$TAG" - - name: Update docs from release branch env: - GH_TOKEN: ${{ steps.app-token.outputs.token }} - TAG: ${{ steps.tag.outputs.tag }} - VERSION: ${{ steps.tag.outputs.version }} - run: make update-all BRANCH="release-$VERSION" RELEASE_TAG="$TAG" + GH_TOKEN: ${{ secrets.GH_PAT }} + run: make update-all BRANCH=release-${{ steps.tag.outputs.version }} RELEASE_TAG=${{ steps.tag.outputs.tag }} - name: Commit and push id: commit run: | - git config user.name "cozystack-ci[bot]" - git config user.email "274107086+cozystack-ci[bot]@users.noreply.github.com" - git add content hugo.yaml + git config user.name "cozystack-bot" + git config user.email "217169706+cozystack-bot@users.noreply.github.com" + git add content if git diff --cached --quiet; then echo "No changes to commit" echo "changed=false" >> $GITHUB_OUTPUT @@ -532,7 +443,7 @@ jobs: - name: Open pull request if: steps.commit.outputs.changed == 'true' env: - GH_TOKEN: ${{ steps.app-token.outputs.token }} + GH_TOKEN: ${{ secrets.GH_PAT }} run: | BRANCH="update-docs-v${{ steps.tag.outputs.version }}" pr_state=$(gh pr view "$BRANCH" --repo cozystack/website --json state --jq .state 2>/dev/null || echo "") diff --git a/.gitignore b/.gitignore index 61003de5..4e1e3119 100644 --- a/.gitignore +++ b/.gitignore @@ -83,4 +83,3 @@ tmp/ # build revision marker (generated by make image-packages) packages/core/platform/.build-revision -.claude/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6492b92d..ac0f7e30 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,17 +1,6 @@ repos: - repo: local hooks: - - id: run-make-generate-root - name: Run 'make generate' at repo root - entry: | - flock -x .git/pre-commit.lock sh -c ' - echo "Running make generate at repo root" - make generate || exit $? - git diff --color=always | cat - ' - language: system - files: ^(api/|pkg/apis/|pkg/generated/|internal/crdinstall/manifests/|packages/system/cozystack-controller/definitions/|packages/system/application-definition-crd/definition/|packages/system/backup-controller/definitions/|packages/system/backupstrategy-controller/definitions/|hack/update-codegen\.sh$|hack/boilerplate\.go\.txt$|Makefile$) - pass_filenames: false - id: run-make-generate name: Run 'make generate' in all app directories entry: | diff --git a/AGENTS.md b/AGENTS.md index 1e9c4d15..eb1febf7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -27,12 +27,6 @@ working with the **Cozystack** project. - Read: [`contributing.md`](./docs/agents/contributing.md) - Action: Read the file to understand git workflow, commit format, PR process -- **Issue and PR labeling, triage** (e.g., "label this issue", "what label should I use", "triage this", "categorize") - - Read: [`.github/labels.yml`](./.github/labels.yml) - - Action: Use labels defined there. Conventions follow the Kubernetes scheme — `kind/*` (type), `area/*` (subsystem), `priority/*` (urgency), `triage/*` (review state), `lifecycle/*` (auto-close), `do-not-merge/*` (PR blockers), `security/*` (severity) - - For `area/*`: accuracy outweighs reuse. If no existing `area/*` truly fits the change, propose a new one via PR (extend `.github/labels.yml` and the scope mapping in `.github/workflows/pr-labeler.yaml`) — do not shoehorn the change into a wrong area. `area/uncategorized` is the auto-labeler fallback; treat it as a signal to pick a fit, create a new area, or correct the PR title - - PR titles: a Conventional Commits header (`type(scope): description`, types from [`contributing.md`](./docs/agents/contributing.md)) auto-applies `kind/*` and `area/*` via `.github/workflows/pr-labeler.yaml`. Append `!` (or add a `BREAKING CHANGE:` footer) to apply `kind/breaking-change` - **Important rules:** - ✅ **ONLY read the file if the task matches the documented process scope** - do not read files for tasks that don't match their purpose - ✅ **ALWAYS read the file FIRST** before starting the task (when applicable) @@ -59,7 +53,7 @@ working with the **Cozystack** project. ### Conventions - **Helm Charts**: Umbrella pattern, vendored upstream charts in `charts/` - **Go Code**: Controller-runtime patterns, kubebuilder style -- **Git Commits**: Conventional Commits (`type(scope): description`) with `--signoff` +- **Git Commits**: `[component] Description` format with `--signoff` ### What NOT to Do - ❌ Edit `/vendor/`, `zz_generated.*.go`, upstream charts directly diff --git a/MAINTAINERS.md b/MAINTAINERS.md index d7061941..9c44daab 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -10,5 +10,3 @@ | Timur Tukaev | [@tym83](https://github.com/tym83) | Ænix | Cozystack Website, Marketing, Community Management | | Kirill Klinchenkov | [@klinch0](https://github.com/klinch0) | Ænix | Core Maintainer | | Nikita Bykov | [@nbykov0](https://github.com/nbykov0) | Ænix | Maintainer of ARM and stuff | -| Matthieu Robin | [@matthieu-robin](https://github.com/matthieu-robin) | Hidora | Managed Applications, Platform Quality & Benchmarking | -| Mattia Eleuteri | [@mattia-eleuteri](https://github.com/mattia-eleuteri) | Hidora | CSI, Storage, Networking & Security | diff --git a/Makefile b/Makefile index acc9b6f6..59a55bfb 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: manifests assets unit-tests helm-unit-tests bats-unit-tests preflight +.PHONY: manifests assets unit-tests helm-unit-tests include hack/common-envs.mk @@ -22,7 +22,6 @@ build: build-deps make -C packages/system/lineage-controller-webhook image make -C packages/system/cilium image make -C packages/system/linstor image - make -C packages/system/linstor-gui image make -C packages/system/kubeovn-webhook image make -C packages/system/kubeovn-plunger image make -C packages/system/dashboard image @@ -83,46 +82,11 @@ test: make -C packages/core/testing apply make -C packages/core/testing test -unit-tests: helm-unit-tests bats-unit-tests go-unit-tests +unit-tests: helm-unit-tests helm-unit-tests: hack/helm-unit-tests.sh -# Scoped go test over the cozystack-api surface that this repo owns. Kept -# narrow intentionally - running `go test ./...` pulls in generated code -# round-trip suites whose behavior depends on tool versions outside this -# repo's control (kubebuilder, openapi-gen, etc.) and is better exercised -# from their generator workflows. -go-unit-tests: - go test ./pkg/registry/... ./pkg/config/... ./pkg/cmd/server/... - -# Discover every hack/*.bats file that is NOT an e2e test and run it -# through cozytest.sh. Drop a new *.bats file in hack/ and it is picked -# up automatically on the next `make unit-tests` run. -# -# Caveat: $(wildcard ...) returns space-separated names, so a filename -# containing a literal space would split into multiple tokens here. All -# current bats files use hyphen-separated names; if the project ever -# introduces whitespace-bearing filenames this recipe must be rewritten -# (e.g. to use `find ... -print0 | xargs -0`). -BATS_UNIT_FILES := $(filter-out hack/e2e-%.bats,$(wildcard hack/*.bats)) - -bats-unit-tests: - @if [ -z "$(BATS_UNIT_FILES)" ]; then \ - echo "ERROR: no hack/*.bats unit test files found"; \ - exit 1; \ - fi - @for f in $(BATS_UNIT_FILES); do \ - echo "--- running $$f ---"; \ - hack/cozytest.sh "$$f" || exit 1; \ - done - -# Operator-facing host preflight check. Warns about a standalone -# containerd.service or docker.service running alongside the embedded -# k3s runtime. Safe to run at any time; always exits 0. -preflight: - @hack/check-host-runtime.sh - prepare-env: make -C packages/core/testing apply make -C packages/core/testing prepare-cluster diff --git a/README.md b/README.md index 5bc8d7be..dc1b0634 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ # Cozystack -**Cozystack** is a free platform and framework for building clouds. +**Cozystack** is a free PaaS platform and framework for building clouds. Cozystack is a [CNCF Sandbox Level Project](https://www.cncf.io/sandbox-projects/) that was originally built and sponsored by [Ænix](https://aenix.io/). diff --git a/api/apps/v1alpha1/kubernetes/types.go b/api/apps/v1alpha1/kubernetes/types.go index 88a3c97e..774fcdc5 100644 --- a/api/apps/v1alpha1/kubernetes/types.go +++ b/api/apps/v1alpha1/kubernetes/types.go @@ -36,9 +36,6 @@ type ConfigSpec struct { // Kubernetes control-plane configuration. // +kubebuilder:default:={} ControlPlane ControlPlane `json:"controlPlane"` - // Optional image overrides for air-gapped or rate-limited registries. - // +kubebuilder:default:={} - Images Images `json:"images"` } type APIServer struct { @@ -69,9 +66,6 @@ type Addons struct { // NVIDIA GPU Operator. // +kubebuilder:default:={} GpuOperator GPUOperatorAddon `json:"gpuOperator"` - // HAMi GPU virtualization middleware. - // +kubebuilder:default:={} - Hami HAMiAddon `json:"hami"` // Ingress-NGINX controller. // +kubebuilder:default:={} IngressNginx IngressNginxAddon `json:"ingressNginx"` @@ -163,21 +157,6 @@ type GatewayAPIAddon struct { Enabled bool `json:"enabled"` } -type HAMiAddon struct { - // Enable HAMi (requires GPU Operator). - // +kubebuilder:default:=false - Enabled bool `json:"enabled"` - // Custom Helm values overrides. - // +kubebuilder:default:={} - ValuesOverride k8sRuntime.RawExtension `json:"valuesOverride"` -} - -type Images struct { - // Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag. - // +kubebuilder:default:="" - WaitForKubeconfig string `json:"waitForKubeconfig,omitempty"` -} - type IngressNginxAddon struct { // Enable the controller (requires nodes labeled `ingress-nginx`). // +kubebuilder:default:=false diff --git a/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go b/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go index e021fbaa..2cbe3ba1 100644 --- a/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go +++ b/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go @@ -49,7 +49,6 @@ func (in *Addons) DeepCopyInto(out *Addons) { in.Fluxcd.DeepCopyInto(&out.Fluxcd) out.GatewayAPI = in.GatewayAPI in.GpuOperator.DeepCopyInto(&out.GpuOperator) - in.Hami.DeepCopyInto(&out.Hami) in.IngressNginx.DeepCopyInto(&out.IngressNginx) in.MonitoringAgents.DeepCopyInto(&out.MonitoringAgents) in.Velero.DeepCopyInto(&out.Velero) @@ -136,7 +135,6 @@ func (in *ConfigSpec) DeepCopyInto(out *ConfigSpec) { } in.Addons.DeepCopyInto(&out.Addons) in.ControlPlane.DeepCopyInto(&out.ControlPlane) - out.Images = in.Images } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigSpec. @@ -262,37 +260,6 @@ func (in *GatewayAPIAddon) DeepCopy() *GatewayAPIAddon { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *HAMiAddon) DeepCopyInto(out *HAMiAddon) { - *out = *in - in.ValuesOverride.DeepCopyInto(&out.ValuesOverride) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HAMiAddon. -func (in *HAMiAddon) DeepCopy() *HAMiAddon { - if in == nil { - return nil - } - out := new(HAMiAddon) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Images) DeepCopyInto(out *Images) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Images. -func (in *Images) DeepCopy() *Images { - if in == nil { - return nil - } - out := new(Images) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *IngressNginxAddon) DeepCopyInto(out *IngressNginxAddon) { *out = *in diff --git a/api/apps/v1alpha1/postgresql/types.go b/api/apps/v1alpha1/postgresql/types.go index a2ff77a8..fa85d51f 100644 --- a/api/apps/v1alpha1/postgresql/types.go +++ b/api/apps/v1alpha1/postgresql/types.go @@ -92,9 +92,6 @@ type Bootstrap struct { // Timestamp (RFC3339) for point-in-time recovery; empty means latest. // +kubebuilder:default:="" RecoveryTime string `json:"recoveryTime,omitempty"` - // Barman server name (S3 path prefix) used by the original cluster when writing backups. Set this only when the original cluster had an explicit barmanObjectStore.serverName that differed from its Kubernetes resource name. - // +kubebuilder:default:="" - ServerName string `json:"serverName,omitempty"` } type Database struct { diff --git a/api/apps/v1alpha1/vmdisk/types.go b/api/apps/v1alpha1/vmdisk/types.go index c493f49c..aefac1e4 100644 --- a/api/apps/v1alpha1/vmdisk/types.go +++ b/api/apps/v1alpha1/vmdisk/types.go @@ -32,28 +32,21 @@ type ConfigSpec struct { } type Source struct { - // Clone an existing vm-disk. - Disk *SourceDisk `json:"disk,omitempty"` // Download image from an HTTP source. Http *SourceHTTP `json:"http,omitempty"` - // Use image by name from default collection. + // Use image by name. Image *SourceImage `json:"image,omitempty"` // Upload local image. Upload *SourceUpload `json:"upload,omitempty"` } -type SourceDisk struct { - // Name of the vm-disk to clone. - Name string `json:"name"` -} - type SourceHTTP struct { // URL to download the image. Url string `json:"url"` } type SourceImage struct { - // Name of the image to use. + // Name of the image to use (uploaded as "golden image" or from the list: `ubuntu`, `fedora`, `cirros`, `alpine`, `talos`). Name string `json:"name"` } diff --git a/api/apps/v1alpha1/vmdisk/zz_generated.deepcopy.go b/api/apps/v1alpha1/vmdisk/zz_generated.deepcopy.go index 301bb1d9..e8ca986e 100644 --- a/api/apps/v1alpha1/vmdisk/zz_generated.deepcopy.go +++ b/api/apps/v1alpha1/vmdisk/zz_generated.deepcopy.go @@ -70,11 +70,6 @@ func (in *ConfigSpec) DeepCopy() *ConfigSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Source) DeepCopyInto(out *Source) { *out = *in - if in.Disk != nil { - in, out := &in.Disk, &out.Disk - *out = new(SourceDisk) - **out = **in - } if in.Http != nil { in, out := &in.Http, &out.Http *out = new(SourceHTTP) @@ -102,21 +97,6 @@ func (in *Source) DeepCopy() *Source { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *SourceDisk) DeepCopyInto(out *SourceDisk) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SourceDisk. -func (in *SourceDisk) DeepCopy() *SourceDisk { - if in == nil { - return nil - } - out := new(SourceDisk) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SourceHTTP) DeepCopyInto(out *SourceHTTP) { *out = *in diff --git a/api/apps/v1alpha1/vminstance/types.go b/api/apps/v1alpha1/vminstance/types.go index 9c48724a..2bb059c6 100644 --- a/api/apps/v1alpha1/vminstance/types.go +++ b/api/apps/v1alpha1/vminstance/types.go @@ -26,9 +26,6 @@ type ConfigSpec struct { // Ports to forward from outside the cluster. // +kubebuilder:default:={22} ExternalPorts []int `json:"externalPorts,omitempty"` - // Whether to accept ICMP traffic to the VM in PortList mode (preserves ping and PMTU discovery). No effect in WholeIP mode. Default true so ping behaves as users expect even when port filtering is in effect. - // +kubebuilder:default:=true - ExternalAllowICMP bool `json:"externalAllowICMP"` // Requested running state of the VirtualMachineInstance // +kubebuilder:default:="Always" RunStrategy RunStrategy `json:"runStrategy"` diff --git a/api/apps/v1alpha1/vminstance/zz_generated.deepcopy.go b/api/apps/v1alpha1/vminstance/zz_generated.deepcopy.go index 86d5ddfc..00b2a710 100644 --- a/api/apps/v1alpha1/vminstance/zz_generated.deepcopy.go +++ b/api/apps/v1alpha1/vminstance/zz_generated.deepcopy.go @@ -63,14 +63,9 @@ func (in *ConfigSpec) DeepCopyInto(out *ConfigSpec) { *out = make([]Disk, len(*in)) copy(*out, *in) } - if in.Networks != nil { - in, out := &in.Networks, &out.Networks - *out = make([]Network, len(*in)) - copy(*out, *in) - } if in.Subnets != nil { in, out := &in.Subnets, &out.Subnets - *out = make([]Network, len(*in)) + *out = make([]Subnet, len(*in)) copy(*out, *in) } if in.Gpus != nil { @@ -126,21 +121,6 @@ func (in *GPU) DeepCopy() *GPU { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Network) DeepCopyInto(out *Network) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Network. -func (in *Network) DeepCopy() *Network { - if in == nil { - return nil - } - out := new(Network) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Resources) DeepCopyInto(out *Resources) { *out = *in @@ -158,3 +138,18 @@ func (in *Resources) DeepCopy() *Resources { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Subnet) DeepCopyInto(out *Subnet) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Subnet. +func (in *Subnet) DeepCopy() *Subnet { + if in == nil { + return nil + } + out := new(Subnet) + in.DeepCopyInto(out) + return out +} diff --git a/api/apps/v1alpha1/vpc/zz_generated.deepcopy.go b/api/apps/v1alpha1/vpc/zz_generated.deepcopy.go index 3cda29c4..cd718167 100644 --- a/api/apps/v1alpha1/vpc/zz_generated.deepcopy.go +++ b/api/apps/v1alpha1/vpc/zz_generated.deepcopy.go @@ -58,16 +58,6 @@ func (in *ConfigSpec) DeepCopyInto(out *ConfigSpec) { *out = make([]Subnet, len(*in)) copy(*out, *in) } - if in.Peers != nil { - in, out := &in.Peers, &out.Peers - *out = make([]Peer, len(*in)) - copy(*out, *in) - } - if in.Routes != nil { - in, out := &in.Routes, &out.Routes - *out = make([]Route, len(*in)) - copy(*out, *in) - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigSpec. @@ -80,36 +70,6 @@ func (in *ConfigSpec) DeepCopy() *ConfigSpec { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Peer) DeepCopyInto(out *Peer) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Peer. -func (in *Peer) DeepCopy() *Peer { - if in == nil { - return nil - } - out := new(Peer) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Route) DeepCopyInto(out *Route) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Route. -func (in *Route) DeepCopy() *Route { - if in == nil { - return nil - } - out := new(Route) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Subnet) DeepCopyInto(out *Subnet) { *out = *in diff --git a/api/backups/v1alpha1/restorejob_types.go b/api/backups/v1alpha1/restorejob_types.go index 9817e8c1..1a2dfa00 100644 --- a/api/backups/v1alpha1/restorejob_types.go +++ b/api/backups/v1alpha1/restorejob_types.go @@ -42,12 +42,6 @@ type RestoreJobSpec struct { // application as referenced by backup.spec.applicationRef. // +optional TargetApplicationRef *corev1.TypedLocalObjectReference `json:"targetApplicationRef,omitempty"` - - // Options is a driver-specific blob of restore options, typed based on - // targetApplicationRef and the current controller implementation. - // +optional - // +kubebuilder:pruning:PreserveUnknownFields - Options *runtime.RawExtension `json:"options,omitempty"` } // RestoreJobStatus represents the observed state of a RestoreJob. diff --git a/api/backups/v1alpha1/zz_generated.deepcopy.go b/api/backups/v1alpha1/zz_generated.deepcopy.go index 61b8f839..b7387772 100644 --- a/api/backups/v1alpha1/zz_generated.deepcopy.go +++ b/api/backups/v1alpha1/zz_generated.deepcopy.go @@ -1,21 +1,5 @@ //go:build !ignore_autogenerated -/* -Copyright 2025 The Cozystack Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - // Code generated by controller-gen. DO NOT EDIT. package v1alpha1 @@ -620,11 +604,6 @@ func (in *RestoreJobSpec) DeepCopyInto(out *RestoreJobSpec) { *out = new(v1.TypedLocalObjectReference) (*in).DeepCopyInto(*out) } - if in.Options != nil { - in, out := &in.Options, &out.Options - *out = new(runtime.RawExtension) - (*in).DeepCopyInto(*out) - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RestoreJobSpec. @@ -666,3 +645,4 @@ func (in *RestoreJobStatus) DeepCopy() *RestoreJobStatus { in.DeepCopyInto(out) return out } + diff --git a/api/v1alpha1/packagesource_types.go b/api/v1alpha1/packagesource_types.go index 75bf217b..c0b943e8 100644 --- a/api/v1alpha1/packagesource_types.go +++ b/api/v1alpha1/packagesource_types.go @@ -132,16 +132,6 @@ type ComponentInstall struct { // DependsOn is a list of component names that must be installed before this component // +optional DependsOn []string `json:"dependsOn,omitempty"` - - // UpgradeCRDs controls how CRDs from the chart's crds/ directory are - // handled on HelmRelease upgrades. Maps to HelmRelease.Spec.Upgrade.CRDs. - // Empty string (default) preserves the helm-controller default (Skip). - // Use "CreateReplace" for operators that evolve their CRD set between - // versions. Warning: CreateReplace overwrites CRDs and may cause data - // loss if upstream drops fields from a CRD with live objects. - // +optional - // +kubebuilder:validation:Enum=Skip;Create;CreateReplace - UpgradeCRDs string `json:"upgradeCRDs,omitempty"` } // Component defines a single Helm release component within a package source diff --git a/cmd/cozystack-controller/main.go b/cmd/cozystack-controller/main.go index bcaca9c9..77c4f9dc 100644 --- a/cmd/cozystack-controller/main.go +++ b/cmd/cozystack-controller/main.go @@ -42,7 +42,6 @@ import ( "github.com/cozystack/cozystack/internal/telemetry" helmv2 "github.com/fluxcd/helm-controller/api/v2" - cosiv1alpha1 "sigs.k8s.io/container-object-storage-interface-api/apis/objectstorage/v1alpha1" // +kubebuilder:scaffold:imports ) @@ -57,7 +56,6 @@ func init() { utilruntime.Must(cozystackiov1alpha1.AddToScheme(scheme)) utilruntime.Must(dashboard.AddToScheme(scheme)) utilruntime.Must(helmv2.AddToScheme(scheme)) - utilruntime.Must(cosiv1alpha1.AddToScheme(scheme)) // +kubebuilder:scaffold:scheme } diff --git a/cmd/kubeovn-plunger/main.go b/cmd/kubeovn-plunger/main.go index 611d030b..210405ce 100644 --- a/cmd/kubeovn-plunger/main.go +++ b/cmd/kubeovn-plunger/main.go @@ -29,7 +29,6 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics" @@ -132,11 +131,6 @@ func main() { HealthProbeBindAddress: probeAddr, LeaderElection: enableLeaderElection, LeaderElectionID: "29a0338b.cozystack.io", - Cache: cache.Options{ - DefaultNamespaces: map[string]cache.Config{ - kubeOVNNamespace: {}, - }, - }, // LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily // when the Manager ends. This requires the binary to immediately end when the // Manager is stopped, otherwise, this setting is unsafe. Setting this significantly diff --git a/dashboards/gpu/gpu-efficiency.json b/dashboards/gpu/gpu-efficiency.json deleted file mode 100644 index 28a24568..00000000 --- a/dashboards/gpu/gpu-efficiency.json +++ /dev/null @@ -1,819 +0,0 @@ -{ - "uid": "gpu-efficiency", - "title": "GPU Efficiency Score", - "description": "Tensor saturation, util/watt and throttling — reveals inefficient GPU workloads", - "tags": [ - "gpu", - "efficiency", - "finops" - ], - "timezone": "browser", - "editable": true, - "graphTooltip": 1, - "time": { - "from": "now-1h", - "to": "now" - }, - "fiscalYearStartMonth": 0, - "schemaVersion": 42, - "panels": [ - { - "type": "row", - "collapsed": false, - "title": "Overall efficiency metrics", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "panels": [] - }, - { - "type": "stat", - "id": 2, - "targets": [ - { - "expr": "avg(gpu:tensor_saturation:avg5m) * 100", - "refId": "A" - } - ], - "title": "Avg Tensor Saturation", - "description": "Mean tensor core saturation across all GPUs. \u003c10% means GPUs are used inefficiently (workloads could move to CPU or optimize their code). Cluster-wide — DCGM metrics cannot be attributed to workload namespaces.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 0, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "graphMode": "area", - "colorMode": "background", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "percent", - "min": 0, - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "red" - }, - { - "value": 10, - "color": "orange" - }, - { - "value": 30, - "color": "yellow" - }, - { - "value": 60, - "color": "green" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "stat", - "id": 3, - "targets": [ - { - "expr": "avg(gpu:util_per_watt:avg5m)", - "refId": "A" - } - ], - "title": "Avg Utilization per Watt", - "description": "NVML utilization % per watt across all GPUs. Higher value = more efficient workload. Cluster-wide — DCGM metrics cannot be attributed to workload namespaces.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 8, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "graphMode": "area", - "colorMode": "background", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "none", - "decimals": 3, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "red" - }, - { - "value": 0.5, - "color": "yellow" - }, - { - "value": 1, - "color": "green" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "stat", - "id": 4, - "targets": [ - { - "expr": "avg(gpu:power_throttle_fraction:rate5m)", - "refId": "A" - } - ], - "title": "Avg Power Throttling", - "description": "Fraction of time GPUs hit the TDP cap and lose performance. \u003e5% means tenants underutilize billed FLOPS. Cluster-wide — rule aggregates by (Hostname, gpu, UUID) and drops namespace label.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 16, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "graphMode": "area", - "colorMode": "background", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "percentunit", - "decimals": 2, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "green" - }, - { - "value": 0.05, - "color": "yellow" - }, - { - "value": 0.2, - "color": "red" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "row", - "collapsed": false, - "title": "NVML vs Tensor (mismatch detector)", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 6 - }, - "id": 10, - "panels": [] - }, - { - "type": "timeseries", - "id": 11, - "targets": [ - { - "expr": "DCGM_FI_DEV_GPU_UTIL", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A" - } - ], - "title": "NVML GPU Utilization", - "description": "Classic utilization metric. Shows activity of any engine (SM, copy, encoder). Cluster-wide — DCGM namespace is the exporter's own namespace, not the workload namespace.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 7 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "percent", - "min": 0, - "max": 100, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 10, - "showPoints": "never" - } - }, - "overrides": [] - } - }, - { - "type": "timeseries", - "id": 12, - "targets": [ - { - "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE * 100", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A" - } - ], - "title": "Tensor Pipe Active", - "description": "Real tensor core load (HMMA). For AI/LLM inference it should be ≥20%, otherwise the workload is unoptimized. Cluster-wide — DCGM namespace is the exporter's own namespace, not the workload namespace.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 7 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "percent", - "min": 0, - "max": 100, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 10, - "showPoints": "never" - } - }, - "overrides": [] - } - }, - { - "type": "row", - "collapsed": false, - "title": "Per-GPU ranking", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 15 - }, - "id": 20, - "panels": [] - }, - { - "type": "table", - "id": 21, - "targets": [ - { - "expr": "topk(20, gpu:tensor_saturation:avg5m * 100)", - "instant": true, - "range": false, - "format": "table", - "refId": "A" - } - ], - "title": "Tensor Saturation per GPU (5m avg)", - "description": "Which GPUs are exercising tensor cores and which are not. Sorted descending. Grouped by Hostname/gpu/UUID — DCGM metrics cannot be attributed to workload namespaces.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "repeatDirection": "h", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": { - "DCGM_FI_DRIVER_VERSION": true, - "Time": true, - "__name__": true, - "cluster": true, - "container": true, - "device": true, - "endpoint": true, - "gpu_driver_version": true, - "instance": true, - "job": true, - "modelName": true, - "namespace": true, - "pci_bus_id": true, - "pod": true, - "prometheus": true, - "service": true, - "tenant": true, - "tier": true, - "uid": true, - "unit": true - }, - "indexByName": { - "Hostname": 0, - "UUID": 2, - "Value": 3, - "gpu": 1 - }, - "renameByName": { - "Hostname": "Node", - "UUID": "UUID", - "Value": "Saturation", - "gpu": "GPU" - } - } - } - ], - "options": { - "frameIndex": 0, - "showHeader": true, - "showTypeIcons": false, - "sortBy": [ - { - "displayName": "Saturation", - "desc": true - } - ], - "footer": { - "show": false, - "reducer": [] - }, - "cellHeight": "sm" - }, - "fieldConfig": { - "defaults": {}, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Saturation" - }, - "properties": [ - { - "id": "unit", - "value": "percent" - }, - { - "id": "min", - "value": 0 - }, - { - "id": "max", - "value": 100 - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "gradient", - "type": "gauge" - } - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "red" - }, - { - "color": "orange", - "value": 10 - }, - { - "color": "yellow", - "value": 30 - }, - { - "color": "green", - "value": 60 - } - ] - } - } - ] - } - ] - } - }, - { - "type": "table", - "id": 22, - "targets": [ - { - "expr": "topk(20, gpu:util_per_watt:avg5m)", - "instant": true, - "range": false, - "format": "table", - "refId": "A" - } - ], - "title": "Utilization / Watt per GPU (5m avg)", - "description": "How efficiently each GPU spends watts. Low value = poor optimization. Grouped by Hostname/gpu/UUID — DCGM metrics cannot be attributed to workload namespaces.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "repeatDirection": "h", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": { - "DCGM_FI_DRIVER_VERSION": true, - "Time": true, - "__name__": true, - "cluster": true, - "container": true, - "device": true, - "endpoint": true, - "gpu_driver_version": true, - "instance": true, - "job": true, - "modelName": true, - "namespace": true, - "pci_bus_id": true, - "pod": true, - "prometheus": true, - "service": true, - "tenant": true, - "tier": true, - "uid": true, - "unit": true - }, - "indexByName": { - "Hostname": 0, - "UUID": 2, - "Value": 3, - "gpu": 1 - }, - "renameByName": { - "Hostname": "Node", - "UUID": "UUID", - "Value": "Util/Watt", - "gpu": "GPU" - } - } - } - ], - "options": { - "frameIndex": 0, - "showHeader": true, - "showTypeIcons": false, - "sortBy": [ - { - "displayName": "Util/Watt", - "desc": true - } - ], - "footer": { - "show": false, - "reducer": [] - }, - "cellHeight": "sm" - }, - "fieldConfig": { - "defaults": {}, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Util/Watt" - }, - "properties": [ - { - "id": "unit", - "value": "none" - }, - { - "id": "decimals", - "value": 3 - }, - { - "id": "min", - "value": 0 - }, - { - "id": "max", - "value": 1.5 - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "gradient", - "type": "gauge" - } - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "red" - }, - { - "color": "yellow", - "value": 0.5 - }, - { - "color": "green", - "value": 1 - } - ] - } - } - ] - } - ] - } - }, - { - "type": "row", - "collapsed": false, - "title": "Throttling", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 24 - }, - "id": 30, - "panels": [] - }, - { - "type": "timeseries", - "id": 31, - "targets": [ - { - "expr": "gpu:power_throttle_fraction:rate5m", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A" - } - ], - "title": "Power throttle fraction per GPU", - "description": "Fraction of time the GPU was power-throttled. 1.0 = always throttled. Cluster-wide — rule aggregates by (Hostname, gpu, UUID) and drops namespace label.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 25 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "percentunit", - "min": 0, - "max": 1, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "green" - }, - { - "value": 0.05, - "color": "yellow" - }, - { - "value": 0.2, - "color": "red" - } - ] - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 25, - "showPoints": "never" - } - }, - "overrides": [] - } - }, - { - "type": "timeseries", - "id": 32, - "targets": [ - { - "expr": "gpu:thermal_throttle_fraction:rate5m", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A" - } - ], - "title": "Thermal throttle fraction per GPU", - "description": "Fraction of time the GPU was thermal-throttled. Cluster-wide — rule aggregates by (Hostname, gpu, UUID) and drops namespace label.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 25 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "percentunit", - "min": 0, - "max": 1, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "green" - }, - { - "value": 0.05, - "color": "yellow" - }, - { - "value": 0.2, - "color": "red" - } - ] - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 25, - "showPoints": "never" - } - }, - "overrides": [] - } - } - ], - "templating": { - "list": [ - { - "type": "datasource", - "name": "ds_prometheus", - "label": "Prometheus", - "skipUrlSync": false, - "query": "prometheus", - "current": { - "selected": false, - "text": "default", - "value": "default" - }, - "multi": false, - "allowCustomValue": true, - "includeAll": false, - "regex": "", - "auto": false, - "auto_min": "10s", - "auto_count": 30 - } - ] - }, - "annotations": {} -} diff --git a/dashboards/gpu/gpu-fleet.json b/dashboards/gpu/gpu-fleet.json deleted file mode 100644 index 60c12bfb..00000000 --- a/dashboards/gpu/gpu-fleet.json +++ /dev/null @@ -1,1157 +0,0 @@ -{ - "uid": "gpu-fleet", - "title": "GPU Fleet Overview", - "description": "Cluster-wide GPU inventory, capacity, utilization and health — admin view across the whole fleet.", - "tags": [ - "gpu", - "fleet", - "admin" - ], - "timezone": "browser", - "editable": true, - "graphTooltip": 1, - "time": { - "from": "now-6h", - "to": "now" - }, - "fiscalYearStartMonth": 0, - "schemaVersion": 42, - "panels": [ - { - "type": "row", - "collapsed": false, - "title": "Cluster snapshot", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "panels": [] - }, - { - "type": "stat", - "id": 2, - "targets": [ - { - "expr": "cluster:gpu_count:total", - "refId": "A" - } - ], - "title": "Total GPUs", - "description": "Physical GPUs discovered by DCGM across all nodes.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "graphMode": "none", - "colorMode": "value", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "blue" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "stat", - "id": 3, - "targets": [ - { - "expr": "cluster:gpu_count:allocated", - "refId": "A" - } - ], - "title": "Allocated", - "description": "Sum of kube GPU requests across all pods. Billable view — includes Pending pods.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "graphMode": "none", - "colorMode": "background", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "blue" - }, - { - "value": 1, - "color": "green" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "stat", - "id": 4, - "targets": [ - { - "expr": "cluster:gpu_count:free", - "refId": "A" - } - ], - "title": "Free", - "description": "Unallocated capacity available for new tenants.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 8, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "graphMode": "none", - "colorMode": "background", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "red" - }, - { - "value": 1, - "color": "yellow" - }, - { - "value": 2, - "color": "green" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "stat", - "id": 5, - "targets": [ - { - "expr": "count(count by (Hostname) (DCGM_FI_DEV_GPU_UTIL))", - "refId": "A" - } - ], - "title": "Nodes with GPU", - "description": "Nodes reporting at least one GPU to DCGM. Counted via DCGM_FI_DEV_GPU_UTIL because kube-state-metrics does not expose node labels by default.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 12, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "graphMode": "none", - "colorMode": "value", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "blue" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "stat", - "id": 6, - "targets": [ - { - "expr": "cluster:gpu_util:avg", - "refId": "A" - } - ], - "title": "Average utilization", - "description": "Legacy NVML utilization across all tenant GPUs.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 16, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "graphMode": "area", - "colorMode": "value", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "percent", - "min": 0, - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "blue" - }, - { - "value": 30, - "color": "green" - }, - { - "value": 80, - "color": "orange" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "stat", - "id": 7, - "targets": [ - { - "expr": "cluster:gpu_power_watts:sum", - "refId": "A" - } - ], - "title": "Total power", - "description": "Live power draw summed across the fleet.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 20, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "graphMode": "area", - "colorMode": "value", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "watt", - "decimals": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "green" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "row", - "collapsed": false, - "title": "Per-node GPU activity", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 10, - "panels": [] - }, - { - "type": "bargauge", - "id": 11, - "targets": [ - { - "expr": "avg by (Hostname) (DCGM_FI_DEV_GPU_UTIL)", - "legendFormat": "{{Hostname}}", - "refId": "A" - } - ], - "title": "GPU utilization per node", - "description": "Average NVML utilization per node — spot idle or saturated hosts at a glance.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 6 - }, - "repeatDirection": "h", - "options": { - "displayMode": "gradient", - "valueMode": "color", - "namePlacement": "auto", - "showUnfilled": true, - "sizing": "auto", - "minVizWidth": 8, - "minVizHeight": 16, - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false, - "calcs": [] - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "maxVizHeight": 300, - "orientation": "horizontal" - }, - "fieldConfig": { - "defaults": { - "unit": "percent", - "min": 0, - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "blue" - }, - { - "value": 30, - "color": "green" - }, - { - "value": 80, - "color": "orange" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "bargauge", - "id": 12, - "targets": [ - { - "expr": "sum by (Hostname) (DCGM_FI_DEV_POWER_USAGE) / clamp_min(sum by (Hostname) (DCGM_FI_DEV_POWER_MGMT_LIMIT), 1) * 100", - "legendFormat": "{{Hostname}}", - "refId": "A" - } - ], - "title": "Power draw per node (% of TDP)", - "description": "GPU power usage as a fraction of the node's combined TDP cap. Works across GPU SKUs without per-model thresholds. Requires DCGM_FI_DEV_POWER_MGMT_LIMIT from custom DCGM CSV (see packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml).", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 6 - }, - "repeatDirection": "h", - "options": { - "displayMode": "gradient", - "valueMode": "color", - "namePlacement": "auto", - "showUnfilled": true, - "sizing": "auto", - "minVizWidth": 8, - "minVizHeight": 16, - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false, - "calcs": [] - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "maxVizHeight": 300, - "orientation": "horizontal" - }, - "fieldConfig": { - "defaults": { - "unit": "percent", - "min": 0, - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "green" - }, - { - "value": 60, - "color": "yellow" - }, - { - "value": 80, - "color": "orange" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "row", - "collapsed": false, - "title": "Usage trends", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 14 - }, - "id": 20, - "panels": [] - }, - { - "type": "timeseries", - "id": 21, - "targets": [ - { - "expr": "avg by (Hostname) (DCGM_FI_DEV_GPU_UTIL)", - "legendFormat": "{{Hostname}}", - "refId": "A" - } - ], - "title": "GPU utilization per node (trend)", - "description": "Per-node NVML utilization over time — shows shifting workload across hosts.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 15 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "percent", - "min": 0, - "max": 100, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 20, - "showPoints": "never" - } - }, - "overrides": [] - } - }, - { - "type": "timeseries", - "id": 22, - "targets": [ - { - "expr": "sum by (Hostname) (DCGM_FI_DEV_POWER_USAGE)", - "legendFormat": "{{Hostname}}", - "refId": "A" - } - ], - "title": "Power draw per node (trend)", - "description": "Per-node GPU power consumption over time.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 15 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "watt", - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 20, - "showPoints": "never" - } - }, - "overrides": [] - } - }, - { - "type": "row", - "collapsed": false, - "title": "Health", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 23 - }, - "id": 30, - "panels": [] - }, - { - "type": "stat", - "id": 31, - "targets": [ - { - "expr": "count((max by (Hostname, gpu) (DCGM_FI_DEV_XID_ERRORS)) \u003e 0) or vector(0)", - "refId": "A" - } - ], - "title": "GPUs with XID errors", - "description": "Count of GPUs reporting a non-zero XID code. XID = NVIDIA hardware/driver error — any non-zero value needs investigation.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 0, - "y": 24 - }, - "repeatDirection": "h", - "options": { - "graphMode": "none", - "colorMode": "background", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "green" - }, - { - "value": 1, - "color": "red" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "stat", - "id": 32, - "targets": [ - { - "expr": "avg(gpu:power_throttle_fraction:rate5m)", - "refId": "A" - } - ], - "title": "Power throttling (cluster avg)", - "description": "Fraction of time the fleet spent power-throttled. \u003e5% = billed FLOPS are being underdelivered.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 6, - "y": 24 - }, - "repeatDirection": "h", - "options": { - "graphMode": "none", - "colorMode": "background", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "percentunit", - "decimals": 2, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "green" - }, - { - "value": 0.05, - "color": "yellow" - }, - { - "value": 0.2, - "color": "red" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "stat", - "id": 33, - "targets": [ - { - "expr": "avg(gpu:thermal_throttle_fraction:rate5m)", - "refId": "A" - } - ], - "title": "Thermal throttling (cluster avg)", - "description": "Fraction of time the fleet spent thermal-throttled. Indicates cooling/datacenter issues.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 12, - "y": 24 - }, - "repeatDirection": "h", - "options": { - "graphMode": "none", - "colorMode": "background", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "percentunit", - "decimals": 2, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "green" - }, - { - "value": 0.05, - "color": "yellow" - }, - { - "value": 0.2, - "color": "red" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "stat", - "id": 34, - "targets": [ - { - "expr": "max(DCGM_FI_DEV_GPU_TEMP)", - "refId": "A" - } - ], - "title": "Max GPU temperature", - "description": "Hottest GPU in the fleet right now. Adjust thresholds to match your GPU model specifications.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 18, - "y": 24 - }, - "repeatDirection": "h", - "options": { - "graphMode": "none", - "colorMode": "background", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "celsius", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "green" - }, - { - "value": 75, - "color": "yellow" - }, - { - "value": 85, - "color": "red" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "timeseries", - "id": 35, - "targets": [ - { - "expr": "DCGM_FI_DEV_GPU_TEMP", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A" - } - ], - "title": "Temperature per GPU", - "description": "Per-GPU temperature trace. Sustained red zone = cooling issue or aggressive workload.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 29 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "celsius", - "min": 20, - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "green" - }, - { - "value": 75, - "color": "yellow" - }, - { - "value": 85, - "color": "red" - } - ] - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 10, - "showPoints": "never" - } - }, - "overrides": [] - } - }, - { - "type": "timeseries", - "id": 36, - "targets": [ - { - "expr": "gpu:power_throttle_fraction:rate5m", - "legendFormat": "power {{Hostname}}/{{gpu}}", - "refId": "A" - }, - { - "expr": "gpu:thermal_throttle_fraction:rate5m", - "legendFormat": "thermal {{Hostname}}/{{gpu}}", - "refId": "B" - } - ], - "title": "Throttle fraction per GPU", - "description": "Red = power throttle, blue = thermal throttle. Both on same chart to compare causes.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 29 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "percentunit", - "min": 0, - "max": 1, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 25, - "showPoints": "never" - } - }, - "overrides": [] - } - }, - { - "type": "row", - "collapsed": false, - "title": "Hardware inventory", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 37 - }, - "id": 40, - "panels": [] - }, - { - "type": "table", - "id": 41, - "targets": [ - { - "expr": "group by (Hostname, modelName, gpu, UUID) (DCGM_FI_DEV_GPU_UTIL)", - "instant": true, - "range": false, - "format": "table", - "refId": "A" - } - ], - "title": "GPU inventory", - "description": "One row per physical GPU. Model / index / UUID sourced from DCGM labels.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 38 - }, - "repeatDirection": "h", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Value": true, - "__name__": true, - "cluster": true, - "container": true, - "device": true, - "endpoint": true, - "instance": true, - "job": true, - "pci_bus_id": true, - "prometheus": true, - "service": true, - "tenant": true, - "tier": true, - "uid": true, - "unit": true - }, - "indexByName": { - "Hostname": 0, - "UUID": 3, - "gpu": 2, - "modelName": 1 - }, - "renameByName": { - "Hostname": "Node", - "UUID": "UUID", - "gpu": "Index", - "modelName": "GPU Model" - } - } - } - ], - "options": { - "frameIndex": 0, - "showHeader": true, - "showTypeIcons": false, - "footer": { - "show": false, - "reducer": [] - }, - "cellHeight": "sm" - } - } - ], - "templating": { - "list": [ - { - "type": "datasource", - "name": "ds_prometheus", - "label": "Prometheus", - "skipUrlSync": false, - "query": "prometheus", - "current": { - "selected": false, - "text": "default", - "value": "default" - }, - "multi": false, - "allowCustomValue": true, - "includeAll": false, - "regex": "", - "auto": false, - "auto_min": "10s", - "auto_count": 30 - } - ] - }, - "annotations": {} -} diff --git a/dashboards/gpu/gpu-performance.json b/dashboards/gpu/gpu-performance.json deleted file mode 100644 index aed9c69a..00000000 --- a/dashboards/gpu/gpu-performance.json +++ /dev/null @@ -1,957 +0,0 @@ -{ - "uid": "gpu-performance", - "title": "GPU Performance", - "tags": [ - "gpu", - "dcgm" - ], - "timezone": "browser", - "editable": true, - "graphTooltip": 1, - "time": { - "from": "now-1h", - "to": "now" - }, - "fiscalYearStartMonth": 0, - "schemaVersion": 42, - "panels": [ - { - "type": "row", - "collapsed": false, - "title": "Overview", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "panels": [] - }, - { - "type": "stat", - "id": 2, - "targets": [ - { - "expr": "cluster:gpu_count:total", - "refId": "A" - } - ], - "title": "Total GPUs", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "graphMode": "none", - "colorMode": "value", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "blue" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "stat", - "id": 3, - "targets": [ - { - "expr": "cluster:gpu_count:allocated", - "refId": "A" - } - ], - "title": "Allocated", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "graphMode": "none", - "colorMode": "value", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "green" - }, - { - "value": 1, - "color": "yellow" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "stat", - "id": 4, - "targets": [ - { - "expr": "cluster:gpu_util:avg", - "refId": "A" - } - ], - "title": "Average utilization", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "graphMode": "area", - "colorMode": "value", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "percent", - "min": 0, - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "blue" - }, - { - "value": 30, - "color": "green" - }, - { - "value": 80, - "color": "orange" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "stat", - "id": 5, - "targets": [ - { - "expr": "cluster:gpu_power_watts:sum", - "refId": "A" - } - ], - "title": "Power draw", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "graphMode": "area", - "colorMode": "value", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "watt", - "decimals": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "green" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "row", - "collapsed": false, - "title": "Utilization", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 10, - "panels": [] - }, - { - "type": "timeseries", - "id": 11, - "targets": [ - { - "expr": "DCGM_FI_DEV_GPU_UTIL{Hostname=~\"$Hostname\"}", - "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", - "refId": "A" - } - ], - "title": "GPU Utilization (NVML)", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 6 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "percent", - "min": 0, - "max": 100, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 10, - "showPoints": "never", - "spanNulls": false - } - }, - "overrides": [] - } - }, - { - "type": "timeseries", - "id": 12, - "targets": [ - { - "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{Hostname=~\"$Hostname\"} * 100", - "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", - "refId": "A" - } - ], - "title": "Tensor Pipe Active (realistic load for LLM/AI)", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 6 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "percent", - "min": 0, - "max": 100, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 10, - "showPoints": "never", - "spanNulls": false - } - }, - "overrides": [] - } - }, - { - "type": "timeseries", - "id": 13, - "targets": [ - { - "expr": "DCGM_FI_PROF_GR_ENGINE_ACTIVE{Hostname=~\"$Hostname\"} * 100", - "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", - "refId": "A" - } - ], - "title": "Graphics Engine Active", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 14 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "percent", - "min": 0, - "max": 100, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 10, - "showPoints": "never", - "spanNulls": false - } - }, - "overrides": [] - } - }, - { - "type": "timeseries", - "id": 14, - "targets": [ - { - "expr": "DCGM_FI_DEV_MEM_COPY_UTIL{Hostname=~\"$Hostname\"}", - "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", - "refId": "A" - } - ], - "title": "Memory Copy Utilization", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 14 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "percent", - "min": 0, - "max": 100, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 10, - "showPoints": "never", - "spanNulls": false - } - }, - "overrides": [] - } - }, - { - "type": "row", - "collapsed": false, - "title": "Memory", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 22 - }, - "id": 20, - "panels": [] - }, - { - "type": "timeseries", - "id": 21, - "targets": [ - { - "expr": "DCGM_FI_DEV_FB_USED{Hostname=~\"$Hostname\"} * 1048576", - "legendFormat": "{{Hostname}}/{{gpu}} {{pod}}", - "refId": "A" - } - ], - "title": "VRAM Used", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 23 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "bytes", - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 25, - "showPoints": "never" - } - }, - "overrides": [] - } - }, - { - "type": "timeseries", - "id": 22, - "targets": [ - { - "expr": "DCGM_FI_DEV_FB_FREE{Hostname=~\"$Hostname\"} * 1048576", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A" - } - ], - "title": "VRAM Free", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 23 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "calcs": [ - "mean", - "min" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "bytes", - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 10, - "showPoints": "never" - } - }, - "overrides": [] - } - }, - { - "type": "row", - "collapsed": false, - "title": "Power \u0026 Temperature", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 31 - }, - "id": 30, - "panels": [] - }, - { - "type": "timeseries", - "id": 31, - "targets": [ - { - "expr": "DCGM_FI_DEV_POWER_USAGE{Hostname=~\"$Hostname\"}", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A" - } - ], - "title": "Power Usage per GPU", - "description": "Hardware-level metric — shown cluster-wide regardless of namespace filter.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 32 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "watt", - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 10, - "showPoints": "never" - } - }, - "overrides": [] - } - }, - { - "type": "timeseries", - "id": 32, - "targets": [ - { - "expr": "DCGM_FI_DEV_GPU_TEMP{Hostname=~\"$Hostname\"}", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A" - } - ], - "title": "GPU Temperature", - "description": "Hardware-level metric — shown cluster-wide regardless of namespace filter.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 32 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "celsius", - "min": 20, - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "green" - }, - { - "value": 75, - "color": "yellow" - }, - { - "value": 85, - "color": "red" - } - ] - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 10, - "showPoints": "never" - } - }, - "overrides": [] - } - }, - { - "type": "row", - "collapsed": false, - "title": "Health", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 40 - }, - "id": 40, - "panels": [] - }, - { - "type": "stat", - "id": 41, - "targets": [ - { - "expr": "max by (Hostname, gpu) (DCGM_FI_DEV_XID_ERRORS{Hostname=~\"$Hostname\"})", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A" - } - ], - "title": "XID errors (latest)", - "description": "Hardware-level metric — shown cluster-wide regardless of namespace filter.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 0, - "y": 41 - }, - "repeatDirection": "h", - "options": { - "graphMode": "none", - "colorMode": "background", - "justifyMode": "auto", - "textMode": "value_and_name", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "green" - }, - { - "value": 1, - "color": "red" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "timeseries", - "id": 42, - "targets": [ - { - "expr": "rate(DCGM_FI_DEV_POWER_VIOLATION{Hostname=~\"$Hostname\"}[5m])", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A" - } - ], - "title": "Power Violation (µs/s throttled due to power)", - "description": "Hardware-level metric — shown cluster-wide regardless of namespace filter.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 8, - "y": 41 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false, - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "µs", - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 10, - "showPoints": "never" - } - }, - "overrides": [] - } - }, - { - "type": "timeseries", - "id": 43, - "targets": [ - { - "expr": "rate(DCGM_FI_DEV_THERMAL_VIOLATION{Hostname=~\"$Hostname\"}[5m])", - "legendFormat": "{{Hostname}}/{{gpu}}", - "refId": "A" - } - ], - "title": "Thermal Violation (µs/s throttled due to thermals)", - "description": "Hardware-level metric — shown cluster-wide regardless of namespace filter.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 5, - "w": 8, - "x": 16, - "y": 41 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false, - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "µs", - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 10, - "showPoints": "never" - } - }, - "overrides": [] - } - } - ], - "templating": { - "list": [ - { - "type": "datasource", - "name": "ds_prometheus", - "label": "Prometheus", - "skipUrlSync": false, - "query": "prometheus", - "current": { - "selected": false, - "text": "default", - "value": "default" - }, - "multi": false, - "allowCustomValue": true, - "includeAll": false, - "regex": "", - "auto": false, - "auto_min": "10s", - "auto_count": 30 - }, - { - "type": "query", - "name": "Hostname", - "label": "Host", - "skipUrlSync": false, - "query": "label_values(DCGM_FI_DEV_GPU_UTIL, Hostname)", - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "multi": true, - "allowCustomValue": true, - "refresh": 2, - "sort": 1, - "includeAll": true, - "auto": false, - "auto_min": "10s", - "auto_count": 30 - } - ] - }, - "annotations": {} -} diff --git a/dashboards/gpu/gpu-quotas.json b/dashboards/gpu/gpu-quotas.json deleted file mode 100644 index 865fb95c..00000000 --- a/dashboards/gpu/gpu-quotas.json +++ /dev/null @@ -1,637 +0,0 @@ -{ - "uid": "gpu-quotas", - "title": "GPU Quotas \u0026 Allocation", - "tags": [ - "gpu", - "quotas" - ], - "timezone": "browser", - "editable": true, - "graphTooltip": 1, - "time": { - "from": "now-6h", - "to": "now" - }, - "fiscalYearStartMonth": 0, - "schemaVersion": 42, - "panels": [ - { - "type": "row", - "collapsed": false, - "title": "Allocation overview", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "panels": [] - }, - { - "type": "stat", - "id": 2, - "targets": [ - { - "expr": "sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"})", - "refId": "A" - } - ], - "title": "GPU allocatable", - "description": "Total GPU capacity the cluster can schedule to pods.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "graphMode": "none", - "colorMode": "value", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "blue" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "stat", - "id": 3, - "targets": [ - { - "expr": "cluster:gpu_count:allocated", - "refId": "A" - } - ], - "title": "GPU requested", - "description": "Sum of GPU requests across all pods cluster-wide, including system namespaces.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "graphMode": "none", - "colorMode": "value", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "green" - }, - { - "value": 1, - "color": "yellow" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "gauge", - "id": 4, - "targets": [ - { - "expr": "cluster:gpu_count:allocated / sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"}) * 100", - "refId": "A" - } - ], - "title": "Allocation ratio", - "description": "Percentage of allocatable GPUs currently requested by pods.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto", - "minVizWidth": 75, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "minVizHeight": 75, - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "percent", - "min": 0, - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "blue" - }, - { - "value": 40, - "color": "green" - }, - { - "value": 80, - "color": "yellow" - }, - { - "value": 95, - "color": "red" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "stat", - "id": 5, - "targets": [ - { - "expr": "count((sum by (namespace, pod) (kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"}) \u003e 0) * on(namespace, pod) group_left() (kube_pod_status_phase{phase=\"Pending\"} == 1)) or vector(0)", - "refId": "A" - } - ], - "title": "Pending pods (GPU)", - "description": "Pods requesting GPUs that are stuck in Pending state — indicates capacity shortage.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "graphMode": "none", - "colorMode": "background", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "green" - }, - { - "value": 1, - "color": "red" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "row", - "collapsed": false, - "title": "Per namespace", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 10, - "panels": [] - }, - { - "type": "bargauge", - "id": 11, - "targets": [ - { - "expr": "sum by (namespace) (namespace:gpu_count:allocated{namespace=~\"$namespace\"})", - "legendFormat": "{{namespace}}", - "refId": "A" - } - ], - "title": "GPU requested per namespace", - "description": "GPU allocation breakdown by namespace — spot top consumers at a glance.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 6 - }, - "repeatDirection": "h", - "options": { - "displayMode": "gradient", - "valueMode": "color", - "namePlacement": "auto", - "showUnfilled": true, - "sizing": "auto", - "minVizWidth": 8, - "minVizHeight": 16, - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false, - "calcs": [] - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "maxVizHeight": 300, - "orientation": "horizontal" - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "min": 0, - "max": 8, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "green" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "timeseries", - "id": 12, - "targets": [ - { - "expr": "sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"})", - "legendFormat": "Allocatable (total)", - "refId": "A" - }, - { - "expr": "sum(namespace:gpu_count:allocated{namespace=~\"$namespace\"})", - "legendFormat": "Requested", - "refId": "B" - } - ], - "title": "GPU allocated over time", - "description": "Requested vs allocatable GPUs over time — shows allocation pressure trends.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 6 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false, - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "custom": { - "drawStyle": "line", - "lineInterpolation": "stepAfter", - "fillOpacity": 10, - "showPoints": "never" - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Allocatable (total)" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - } - ] - } - }, - { - "type": "row", - "collapsed": false, - "title": "Pods with GPU", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 14 - }, - "id": 20, - "panels": [] - }, - { - "type": "table", - "id": 21, - "targets": [ - { - "expr": "kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"} * on(namespace, pod) group_left(phase) (kube_pod_status_phase{phase=~\"Running|Pending\"} == 1)", - "instant": true, - "range": false, - "format": "table", - "refId": "requested" - }, - { - "expr": "kube_pod_container_resource_limits{resource=\"nvidia_com_gpu\", namespace=~\"$namespace\"} * on(namespace, pod) group_left() (kube_pod_status_phase{phase=~\"Running|Pending\"} == 1)", - "instant": true, - "range": false, - "format": "table", - "refId": "limits" - } - ], - "title": "Pods requesting GPU", - "description": "Per-pod GPU requests and limits with scheduling status — Running or Pending.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 15 - }, - "repeatDirection": "h", - "transformations": [ - { - "id": "joinByField", - "options": { - "byField": "pod", - "mode": "outer" - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Time 1": true, - "Time 2": true, - "__name__": true, - "__name__ 1": true, - "__name__ 2": true, - "cluster": true, - "cluster 1": true, - "cluster 2": true, - "container 2": true, - "endpoint": true, - "endpoint 1": true, - "endpoint 2": true, - "instance": true, - "instance 1": true, - "instance 2": true, - "job": true, - "job 1": true, - "job 2": true, - "namespace 2": true, - "node 2": true, - "prometheus": true, - "prometheus 1": true, - "prometheus 2": true, - "resource": true, - "resource 1": true, - "resource 2": true, - "service": true, - "service 1": true, - "service 2": true, - "tenant": true, - "tenant 1": true, - "tenant 2": true, - "tier": true, - "tier 1": true, - "tier 2": true, - "uid": true, - "uid 1": true, - "uid 2": true, - "unit": true, - "unit 1": true, - "unit 2": true - }, - "indexByName": { - "Value #limits": 5, - "Value #requested": 4, - "container 1": 2, - "namespace 1": 0, - "node 1": 3, - "phase": 6, - "pod": 1 - }, - "renameByName": { - "Value #limits": "Limit", - "Value #requested": "Req", - "container 1": "Container", - "namespace 1": "Namespace", - "node 1": "Node", - "phase": "Status" - } - } - } - ], - "options": { - "frameIndex": 0, - "showHeader": true, - "showTypeIcons": false, - "footer": { - "show": false, - "reducer": [] - }, - "cellHeight": "sm" - }, - "fieldConfig": { - "defaults": {}, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Status" - }, - "properties": [ - { - "id": "custom.cellOptions", - "value": { - "mode": "basic", - "type": "color-background" - } - }, - { - "id": "mappings", - "value": [ - { - "options": { - "Failed": { - "color": "red", - "index": 2 - }, - "Pending": { - "color": "orange", - "index": 1 - }, - "Running": { - "color": "green", - "index": 0 - } - }, - "type": "value" - } - ] - } - ] - } - ] - } - } - ], - "templating": { - "list": [ - { - "type": "datasource", - "name": "ds_prometheus", - "label": "Prometheus", - "skipUrlSync": false, - "query": "prometheus", - "current": { - "selected": false, - "text": "default", - "value": "default" - }, - "multi": false, - "allowCustomValue": true, - "includeAll": false, - "regex": "", - "auto": false, - "auto_min": "10s", - "auto_count": 30 - }, - { - "type": "query", - "name": "namespace", - "label": "Namespace", - "skipUrlSync": false, - "query": "label_values(kube_pod_container_resource_requests{resource=\"nvidia_com_gpu\"}, namespace)", - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "multi": true, - "allowCustomValue": true, - "refresh": 2, - "sort": 1, - "includeAll": true, - "auto": false, - "auto_min": "10s", - "auto_count": 30 - } - ] - }, - "annotations": {} -} diff --git a/dashboards/gpu/gpu-tenants.json b/dashboards/gpu/gpu-tenants.json deleted file mode 100644 index 7fef321d..00000000 --- a/dashboards/gpu/gpu-tenants.json +++ /dev/null @@ -1,1098 +0,0 @@ -{ - "uid": "gpu-tenants", - "title": "GPU Tenant Usage", - "description": "Per-tenant GPU consumption: live allocation, utilization, tensor saturation, power, and 24h GPU-hours / kWh — admin view across all namespaces.", - "tags": [ - "gpu", - "tenants", - "admin" - ], - "timezone": "browser", - "editable": true, - "graphTooltip": 1, - "time": { - "from": "now-24h", - "to": "now" - }, - "fiscalYearStartMonth": 0, - "schemaVersion": 42, - "panels": [ - { - "type": "row", - "collapsed": false, - "title": "Tenant fleet snapshot", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "panels": [] - }, - { - "type": "stat", - "id": 2, - "targets": [ - { - "expr": "count(namespace:gpu_count:allocated{namespace=~\"$namespace\"} \u003e 0)", - "refId": "A" - } - ], - "title": "Active tenants", - "description": "Namespaces with at least one GPU request (Pending + Running).", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "graphMode": "none", - "colorMode": "value", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "blue" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "stat", - "id": 3, - "targets": [ - { - "expr": "sum(namespace:gpu_count:allocated{namespace=~\"$namespace\"})", - "refId": "A" - } - ], - "title": "Allocated GPUs", - "description": "Sum of kube GPU requests across tenants. Billable view — includes Pending pods.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "graphMode": "none", - "colorMode": "background", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "blue" - }, - { - "value": 1, - "color": "green" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "stat", - "id": 4, - "targets": [ - { - "expr": "cluster:gpu_util:avg", - "refId": "A" - } - ], - "title": "Avg cluster GPU util", - "description": "Average NVML utilization across all GPUs in the cluster. Cluster-wide — DCGM hardware metrics cannot be attributed to workload namespaces.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 8, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "graphMode": "area", - "colorMode": "value", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "percent", - "min": 0, - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "blue" - }, - { - "value": 30, - "color": "green" - }, - { - "value": 80, - "color": "orange" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "stat", - "id": 5, - "targets": [ - { - "expr": "cluster:gpu_power_watts:sum", - "refId": "A" - } - ], - "title": "Total GPU power", - "description": "Live power draw summed across all GPUs in the cluster. Cluster-wide — DCGM hardware metrics cannot be attributed to workload namespaces.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 12, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "graphMode": "area", - "colorMode": "value", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "watt", - "decimals": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "green" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "stat", - "id": 6, - "targets": [ - { - "expr": "sum(sum_over_time(node:power_watts:sum[24h:1m])) / 60 / 1000", - "refId": "A" - } - ], - "title": "Energy (last 24h)", - "description": "Integrated cluster-wide GPU power draw over 24h. Cluster-wide — DCGM hardware metrics cannot be attributed to workload namespaces.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 16, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "graphMode": "none", - "colorMode": "value", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "kwatth", - "decimals": 1, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "green" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "stat", - "id": 7, - "targets": [ - { - "expr": "sum(sum_over_time(namespace:gpu_count:allocated{namespace=~\"$namespace\"}[24h:1m])) / 60", - "refId": "A" - } - ], - "title": "GPU-hours (last 24h)", - "description": "Sum of (GPU count × time requested) across all tenants over the last 24h. Integrated from kube requests, so Pending pods count.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 20, - "y": 1 - }, - "repeatDirection": "h", - "options": { - "graphMode": "none", - "colorMode": "value", - "justifyMode": "auto", - "textMode": "value", - "wideLayout": true, - "showPercentChange": false, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "percentChangeColorMode": "standard", - "orientation": "" - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "decimals": 1, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "blue" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "row", - "collapsed": false, - "title": "Current allocation", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 10, - "panels": [] - }, - { - "type": "bargauge", - "id": 11, - "targets": [ - { - "expr": "namespace:gpu_count:allocated{namespace=~\"$namespace\"}", - "legendFormat": "{{namespace}}", - "refId": "A" - } - ], - "title": "GPUs per namespace", - "description": "Kube GPU requests per tenant (billable). Includes Pending pods.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 6 - }, - "repeatDirection": "h", - "options": { - "displayMode": "gradient", - "valueMode": "color", - "namePlacement": "auto", - "showUnfilled": true, - "sizing": "auto", - "minVizWidth": 8, - "minVizHeight": 16, - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false, - "calcs": [] - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "maxVizHeight": 300, - "orientation": "horizontal" - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "green" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "bargauge", - "id": 12, - "targets": [ - { - "expr": "node:fb_used_bytes:sum", - "legendFormat": "{{Hostname}}", - "refId": "A" - } - ], - "title": "VRAM used per node", - "description": "FB memory actively used per node. DCGM hardware metrics cannot be attributed to workload namespaces — shown per-node instead.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 6 - }, - "repeatDirection": "h", - "options": { - "displayMode": "gradient", - "valueMode": "color", - "namePlacement": "auto", - "showUnfilled": true, - "sizing": "auto", - "minVizWidth": 8, - "minVizHeight": 16, - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false, - "calcs": [] - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "maxVizHeight": 300, - "orientation": "horizontal" - }, - "fieldConfig": { - "defaults": { - "unit": "bytes", - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "green" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "row", - "collapsed": false, - "title": "Utilization (per-node)", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 14 - }, - "id": 20, - "panels": [] - }, - { - "type": "timeseries", - "id": 21, - "targets": [ - { - "expr": "node:gpu_util:avg", - "legendFormat": "{{Hostname}}", - "refId": "A" - } - ], - "title": "NVML utilization per node", - "description": "Per-node NVML utilization. Stacked to show fleet-wide pressure over time. DCGM hardware metrics cannot be attributed to workload namespaces — shown per-node instead.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 15 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "percent", - "min": 0, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 40, - "showPoints": "never", - "stacking": { - "mode": "normal" - } - } - }, - "overrides": [] - } - }, - { - "type": "timeseries", - "id": 22, - "targets": [ - { - "expr": "node:tensor_active:avg * 100", - "legendFormat": "{{Hostname}}", - "refId": "A" - } - ], - "title": "Tensor saturation per node", - "description": "Real tensor core load per node. Useful to spot nodes with idle tensor cores. DCGM hardware metrics cannot be attributed to workload namespaces — shown per-node instead.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 15 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "percent", - "min": 0, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 40, - "showPoints": "never", - "stacking": { - "mode": "normal" - } - } - }, - "overrides": [] - } - }, - { - "type": "row", - "collapsed": false, - "title": "Power \u0026 allocation trend", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 23 - }, - "id": 30, - "panels": [] - }, - { - "type": "timeseries", - "id": 31, - "targets": [ - { - "expr": "node:power_watts:sum", - "legendFormat": "{{Hostname}}", - "refId": "A" - } - ], - "title": "Power draw per node", - "description": "Per-node GPU power draw. Stacked to see cluster-wide energy burn over time. DCGM hardware metrics cannot be attributed to workload namespaces — shown per-node instead.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "watt", - "min": 0, - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "fillOpacity": 40, - "showPoints": "never", - "stacking": { - "mode": "normal" - } - } - }, - "overrides": [] - } - }, - { - "type": "timeseries", - "id": 32, - "targets": [ - { - "expr": "namespace:gpu_count:allocated{namespace=~\"$namespace\"}", - "legendFormat": "{{namespace}}", - "refId": "A" - } - ], - "title": "Allocated GPUs per namespace", - "description": "Kube-requested GPU count per tenant over time. Step-after interpolation reflects discrete allocation changes.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "repeatDirection": "h", - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "" - } - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "min": 0, - "custom": { - "drawStyle": "line", - "lineInterpolation": "stepAfter", - "fillOpacity": 40, - "showPoints": "never", - "stacking": { - "mode": "normal" - } - } - }, - "overrides": [] - } - }, - { - "type": "row", - "collapsed": false, - "title": "Top consumers (live)", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 32 - }, - "id": 40, - "panels": [] - }, - { - "type": "table", - "id": 41, - "targets": [ - { - "expr": "namespace:gpu_count:allocated{namespace=~\"$namespace\"}", - "instant": true, - "range": false, - "format": "table", - "refId": "gpus" - } - ], - "title": "Top consumers (allocation)", - "description": "Live ranking of tenants by GPU allocation. Hardware metrics (util, tensor, power, VRAM) cannot be attributed to workload namespaces — see Fleet and Performance dashboards for per-node/per-GPU hardware views.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 33 - }, - "repeatDirection": "h", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Time 1": true, - "Time 2": true, - "Time 3": true, - "Time 4": true, - "Time 5": true, - "__name__": true, - "__name__ 1": true, - "__name__ 2": true, - "__name__ 3": true, - "__name__ 4": true, - "__name__ 5": true, - "cluster": true, - "cluster 1": true, - "cluster 2": true, - "cluster 3": true, - "cluster 4": true, - "cluster 5": true, - "endpoint": true, - "endpoint 1": true, - "endpoint 2": true, - "endpoint 3": true, - "endpoint 4": true, - "endpoint 5": true, - "instance": true, - "instance 1": true, - "instance 2": true, - "instance 3": true, - "instance 4": true, - "instance 5": true, - "job": true, - "job 1": true, - "job 2": true, - "job 3": true, - "job 4": true, - "job 5": true, - "prometheus": true, - "prometheus 1": true, - "prometheus 2": true, - "prometheus 3": true, - "prometheus 4": true, - "prometheus 5": true, - "service": true, - "service 1": true, - "service 2": true, - "service 3": true, - "service 4": true, - "service 5": true, - "tenant": true, - "tenant 1": true, - "tenant 2": true, - "tenant 3": true, - "tenant 4": true, - "tenant 5": true, - "tier": true, - "tier 1": true, - "tier 2": true, - "tier 3": true, - "tier 4": true, - "tier 5": true, - "uid": true, - "uid 1": true, - "uid 2": true, - "uid 3": true, - "uid 4": true, - "uid 5": true, - "unit": true, - "unit 1": true, - "unit 2": true, - "unit 3": true, - "unit 4": true, - "unit 5": true - }, - "indexByName": { - "Value #gpus": 1, - "namespace": 0 - }, - "renameByName": { - "Value #gpus": "GPUs", - "namespace": "Namespace" - } - } - } - ], - "options": { - "frameIndex": 0, - "showHeader": true, - "showTypeIcons": false, - "sortBy": [ - { - "displayName": "GPUs", - "desc": true - } - ], - "footer": { - "show": false, - "reducer": [] - }, - "cellHeight": "sm" - } - }, - { - "type": "row", - "collapsed": false, - "title": "Historical usage (last 24h)", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 43 - }, - "id": 50, - "panels": [] - }, - { - "type": "bargauge", - "id": 51, - "targets": [ - { - "expr": "sum_over_time(namespace:gpu_count:allocated{namespace=~\"$namespace\"}[24h:1m]) / 60", - "legendFormat": "{{namespace}}", - "refId": "A" - } - ], - "title": "GPU-hours per namespace (last 24h)", - "description": "Billable GPU-hours per tenant over the last 24h window.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 44 - }, - "repeatDirection": "h", - "options": { - "displayMode": "gradient", - "valueMode": "color", - "namePlacement": "auto", - "showUnfilled": true, - "sizing": "auto", - "minVizWidth": 8, - "minVizHeight": 16, - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false, - "calcs": [] - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "maxVizHeight": 300, - "orientation": "horizontal" - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "decimals": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "blue" - } - ] - } - }, - "overrides": [] - } - }, - { - "type": "bargauge", - "id": 52, - "targets": [ - { - "expr": "sum_over_time(node:power_watts:sum[24h:1m]) / 60 / 1000", - "legendFormat": "{{Hostname}}", - "refId": "A" - } - ], - "title": "Energy per node (last 24h)", - "description": "kWh consumed per node over the last 24h. Integrated from 1-minute power samples. DCGM hardware metrics cannot be attributed to workload namespaces — shown per-node instead.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 44 - }, - "repeatDirection": "h", - "options": { - "displayMode": "gradient", - "valueMode": "color", - "namePlacement": "auto", - "showUnfilled": true, - "sizing": "auto", - "minVizWidth": 8, - "minVizHeight": 16, - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false, - "calcs": [] - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "maxVizHeight": 300, - "orientation": "horizontal" - }, - "fieldConfig": { - "defaults": { - "unit": "kwatth", - "decimals": 2, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "value": null, - "color": "green" - } - ] - } - }, - "overrides": [] - } - } - ], - "templating": { - "list": [ - { - "type": "datasource", - "name": "ds_prometheus", - "label": "Prometheus", - "skipUrlSync": false, - "query": "prometheus", - "current": { - "selected": false, - "text": "default", - "value": "default" - }, - "multi": false, - "allowCustomValue": true, - "includeAll": false, - "regex": "", - "auto": false, - "auto_min": "10s", - "auto_count": 30 - }, - { - "type": "query", - "name": "namespace", - "label": "Namespace", - "skipUrlSync": false, - "query": "label_values(namespace:gpu_count:allocated, namespace)", - "datasource": { - "type": "prometheus", - "uid": "$ds_prometheus" - }, - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "multi": true, - "allowCustomValue": true, - "refresh": 2, - "sort": 1, - "includeAll": true, - "auto": false, - "auto_min": "10s", - "auto_count": 30 - } - ] - }, - "annotations": {} -} diff --git a/docs/agents/changelog.md b/docs/agents/changelog.md index 2fde1224..feb70597 100644 --- a/docs/agents/changelog.md +++ b/docs/agents/changelog.md @@ -6,20 +6,6 @@ This file contains detailed instructions for AI-powered IDE on how to generate c Follow these instructions when the user explicitly asks to generate a changelog. -## Scope and boundaries - -**Your single deliverable is the file `docs/changelogs/v.md`.** Write the complete, verified changelog to that path. That is the entire task. Exit as soon as the file is written and verified against the checklist in Step 9. - -Unless the caller explicitly instructs otherwise: - -- **In the cozystack working tree**, do not run `git commit`, `git push`, `git checkout` (to switch branches), `git branch`, `git tag`, `git reset`, `git merge`, or `git rebase`. Do not write to local branches, tags, or HEAD. `git fetch` is expected and fine (see the read-only analysis list below). -- **Do not** push to any remote, open pull requests, or issue GitHub API write calls (POST / PATCH / DELETE) for any repository. -- In the cozystack working tree, the **only** file you create or modify is `docs/changelogs/v.md`. Cloning auxiliary repositories under `_repos/` for cross-repo analysis (see Step 6) is fine; local git operations inside those disposable clones (`git checkout`, `git pull`, etc.) are allowed — just never push from them or open PRs against them. - -The caller — a GitHub Actions workflow in CI, or a developer running you interactively — owns branching, committing, pushing, and PR creation. They will perform those actions after you exit. Do not pre-empt them even if the working tree looks ready. - -Read-only analysis is expected and encouraged: `git log`, `git show`, `git fetch`, `git diff`, `gh pr view`, `gh api` GET requests, and reading any file in the repository. - ## Required Tools Before generating changelogs, ensure you have access to `gh` (GitHub CLI) tool, which is used to fetch commit and PR author information. The GitHub CLI is used to correctly identify PR authors from commits and pull requests. @@ -36,7 +22,7 @@ When the user asks to generate a changelog, follow these steps in the specified - [ ] Step 5: Get the list of commits for the release period - [ ] Step 6: Check additional repositories (website is REQUIRED, optional repos if tags exist) - [ ] **MANDATORY**: Check website repository for documentation changes WITH authors and PR links via GitHub CLI - - [ ] **MANDATORY**: Check ALL optional repositories (talm, boot-to-talos, cozyhr, cozy-proxy, external-apps-example, ansible-cozystack) for tags during release period + - [ ] **MANDATORY**: Check ALL optional repositories (talm, boot-to-talos, cozyhr, cozy-proxy) for tags during release period - [ ] **MANDATORY**: For ALL commits from additional repos, get GitHub username via CLI, prioritizing PR author over commit author. - [ ] Step 7: Analyze commits (extract PR numbers, authors, user impact) - [ ] **MANDATORY**: For EVERY PR in main repo, get PR author via `gh pr view --json author --jq .author.login` (do NOT skip this step) @@ -162,8 +148,6 @@ Cozystack release may include changes from related repositories. Check and inclu - [https://github.com/cozystack/boot-to-talos](https://github.com/cozystack/boot-to-talos) - [https://github.com/cozystack/cozyhr](https://github.com/cozystack/cozyhr) - [https://github.com/cozystack/cozy-proxy](https://github.com/cozystack/cozy-proxy) -- [https://github.com/cozystack/external-apps-example](https://github.com/cozystack/external-apps-example) -- [https://github.com/cozystack/ansible-cozystack](https://github.com/cozystack/ansible-cozystack) **⚠️ IMPORTANT**: You MUST check ALL optional repositories for tags created during the release period. Do NOT skip this step even if you think there might not be any tags. Use the process below to verify. @@ -211,7 +195,7 @@ Cozystack release may include changes from related repositories. Check and inclu 3. **For optional repositories, check if tags exist during release period:** - **⚠️ MANDATORY: You MUST check ALL optional repositories (talm, boot-to-talos, cozyhr, cozy-proxy, external-apps-example, ansible-cozystack). Do NOT skip any repository!** + **⚠️ MANDATORY: You MUST check ALL optional repositories (talm, boot-to-talos, cozyhr, cozy-proxy). Do NOT skip any repository!** **Use the helper script:** ```bash @@ -224,7 +208,7 @@ Cozystack release may include changes from related repositories. Check and inclu ``` The script will: - - Check ALL optional repositories (talm, boot-to-talos, cozyhr, cozy-proxy, external-apps-example, ansible-cozystack) + - Check ALL optional repositories (talm, boot-to-talos, cozyhr, cozy-proxy) - Look for tags created during the release period - Get commits between tags (if tags exist) or by date range (if no tags) - Extract PR numbers from commit messages @@ -585,7 +569,7 @@ Create a new changelog file in the format matching previous versions: - [ ] Step 5 completed: **ALL commits included** (including merge commits and backports) - do not skip any commits - [ ] Step 5 completed: **Backports identified and handled correctly** - original PR author used, both original and backport PR numbers included - [ ] Step 6 completed: Website repository checked for documentation changes WITH authors and PR links via GitHub CLI -- [ ] Step 6 completed: **ALL** optional repositories (talm, boot-to-talos, cozyhr, cozy-proxy, external-apps-example, ansible-cozystack) checked for tags during release period +- [ ] Step 6 completed: **ALL** optional repositories (talm, boot-to-talos, cozyhr, cozy-proxy) checked for tags during release period - [ ] Step 6 completed: For ALL commits from additional repos, GitHub username obtained via GitHub CLI (not skipped). For commits with PR numbers, PR author used via `gh pr view` (not commit author) - [ ] Step 7 completed: For EVERY PR in main repo (including backports), PR author obtained via `gh pr view --json author --jq .author.login` (not skipped or assumed). Commit author NOT used - always use PR author - [ ] Step 7 completed: **Backports verified** - for each backport PR, original PR found and original PR author used in changelog @@ -622,8 +606,6 @@ Create a new changelog file in the format matching previous versions: **Save the changelog:** Save the changelog to file `docs/changelogs/v.md` according to the version for which the changelog is being generated. -**Then exit.** Do not commit, push, create a branch, or open a pull request — the caller handles all git and GitHub operations after you return. See the "Scope and boundaries" section at the top of this document. - ### Important notes - **After fetch with --force** local tags are up-to-date, use them for work @@ -646,7 +628,7 @@ Save the changelog to file `docs/changelogs/v.md` according to the vers - **Additional repositories (Step 6) - MANDATORY**: - **⚠️ CRITICAL**: Always check the **website** repository for documentation changes during the release period. This is a required step and MUST NOT be skipped. - - **⚠️ CRITICAL**: You MUST check ALL optional repositories (talm, boot-to-talos, cozyhr, cozy-proxy, external-apps-example, ansible-cozystack) for tags during the release period. Do NOT skip any repository even if you think there might not be tags. + - **⚠️ CRITICAL**: You MUST check ALL optional repositories (talm, boot-to-talos, cozyhr, cozy-proxy) for tags during the release period. Do NOT skip any repository even if you think there might not be tags. - **CRITICAL**: For ALL entries from additional repositories (website and optional), you MUST: - **MANDATORY**: Extract PR number from commit message first - **MANDATORY**: For commits with PR numbers, ALWAYS use `gh pr view --repo cozystack/ --json author --jq .author.login` to get PR author (not commit author) @@ -655,7 +637,7 @@ Save the changelog to file `docs/changelogs/v.md` according to the vers - **MANDATORY**: Do NOT use commit author for PRs - always use PR author - Include PR link or commit hash reference - Format: `* **[repo] Description**: details ([**@username**](https://github.com/username) in cozystack/repo#123)` - - For **optional repositories** (talm, boot-to-talos, cozyhr, cozy-proxy, external-apps-example, ansible-cozystack), you MUST check ALL of them for tags during the release period. Use the loop provided in Step 6 to check each repository systematically. + - For **optional repositories** (talm, boot-to-talos, cozyhr, cozy-proxy), you MUST check ALL of them for tags during the release period. Use the loop provided in Step 6 to check each repository systematically. - When including changes from additional repositories, use the format: `[repo-name] Description` and link to the repository's PR/issue if available - **Prefer PR numbers over commit hashes**: For commits from additional repositories, extract PR number from commit message using GitHub API. Use PR format (`cozystack/website#123`) instead of commit hash (`cozystack/website@abc1234`) when available - **Never add entries without author and PR/commit reference**: Every entry from additional repositories must have both author and link diff --git a/docs/agents/contributing.md b/docs/agents/contributing.md index 88b062ea..d5a4366e 100644 --- a/docs/agents/contributing.md +++ b/docs/agents/contributing.md @@ -1,140 +1,167 @@ -# Contributing Conventions for AI Agents +# Instructions for AI Agents -Project-side conventions for commits, branches, and pull requests in Cozystack. +Guidelines for AI agents contributing to Cozystack. ## Checklist for Creating a Pull Request -- [ ] Commit message follows Conventional Commits format +- [ ] Changes are made and tested +- [ ] Commit message uses correct `[component]` prefix - [ ] Commit is signed off with `--signoff` - [ ] Branch is rebased on `upstream/main` (no extra commits) - [ ] PR body includes description and release note -- [ ] Ran `make generate` in every package whose `values.yaml`, `values.schema.json`, `Chart.yaml`, or `README.md` was touched, and committed the regenerated files +- [ ] PR is pushed and created with `gh pr create` -## Regenerate Artifacts Before Committing +## How to Commit and Create Pull Requests -Several files in each package are produced by `make generate` from `values.yaml` + `values.schema.json` and must stay in sync with the hand-edited sources: +### 1. Make Your Changes -- `packages/(apps|extra)//README.md` — regenerated by `cozyvalues-gen` (parameter table, formatting). -- `packages/(apps|extra)//values.schema.json` — `cozyvalues-gen` rewrites ordering and derived fields. -- `packages/system/-rd/cozyrds/.yaml` — produced by `hack/update-crd.sh`, which `make generate` invokes. +Edit the necessary files in the codebase. -**Before committing edits to any of those sources**, run `make generate` inside the package and stage the full diff: +### 2. Commit with Proper Format + +Use the `[component]` prefix and `--signoff` flag: ```bash -make -C packages// generate -git add packages/// packages/system/-rd/ +git commit --signoff -m "[component] Brief description of changes" ``` -The repo's pre-commit CI job runs `make generate` in every package and then `git diff --exit-code`. Any unstaged generator output fails the job with exit code 123 and blocks the PR. Also rerun `make generate` after a `git commit --amend` if the amended change touched any of the sources above. - -To locate packages a WIP branch likely needs to be regenerated: - -```bash -git diff --name-only | xargs -n1 dirname | sort -u | grep ^packages/ -``` - -## Commit Format - -Follow [Conventional Commits](https://www.conventionalcommits.org/) with `--signoff`: - -```bash -git commit --signoff -m "type(scope): brief description" -``` - -**Types:** `feat`, `fix`, `docs`, `style`, `refactor`, `perf`, `test`, `build`, `ci`, `chore` - -**Scopes** (examples — not an exhaustive list; pick the most specific scope that describes the change, and introduce a new one if a genuinely new area needs its own): - -- System, e.g.: `dashboard`, `platform`, `operator`, `cilium`, `kube-ovn`, `linstor`, `fluxcd`, `cluster-api` -- Apps, e.g.: `postgres`, `mariadb`, `redis`, `kafka`, `clickhouse`, `virtual-machine`, `kubernetes` -- Other, e.g.: `api`, `hack`, `tests`, `ci`, `docs`, `agents`, `maintenance` - -Breaking changes: append `!` after type/scope (`feat(api)!: ...`) or add a `BREAKING CHANGE:` footer. +**Component prefixes:** +- System: `[dashboard]`, `[platform]`, `[cilium]`, `[kube-ovn]`, `[linstor]`, `[fluxcd]`, `[cluster-api]` +- Apps: `[postgres]`, `[mariadb]`, `[redis]`, `[kafka]`, `[clickhouse]`, `[virtual-machine]`, `[kubernetes]` +- Other: `[tests]`, `[ci]`, `[docs]`, `[maintenance]` **Examples:** ```bash -git commit --signoff -m "feat(dashboard): add config hash annotations to restart pods on config changes" -git commit --signoff -m "fix(postgres): update operator to version 1.2.3" -git commit --signoff -m "docs(contributing): add installation guide" +git commit --signoff -m "[dashboard] Add config hash annotations to restart pods on config changes" +git commit --signoff -m "[postgres] Update operator to version 1.2.3" +git commit --signoff -m "[docs] Add installation guide" ``` -## PR Title Auto-Labeling +### 3. Rebase on upstream/main (if needed) -`.github/workflows/pr-labeler.yaml` parses the PR title on `opened`, `edited`, `reopened`, and `synchronize` events and applies labels additively (never removes). The title is expected to follow Conventional Commits — same format as commit messages above. - -**Type → `kind/*`:** - -| type | label | -| --------- | ------------------ | -| feat | kind/feature | -| fix | kind/bug | -| docs | kind/documentation | -| chore | kind/cleanup | -| refactor | kind/cleanup | -| style, perf, test, build, ci, revert | (no kind label) | - -**Scope → `area/*`** (full mapping in `.github/workflows/pr-labeler.yaml`): - -| scope (examples) | label | -| --- | --- | -| agents, ai | area/ai | -| api, cozystack-api | area/api | -| build | area/build | -| ci | area/ci | -| dashboard | area/dashboard | -| postgres, mariadb, redis, etcd, kafka, clickhouse, postgres-operator, mariadb-operator | area/database | -| extra | area/extra | -| kubernetes | area/kubernetes | -| monitoring, vlogs, vmstack, grafana, workloadmonitor | area/monitoring | -| ingress, gateway, vpn, metallb, cilium, kube-ovn, cozy-proxy, ... | area/networking | -| platform, bundle, flux, fluxcd, cluster-api, talos, installer, cozyctl, cozystack-engine, cozy-lib | area/platform | -| backport, release | area/release | -| seaweedfs, bucket, linstor, velero, harbor, backups | area/storage | -| tests, e2e | area/testing | -| kubevirt, cdi, vmi, vm-import, virtual-machine, hami, gpu-operator | area/virtualization | - -**Special handling:** - -- `[Backport release-1.x]` prefix is stripped before parsing; `area/release` and `backport` labels are added. -- Composite scope (`feat(platform, system, apps): ...`) — each comma-separated part is mapped independently. -- `!` after type or `BREAKING CHANGE:` footer in the body → `kind/breaking-change`. -- Unmapped scope or non-conventional title → `area/uncategorized` (signals the PR needs manual area selection). -- Bracket-style fallback (`[scope] description`) maps `scope` → `area/*` but cannot infer `kind/*`. - -### AI Agent Attribution - -When an AI agent authors or materially assists with a commit, add an `Assisted-By:` trailer naming the model: - -```text -Assisted-By: Claude -Assisted-By: GPT-5 -Assisted-By: Gemini -``` - -This sits alongside the `Signed-off-by:` trailer produced by `--signoff`. Use one trailer per model if multiple contributed. - -## Rebasing on upstream/main - -If the branch has extra commits, clean it up: +If your branch has extra commits, clean it up: ```bash +# Fetch latest git fetch upstream + +# Create clean branch from upstream/main git checkout -b my-feature upstream/main + +# Cherry-pick only your commit git cherry-pick -git push -f origin my-feature + +# Force push to your branch +git push -f origin my-feature:my-branch-name ``` -## Pull Request Body +### 4. Push Your Branch -Fill in the template at [`.github/PULL_REQUEST_TEMPLATE.md`](../../.github/PULL_REQUEST_TEMPLATE.md). It includes the required `release-note` block. +```bash +git push origin +``` -Create the PR with `gh pr create --title "type(scope): brief description" --body-file `. +### 5. Create Pull Request -## Fetching Unresolved Review Comments +Write the PR body to a temporary file: -Cozystack uses GitHub review threads with resolution status. Only unresolved threads are actionable — resolved threads are already handled. +```bash +cat > /tmp/pr_body.md << 'EOF' +## What this PR does -The REST endpoint `/pulls/{pr}/reviews` returns review summaries, not individual review comments. Use the GraphQL API to access `reviewThreads` with `isResolved` status: +Brief description of the changes. + +Changes: +- Change 1 +- Change 2 + +### Release note + +```release-note +[component] Description for changelog +``` +EOF +``` + +Create the PR: + +```bash +gh pr create --title "[component] Brief description" --body-file /tmp/pr_body.md +``` + +Clean up: + +```bash +rm /tmp/pr_body.md +``` + +## Addressing AI Bot Reviewer Comments + +When the user asks to fix comments from AI bot reviewers (like Qodo, Copilot, etc.): + +### 1. Get PR Comments + +View all comments on the pull request: + +```bash +gh pr view --comments +``` + +Or for the current branch: + +```bash +gh pr view --comments +``` + +### 2. Review Each Comment Carefully + +**Important**: Do NOT blindly apply all suggestions. Each comment should be evaluated: + +- **Consider context** - Does the suggestion make sense for this specific case? +- **Check project conventions** - Does it align with Cozystack patterns? +- **Evaluate impact** - Will this improve code quality or introduce issues? +- **Question validity** - AI bots can be wrong or miss context + +**When to apply:** +- ✅ Legitimate bugs or security issues +- ✅ Clear improvements to code quality +- ✅ Better error handling or edge cases +- ✅ Conformance to project conventions + +**When to skip:** +- ❌ Stylistic preferences that don't match project style +- ❌ Over-engineering simple code +- ❌ Changes that break existing patterns +- ❌ Suggestions that show misunderstanding of the code + +### 3. Apply Valid Fixes + +Make changes addressing the valid comments. Use your judgment. + +### 4. Leave Changes Uncommitted + +**Critical**: Do NOT commit or push the changes automatically. + +Leave the changes in the working directory so the user can: +- Review the fixes +- Decide whether to commit them +- Make additional adjustments if needed + +```bash +# After making changes, show status but DON'T commit +git status +git diff +``` + +The user will commit and push when ready. + +## Code Review Comments + +When asked to fix code review comments, **always work only with unresolved (open) comments**. Resolved comments should be ignored as they have already been addressed. + +### Getting Unresolved Review Comments + +Use GitHub GraphQL API to fetch only unresolved review comments from a pull request: ```bash gh api graphql -F owner=cozystack -F repo=cozystack -F pr= -f query=' @@ -162,7 +189,27 @@ query($owner: String!, $repo: String!, $pr: Int!) { }' --jq '.data.repository.pullRequest.reviewThreads.nodes[] | select(.isResolved == false) | .comments.nodes[]' ``` -Compact one-line variant: +### Filtering for Unresolved Comments + +The key filter is `select(.isResolved == false)` which ensures only unresolved review threads are processed. Each thread can contain multiple comments, but if the thread is resolved, all its comments should be ignored. + +### Working with Review Comments + +1. **Fetch unresolved comments** using the GraphQL query above +2. **Parse the results** to identify: + - File path (`path`) + - Line number (`line` or `originalLine`) + - Comment text (`bodyText`) + - Author (`author.login`) +3. **Address each unresolved comment** by: + - Locating the relevant code section + - Making the requested changes + - Ensuring the fix addresses the concern raised +4. **Do NOT process resolved comments** - they have already been handled + +### Example: Compact List of Unresolved Comments + +For a quick overview of unresolved comments: ```bash gh api graphql -F owner=cozystack -F repo=cozystack -F pr= -f query=' @@ -186,3 +233,43 @@ query($owner: String!, $repo: String!, $pr: Int!) { } }' --jq '.data.repository.pullRequest.reviewThreads.nodes[] | select(.isResolved == false) | .comments.nodes[] | "\(.path):\(.line // "N/A") - \(.author.login): \(.bodyText[:150])"' ``` + +### Important Notes + +- **REST API limitation**: The REST endpoint `/pulls/{pr}/reviews` returns review summaries, not individual review comments. Use GraphQL API for accessing `reviewThreads` with `isResolved` status. +- **Thread-based resolution**: Comments are organized in threads. If a thread is resolved (`isResolved: true`), ignore all comments in that thread. +- **Always filter**: Never process comments from resolved threads, even if they appear in the results. + +### Example Workflow + +```bash +# Get PR comments +gh pr view 1234 --comments + +# Review comments and identify valid ones +# Make necessary changes to address valid comments +# ... edit files ... + +# Show what was changed (but don't commit) +git status +git diff + +# Tell the user what was fixed and what was skipped +``` + +## Git Permissions + +Request these permissions when needed: +- `git_write` - For commit, rebase, cherry-pick, branch operations +- `network` - For push, fetch, pull operations + +## Common Issues + +**PR has extra commits?** +→ Rebase on `upstream/main` and cherry-pick only your commits + +**Wrong commit message?** +→ `git commit --amend --signoff -m "[correct] message"` then `git push -f` + +**Need to update PR?** +→ `gh pr edit --body "new description"` diff --git a/docs/agents/overview.md b/docs/agents/overview.md index ae0fd1c4..b6d69e68 100644 --- a/docs/agents/overview.md +++ b/docs/agents/overview.md @@ -78,18 +78,11 @@ packages/// - Add proper error handling and structured logging ### Git Commits -- Follow [Conventional Commits](https://www.conventionalcommits.org/) format: `type(scope): description` +- Use format: `[component] Description` - Always use `--signoff` flag - Reference PR numbers when available - Keep commits atomic and focused - -### PackageSource CRD upgrade policy - -Each component in a `PackageSource` may set `install.upgradeCRDs` to control how CRDs from the chart's `crds/` directory are handled on `HelmRelease` upgrades. Allowed values: `Skip` (default — helm-controller does not touch CRDs on upgrade), `Create` (create new CRDs only), `CreateReplace` (create new and overwrite existing). - -Set `upgradeCRDs: CreateReplace` for operators whose upstream regularly adds new CRDs between versions (etcd-operator, cnpg, kubevirt, kamaji). Without it, new CRDs from a chart bump do not land on existing clusters — only fresh installs get them. - -Do **not** set `CreateReplace` blindly: it overwrites every CRD in `crds/` and can cause silent data loss if upstream drops a field from a CRD that has live objects. Only enable it for operators whose schema evolution is additive-only. When in doubt, leave it unset and apply new CRDs manually. +- Follow conventional commit format for changelogs ### Documentation diff --git a/docs/changelogs/v1.1.6.md b/docs/changelogs/v1.1.6.md deleted file mode 100644 index 77c49475..00000000 --- a/docs/changelogs/v1.1.6.md +++ /dev/null @@ -1,19 +0,0 @@ - - -## Fixes - -* **[build] Filter git describe to match only v* tags**: Adds `--match 'v*'` to all `git describe` calls in `hack/common-envs.mk`. The `api/apps/v1alpha1/*` subtags share the same commit as release tags, causing `git describe --exact-match` to pick `api/apps/v1alpha1/vX.Y.Z` instead of `vX.Y.Z`, producing invalid Docker image tags ([**@kvaps**](https://github.com/kvaps) in #2386, #2388). - -## Development, Testing, and CI/CD - -* **[ci] Replace cozystack-bot PAT with cozystack-ci GitHub App**: Replaces the long-lived `cozystack-bot` personal access token with short-lived, scoped tokens from the `cozystack-ci` GitHub App across all release workflows. Improves security and auditability of CI operations ([**@tym83**](https://github.com/tym83) in #2351). - -* **[ci] Replace GH_PAT with cozystack-ci GitHub App token in pull-requests workflow**: Switches the pull-requests release workflow to use the cozystack-ci GitHub App token instead of the personal access token ([**@kvaps**](https://github.com/kvaps) in #2383). - -* **[ci] Use cozystack org noreply email for bot commits**: Updates CI workflows to use the cozystack organization noreply email for bot commits ([**@kvaps**](https://github.com/kvaps) in #2392). - ---- - -**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.1.5...v1.1.6 diff --git a/docs/changelogs/v1.2.2.md b/docs/changelogs/v1.2.2.md deleted file mode 100644 index 2b6a1ae5..00000000 --- a/docs/changelogs/v1.2.2.md +++ /dev/null @@ -1,63 +0,0 @@ - - -## Features and Improvements - -* **[linstor] Update piraeus-server to v1.33.2 with selected backports**: Bumps LINSTOR server from v1.33.1 to v1.33.2 and adds backported patches for improved storage reliability: a stale bitmap adjust retry mechanism for automatic recovery after bitmap attach errors, LUKS2 header sizing and optimal I/O size detection improvements for more reliable disk formatting, and the maintainer implementation backport. All patches verified against upstream v1.33.2 with `git apply --check` and `gradlew compileJava` ([**@kvaps**](https://github.com/kvaps) in #2331, #2377). - -## Fixes - -* **[postgres] Fix system PostgreSQL images to 17.7-standard-trixie**: Hardcodes PostgreSQL 17.7-standard-trixie images for system PostgreSQL instances. This ensures system databases use the correct image variant consistent with the monitoring stack requirements introduced in v1.2.1 ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2364, #2369). - -* **[cilium] Opt-out of cri-containerd.apparmor.d for nsenter init containers**: On Ubuntu 22.04+, Debian, and other distributions that load the `cri-containerd.apparmor.d` AppArmor profile by default for containerd workloads, the kernel denied `nsenter` namespace entry in cilium-agent init containers (`mount-cgroup`, `apply-sysctl-overwrites`, `clean-cilium-state`), causing the agent to land in `Init:CrashLoopBackOff` and cascading platform failures. Per-container `container.apparmor.security.beta.kubernetes.io` annotations now opt the affected containers out of this profile, applied only on non-Talos cilium variants (`cilium-generic`, `kubeovn-cilium-generic`). The vendored daemonset template is also patched to strip the upstream `semverCompare "<1.30.0"` AppArmor block, preventing duplicate annotation keys. Talos variants are untouched as Talos does not load the AppArmor LSM ([**@lexfrei**](https://github.com/lexfrei) in #2370, #2378). - -* **[virtual-machine] Exclude external VM services from Cilium BPF LB**: Adds the `service.kubernetes.io/service-proxy-name: "cozy-proxy"` label to VM LoadBalancer services when `external: true`, telling Cilium to skip BPF processing entirely for these services. This fixes two issues: inter-tenant connectivity via public LB IPs (Cilium's DNAT caused cross-tenant pod-to-pod flow classification, triggering CiliumClusterwideNetworkPolicy blocks) and WholeIP broken on Cilium 1.19+ (wildcard service drop entries blocked traffic to LB IPs on undeclared ports before it reached netfilter/cozy-proxy). MetalLB L2 advertisement and kube-ovn routing remain unaffected ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #2357, #2361). - -* **[monitoring] Fix infra dashboards missing in default variant**: The default platform variant deploys the monitoring chart to the `cozy-monitoring` namespace, but the dashboard rendering condition introduced in #2197 only checked for `tenant-root`. Infrastructure dashboards were not rendered in the default variant. The `cozy-monitoring` namespace is now included in the rendering condition, consistent with the existing pattern in `vmagent.yaml` ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #2365, #2367). - -* **[build] Filter git describe to match only v* tags**: Adds `--match 'v*'` to all `git describe` calls in `hack/common-envs.mk`. The `api/apps/v1alpha1/*` subtags share the same commit as release tags, causing `git describe --exact-match` to pick `api/apps/v1alpha1/vX.Y.Z` instead of `vX.Y.Z`, producing invalid Docker image tags ([**@kvaps**](https://github.com/kvaps) in #2386, #2389). - -## Development, Testing, and CI/CD - -* **[ci] Replace cozystack-bot PAT with cozystack-ci GitHub App**: Replaces the long-lived `cozystack-bot` personal access token with short-lived, scoped tokens from the `cozystack-ci` GitHub App across all release workflows (`tags.yaml`, `auto-release.yaml`, `pull-requests-release.yaml`). Improves security and auditability of CI operations ([**@tym83**](https://github.com/tym83) in #2351). - -* **[ci] Use cozystack org noreply email for bot commits**: Updates CI workflows to use the cozystack organization noreply email for bot commits ([**@kvaps**](https://github.com/kvaps) in #2392, #2393). - -* **[ci] Replace GH_PAT with cozystack-ci GitHub App token in pull-requests workflow**: Switches the pull-requests release workflow to use the cozystack-ci GitHub App token instead of the personal access token ([**@kvaps**](https://github.com/kvaps) in #2383, #2384). - -## Documentation - -* **[website] Add ApplicationDefinition naming convention reference**: Added reference documentation on ApplicationDefinition naming conventions and how `cozystack-api` resolves kinds to their backing definitions ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#478). - -* **[website] Document Talos / talosctl / Cozystack version pairing**: Added documentation covering Talos, talosctl, and Cozystack version compatibility matrix for installation ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#484). - -* **[website] Fix KubeOVN MASTER_NODES example path and key in troubleshooting**: Corrected the MASTER_NODES example path and key in the KubeOVN troubleshooting guide ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#483). - -* **[website] Prefix bundle package names with cozystack. in v1 examples**: Updated documentation examples to use the correct `cozystack.` prefix for bundle package names in enabled/disabledPackages ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#482). - -* **[website] Finish isolated-field removal and document opt-in policy labels**: Removed the obsolete `isolated` field from tenant documentation and documented the new opt-in policy labels approach ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#481). - -* **[website] Add --take-ownership flag and describe networking.* fields**: Added documentation for the `--take-ownership` flag and described the `networking.*` fields in the installation guide ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#480). - -* **[website] Add bonding (LACP) configuration how-to guide**: Added a guide for configuring network bonding with LACP on Cozystack installations ([**@sircthulhu**](https://github.com/sircthulhu) in cozystack/website#459). - -* **[website] Improve registry mirrors for tenant Kubernetes in air-gapped guide**: Improved documentation for configuring registry mirrors in tenant Kubernetes clusters for air-gapped environments ([**@sircthulhu**](https://github.com/sircthulhu) in cozystack/website#461). - -* **[website] Update backup/restore documentation for VMI/VMDisk**: Updated backup documentation with information related to VM instance and VM disk restore improvements ([**@androndo**](https://github.com/androndo) in cozystack/website#466). - -* **[website] Add updated OpenAPI spec**: Updated the OpenAPI specification for managed applications reference ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#469). - -* **[website] Add OSS Health pages and OpenSSF badge**: Added OSS Health section with OpenSSF Scorecard and Best Practices badge to the website footer ([**@tym83**](https://github.com/tym83) in cozystack/website#470). - -* **[website] Add CozySummit Virtual 2026 program announcement**: Published the CozySummit Virtual 2026 program announcement blog post ([**@tym83**](https://github.com/tym83) in cozystack/website#472). - -* **[website] Add missing release announcements for v0.1–v0.41**: Backfilled missing release announcement blog posts for Cozystack versions v0.1 through v0.41 ([**@tym83**](https://github.com/tym83) in cozystack/website#468). - -* **[talm] Render templates online in apply to resolve lookups**: Fixed talm `apply` command to render templates online, resolving template lookup failures when using modeline templates ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/talm#119). - -* **[talm] Update default Talos image to v1.12.6**: Updated the default Talos image version to v1.12.6 in talm ([**@kvaps**](https://github.com/kvaps) in cozystack/talm@03e9b6e). - ---- - -**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.2.1...v1.2.2 diff --git a/docs/changelogs/v1.2.3.md b/docs/changelogs/v1.2.3.md deleted file mode 100644 index 6d1b3b74..00000000 --- a/docs/changelogs/v1.2.3.md +++ /dev/null @@ -1,58 +0,0 @@ - - -# v1.2.3 (2026-04-20) - -A patch release with bug fixes and documentation updates. - -## Features and Improvements - -_No notable features in this patch release._ - -## Fixes - -* **fix(kubernetes): set explicit ephemeral-storage on virt-launcher pods**: Prevents VM crashes caused by ephemeral-storage eviction by setting explicit `domain.resources` ephemeral-storage on the VirtualMachine spec. Uses sanitized limits and requests so virt-launcher pods do not inherit too-small namespace defaults. ([**@kvaps**](https://github.com/kvaps) in #2317, backport #2423). - -## Documentation - -* **[website] feat: add Telemetry page under OSS Health section**: Add Telemetry page and initial data seeding to OSS Health docs ([**@tym83**](https://github.com/tym83) in cozystack/website#471). -* **[website] Refactor docs versions to major.minor variants**: Move docs to major.minor versioning for v1.x series ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#477). -* **[website] docs(tenants): document namespace layout and parent/child derivation** ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#479). -* **[website] docs(tenants): document the checkbox-then-edit-CR customization pattern** ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#485). -* **[website] docs: fix 14 broken links and stale talm anchor across v1 docs** ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#486). -* **[website] fix(og): update social badge image and title** ([**@tym83**](https://github.com/tym83) in cozystack/website#487). -* **[website] docs(external-apps): rewrite guide for ApplicationDefinition API** ([**@kitsunoff**](https://github.com/kitsunoff) in cozystack/website#488). -* **[website] docs: add CLAUDE.md for AI agent guidance** ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#489). -* **[website] fix: update /docs/v1/ redirect to latest v1.2** ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#492). -* **[website] fix(ci): add OpenAPI spec download to GitHub Pages build** ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#494). -* **[website] feat(blog): add managed PostgreSQL with synchronous replication post** ([**@tym83**](https://github.com/tym83) in cozystack/website#497). -* **[website] chore(blog): add images frontmatter for social preview on existing posts** ([**@tym83**](https://github.com/tym83) in cozystack/website#498). -* **[website] feat(blog): taxonomies and client-side filter UI** ([**@tym83**](https://github.com/tym83) in cozystack/website#499). -* **[website] style(oss-health): add breathing room between navbar and hero** ([**@tym83**](https://github.com/tym83) in cozystack/website#500). - -## Other repositories - -* **[talm] feat(config): migrate to Talos v1.12 multi-document config format**: Upgrade Talos config format and modernize configuration handling ([**@lexfrei**](https://github.com/lexfrei) in cozystack/talm#116). -* **[talm] chore(deps): bump dependencies and modernize codebase** ([**@lexfrei**](https://github.com/lexfrei) in cozystack/talm#124). -* **[external-apps-example] feat: replace MongoDB example with Minecraft apps from cozylex** ([**@lexfrei**](https://github.com/lexfrei) in cozystack/external-apps-example#2). -* **[ansible-cozystack] fix(examples): add v prefix to collection version in requirements.yml** ([**@lexfrei**](https://github.com/lexfrei) in cozystack/ansible-cozystack#23). -* **[ansible-cozystack] fix(plugins): replace ansible.utils.ipaddr with stdlib-based test plugin** ([**@lexfrei**](https://github.com/lexfrei) in cozystack/ansible-cozystack#24). -* **[ansible-cozystack] feat(examples): comprehensive node prerequisites audit (fixes #19)** ([**@lexfrei**](https://github.com/lexfrei) in cozystack/ansible-cozystack#27). -* **[ansible-cozystack] chore(deps): update dependency cozystack.installer to v1.2.3** ([**@app/renovate**](https://github.com/apps/renovate) in cozystack/ansible-cozystack#29). -* **[ansible-cozystack] feat(role): expose publishing.externalIPs and tenant-root ingress via role variables** ([**@lexfrei**](https://github.com/lexfrei) in cozystack/ansible-cozystack#30). - -## Contributors - -Thanks to everyone who contributed to this patch release: - -* [**@app/github-actions**](https://github.com/apps/github-actions) -* [**@app/renovate**](https://github.com/apps/renovate) -* [**@kitsunoff**](https://github.com/kitsunoff) -* [**@kvaps**](https://github.com/kvaps) -* [**@lexfrei**](https://github.com/lexfrei) -* [**@myasnikovdaniil**](https://github.com/myasnikovdaniil) -* [**@tym83**](https://github.com/tym83) - - -**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.2.2...v1.2.3 diff --git a/docs/changelogs/v1.3.0-rc.1.md b/docs/changelogs/v1.3.0-rc.1.md deleted file mode 100644 index 9b63c106..00000000 --- a/docs/changelogs/v1.3.0-rc.1.md +++ /dev/null @@ -1,173 +0,0 @@ - - -# Cozystack v1.3.0-rc.1 - -Cozystack v1.3.0-rc.1 is the first release candidate for v1.3.0, bringing **storage-aware scheduling** via the LINSTOR scheduler extender, a managed **LINSTOR GUI** web UI with Keycloak SSO, a **VM Default Images** catalog for out-of-the-box virtual machine provisioning, **WorkloadsReady conditions** with a real-time Events tab in the dashboard, and **cross-namespace VM backup restore** capabilities. Additional highlights include stricter tenant name validation, VM network selector improvements, Keycloak theme injection and SMTP configuration, and a comprehensive host runtime preflight check. - -> **Note:** Fixes marked with *(backported to v1.2.x)* were also included in v1.2.1 or v1.2.2 patch releases. - -## Feature Highlights - -### Storage-Aware Scheduling via LINSTOR Extender - -The `cozystack-scheduler` now calls the **LINSTOR scheduler extender** for storage-locality-aware pod placement. When a pod declares both a `SchedulingClass` and LINSTOR-backed PVCs, the scheduler consults LINSTOR to prefer nodes where volume replicas already exist — reducing cross-node replication traffic and improving I/O latency for storage-heavy workloads ([**@lllamnyp**](https://github.com/lllamnyp) in #2330). - -### LINSTOR GUI: Managed Web UI for Storage Administration - -A new opt-in `linstor-gui` system package deploys **LINBIT's linstor-gui web UI** alongside the LINSTOR controller with mTLS client authentication, non-root security context, and ClusterIP-only service. An optional **Keycloak-protected Ingress** (via oauth2-proxy) can be enabled for SSO-authenticated browser access when OIDC is configured on the platform ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2382, #2390). - -### VM Default Images: Out-of-the-Box VM Provisioning - -The new `vm-default-images` package provides a curated set of **cluster-wide virtual machine images** (Ubuntu, Debian, CentOS Stream, and others) as pre-populated DataVolumes. The package is opt-in via the `iaas` bundle and defaults to replicated storage for high availability. A companion migration (migration 38) renames legacy `vm-image-*` DataVolumes to the new `vm-default-images-*` naming scheme. The `vm-disk` chart also gains a new "disk" source type for cloning from existing vm-disks in the same namespace ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2258). - -### WorkloadsReady Condition and Events Tab - -Applications now expose a **WorkloadsReady** condition on their status by querying associated WorkloadMonitor resources, giving operators a single place to check whether all underlying workloads (Deployments, StatefulSets, DaemonSets) are healthy. The dashboard gains a new **Events tab** showing namespace-scoped Kubernetes events for each application, with fallback to `.firstTimestamp` when `.eventTime` is absent. A bug where WorkloadMonitor's `Operational` status was never persisted is also fixed ([**@lexfrei**](https://github.com/lexfrei) in #2356). - -### Cross-Namespace VM Backup Restore - -The backup system now supports **restoring VMInstance backups into a different namespace** (cross-namespace copy restores), with IP/MAC preservation and safe rename semantics. In-place backup/restores for VMDisk and VMInstance are improved: HelmReleases and DataVolumes are properly handled, and Velero failure messages are propagated to the Application status. The backup status structure has been refactored to store underlying resources as a generic opaque JSON object, enabling arbitrary application-specific metadata ([**@androndo**](https://github.com/androndo) in #2251, #2329, #2319). - -## Major Features and Improvements - -* **[api] Reject tenant names with dashes at Create time**: Enforces alphanumeric-only naming for Tenants at the API level, preventing names with hyphens that would silently fail during Helm reconciliation. A corresponding regex tightening and regression test suite hardens the validation ([**@lexfrei**](https://github.com/lexfrei) in #2380). - -* **[platform] Validate computed tenant namespace length**: Rejects Tenant creation when the computed ancestor-chain namespace would exceed the 63-character Kubernetes namespace limit, preventing opaque HelmRelease reconcile errors downstream ([**@lexfrei**](https://github.com/lexfrei) in #2376). - -* **[vm-instance] Rename subnets to networks and add dropdown selector**: Renames the misleading `subnets` field to `networks` in VMInstance for clarity, adds a dropdown selector for available networks in the dashboard form, and includes a migration to copy existing `subnets` values. The old field remains supported for backward compatibility ([**@sircthulhu**](https://github.com/sircthulhu) in #2263). - -* **[keycloak] Enable injecting themes**: Cozystack administrators can now inject custom Keycloak themes via `initContainers` for UI white-labeling and customization ([**@lllamnyp**](https://github.com/lllamnyp) in #2142). - -* **[keycloak-configure] Add email verification and SMTP configuration**: Adds configurable Keycloak settings for user self-registration, email verification, and SMTP server configuration, enabling automated user onboarding flows ([**@BROngineer**](https://github.com/BROngineer) in #2318). - -* **[postgres] Hardcode PostgreSQL 17 for monitoring databases**: Pins PostgreSQL 17.7 images for system databases (Grafana, Alerta, Harbor, Keycloak, SeaweedFS) and adds migration 37 to backfill `spec.version=v17` for existing PostgreSQL resources, preventing CNPG from defaulting to PostgreSQL 18 *(backported to v1.2.1)* ([**@IvanHunters**](https://github.com/IvanHunters) in #2304). - -* **[hack] Add host runtime preflight check**: New `check-host-runtime.sh` script and `make preflight` target that warns operators when a standalone containerd or docker runtime is running alongside the embedded k3s runtime, helping diagnose container runtime conflicts ([**@lexfrei**](https://github.com/lexfrei) in #2371). - -* **[hack] Add check-readiness.sh diagnostic script**: A new diagnostic script for tracking platform reconciliation by checking readiness of Packages, ArtifactGenerators, ExternalArtifacts, and HelmReleases, with support for watch mode and continuous monitoring ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2294). - -* **[mariadb] Always enable replication for consistent service naming**: MariaDB now always enables replication, creating `-primary`/`-secondary` services even for single-replica instances. This fixes dashboard visibility and backup functionality for single-replica setups ([**@sircthulhu**](https://github.com/sircthulhu) in #2279). - -* **[platform] Prevent installed packages deletion**: Adds `helm.sh/resource-policy: keep` annotation to packages, preventing automatic deletion when packages are disabled and restoring documented behavior *(backported to v1.2.1)* ([**@kvaps**](https://github.com/kvaps) in #2273). - -## Bug Fixes - -* **[cilium] Opt-out of cri-containerd.apparmor.d for nsenter init containers**: Opts cilium-agent init containers out of the `cri-containerd.apparmor.d` AppArmor profile on non-Talos variants, fixing `Init:CrashLoopBackOff` on Ubuntu 22.04+ and Debian *(backported to v1.2.2)* ([**@lexfrei**](https://github.com/lexfrei) in #2370). - -* **[virtual-machine] Exclude external VM services from Cilium BPF LB**: Adds `service-proxy-name: cozy-proxy` label to VM LoadBalancer services, telling Cilium to skip BPF processing. Fixes inter-tenant connectivity via public LB IPs and WholeIP functionality on Cilium 1.19+ *(backported to v1.2.2)* ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #2357). - -* **[monitoring] Fix infra dashboards missing in default variant**: Includes `cozy-monitoring` namespace in the dashboard rendering condition, fixing infrastructure Grafana dashboards not rendering in the default platform variant *(backported to v1.2.2)* ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #2365). - -* **[postgres] Fix system PostgreSQL images to 17.7-standard-trixie**: Normalizes system PostgreSQL image tags to use `17.7-standard-trixie` variant with migration logic for existing CNPG clusters *(backported to v1.2.2)* ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2364). - -* **[build] Filter git describe to match only v\* tags**: Adds `--match 'v*'` to `git describe` calls, preventing API subtags from being picked up instead of release tags and producing invalid Docker image tags *(backported to v1.2.2)* ([**@kvaps**](https://github.com/kvaps) in #2386). - -* **[platform] Fix resource allocation ratios not propagated to packages**: Restores propagation of CPU, memory, and ephemeral-storage allocation ratios to managed applications and KubeVirt, which were silently ignored since the bundle restructure *(backported to v1.2.1)* ([**@sircthulhu**](https://github.com/sircthulhu) in #2296). - -* **[kubernetes] Set explicit ephemeral-storage on virt-launcher pods**: Sets explicit `domain.resources` with ephemeral-storage on VirtualMachine spec to prevent virt-launcher pods from being evicted due to LimitRange defaults being too low for actual emptyDisk capacity ([**@kvaps**](https://github.com/kvaps) in #2317). - -* **[multus] Pin master CNI to 05-cilium.conflist**: Prevents a boot-time race condition where multus could auto-detect kube-ovn's conflist instead of Cilium's *(backported to v1.2.1)* ([**@kvaps**](https://github.com/kvaps) in #2315). - -* **[multus] Build custom image with DEL cache fix**: Fixes sandbox cleanup deadlock when CNI ADD never completes, preventing stale sandbox name reservations from permanently blocking pod creation *(backported to v1.2.1)* ([**@kvaps**](https://github.com/kvaps) in #2313). - -* **[linstor] Set verify-alg to crc32c**: Prevents DRBD connection failures on kernels where `crct10dif` is unavailable (e.g., Talos v1.12.6 with kernel 6.18.18) *(backported to v1.2.1)* ([**@kvaps**](https://github.com/kvaps) in #2303). - -* **[linstor] Preserve TCP ports during toggle-disk operations**: Fixes TCP port mismatches after toggle-disk operations that could cause DRBD resources to enter StandAlone state *(backported to v1.2.1)* ([**@kvaps**](https://github.com/kvaps) in #2292). - -## Dependencies & Version Updates - -* **[linstor] Update piraeus-server to v1.33.2 with selected backports**: Bumps LINSTOR server from v1.33.1 to v1.33.2 with backported patches for stale bitmap adjust retry, LUKS2 header sizing, and optimal I/O size detection *(backported to v1.2.2)* ([**@kvaps**](https://github.com/kvaps) in #2331). - -* **[kamaji] Update to 26.3.5-edge, drop upstreamed patches**: Updates Kamaji from edge-26.2.4 to 26.3.5-edge and removes two patches accepted upstream. Adds configurable probe tuning and DataStore readiness conditions ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2260). - -* **[talm] Release v0.23.0, v0.23.1, v0.24.0** (github.com/cozystack/talm): Migrates to the Talos v1.12 multi-document machine config format ([**@lexfrei**](https://github.com/lexfrei) in cozystack/talm#116); fixes template rendering in `apply` command to resolve lookups ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/talm#119); bumps dependencies and modernizes codebase ([**@lexfrei**](https://github.com/lexfrei) in cozystack/talm#124). - -* **[ansible-cozystack] Release v1.2.1, v1.2.2** (github.com/cozystack/ansible-cozystack): Exposes `publishing.externalIPs` and tenant-root ingress via role variables ([**@lexfrei**](https://github.com/lexfrei) in cozystack/ansible-cozystack#30); adds comprehensive node prerequisites audit ([**@lexfrei**](https://github.com/lexfrei) in cozystack/ansible-cozystack#27); replaces `ansible.utils.ipaddr` with a stdlib-based test plugin ([**@lexfrei**](https://github.com/lexfrei) in cozystack/ansible-cozystack#24). - -## Security - -* **docs: add SECURITY.md**: Adds vulnerability reporting procedures, disclosure expectations, and supported release lines ([**@kvaps**](https://github.com/kvaps) in #2230). - -* **docs: add OpenSSF Best Practices badge to README**: Adds the OpenSSF Best Practices passing badge to the project README ([**@lexfrei**](https://github.com/lexfrei) in #2320). - -## Development, Testing, and CI/CD - -* **[ci] Replace cozystack-bot PAT with cozystack-ci GitHub App**: Replaces the long-lived cozystack-bot personal access token with short-lived, scoped tokens from the cozystack-ci GitHub App across all CI release workflows ([**@tym83**](https://github.com/tym83) in #2351; [**@kvaps**](https://github.com/kvaps) in #2383, #2392). - -* **[ci] Add Gemini Code Assist and CodeRabbit configuration**: Adds repository-level configuration for AI code reviewers with ignore patterns for vendored/generated code and incremental review settings ([**@lexfrei**](https://github.com/lexfrei) in #2385). - -* **[ci] Make tags workflow idempotent on re-runs**: Fixes CI to force-update API subtags and handle re-runs gracefully ([**@kvaps**](https://github.com/kvaps)). - -* **[tests] Fix Kafka E2E test timeout and retry race condition**: Increases Kafka E2E test timeout from 60s to 300s and fixes a retry race condition where `kubectl apply` could hit a still-deleting resource ([**@lexfrei**](https://github.com/lexfrei) in #2358). - -* **docs: adopt Conventional Commits for commit and PR titles**: Standardizes commit and PR title format to `type(scope): description` across all contributing docs and the PR template ([**@lexfrei**](https://github.com/lexfrei) in #2395). - -* **docs(ci): require screenshots for UI changes in PR template**: Adds a mandatory screenshots section to the PR template for UI-related changes ([**@kitsunoff**](https://github.com/kitsunoff) in #2407). - -## Documentation - -* **[website] Add ApplicationDefinition naming convention reference**: Documents how `cozystack-api` resolves kinds to their backing definitions ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#478). - -* **[website] Document Talos / talosctl / Cozystack version pairing**: Adds version compatibility matrix for installation ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#484). - -* **[website] Document namespace layout and parent/child derivation**: Explains tenant namespace hierarchy and parent/child namespace derivation rules ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#479). - -* **[website] Document the checkbox-then-edit-CR customization pattern for tenants**: Describes the workflow for customizing tenant settings via the CR after initial checkbox-based creation ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#485). - -* **[website] Add custom Keycloak themes documentation**: Covers the theme image contract, configuration, `imagePullSecrets`, and theme activation in the Keycloak admin console ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#463). - -* **[website] Add bonding (LACP) configuration how-to guide**: Covers network bonding configuration for Cozystack installations ([**@sircthulhu**](https://github.com/sircthulhu) in cozystack/website#459). - -* **[website] Improve registry mirrors for tenant Kubernetes in air-gapped guide**: Improved documentation for configuring registry mirrors in air-gapped environments ([**@sircthulhu**](https://github.com/sircthulhu) in cozystack/website#461). - -* **[website] Rewrite guide for ApplicationDefinition API (external-apps)**: Comprehensive rewrite of the external apps guide using the ApplicationDefinition API ([**@kitsunoff**](https://github.com/kitsunoff) in cozystack/website#488). - -* **[website] Add documentation for Go types usage**: Guide for using generated Go types for Cozystack managed applications as a Go module ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#465). - -* **[website] Update backup/restore documentation for VMI/VMDisk**: Updated backup documentation with VM instance and VM disk restore improvements ([**@androndo**](https://github.com/androndo) in cozystack/website#466). - -* **[website] Add OSS Health pages and OpenSSF badge**: Added OSS Health section with OpenSSF Scorecard and Best Practices badge to the website ([**@tym83**](https://github.com/tym83) in cozystack/website#470). - -* **[website] Add CozySummit Virtual 2026 program announcement**: Published the CozySummit Virtual 2026 program announcement blog post ([**@tym83**](https://github.com/tym83) in cozystack/website#472). - -* **[website] Add missing release announcements for v0.1–v0.41**: Backfilled missing release announcement blog posts for historical Cozystack versions ([**@tym83**](https://github.com/tym83) in cozystack/website#468). - -* **[website] Fix broken links and stale anchors across v1 docs**: Fixes 14 broken links and stale talm anchors ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#486). - -* **[website] Prefix bundle package names with cozystack. in v1 examples**: Corrects package naming in documentation examples ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#482). - -* **[website] Finish isolated-field removal and document opt-in policy labels**: Removes obsolete `isolated` field from tenant documentation and documents the new approach ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#481). - -* **[website] Add --take-ownership flag and describe networking.* fields**: Documents the `--take-ownership` flag and `networking.*` fields in the installation guide ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#480). - -* **[website] Fix KubeOVN MASTER_NODES example path and key in troubleshooting**: Corrects the MASTER_NODES example path ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#483). - -* **[external-apps-example] Replace MongoDB example with Minecraft apps**: Refactors the external apps example to use ApplicationDefinition API with Minecraft server applications ([**@lexfrei**](https://github.com/lexfrei) in cozystack/external-apps-example#2). - -## Governance - -* **Add Mattia Eleuteri ([@mattia-eleuteri](https://github.com/mattia-eleuteri)) as Maintainer**: CSI, Storage, Networking & Security ([**@tym83**](https://github.com/tym83) in #2345). - -* **Add Matthieu Robin ([@matthieu-robin](https://github.com/matthieu-robin)) as Maintainer**: Managed applications, platform quality, and benchmarking ([**@tym83**](https://github.com/tym83) in #2346). - -## Contributors - -We'd like to thank all contributors who made this release possible: - -* [**@androndo**](https://github.com/androndo) -* [**@BROngineer**](https://github.com/BROngineer) -* [**@IvanHunters**](https://github.com/IvanHunters) -* [**@kitsunoff**](https://github.com/kitsunoff) -* [**@kvaps**](https://github.com/kvaps) -* [**@lexfrei**](https://github.com/lexfrei) -* [**@lllamnyp**](https://github.com/lllamnyp) -* [**@mattia-eleuteri**](https://github.com/mattia-eleuteri) -* [**@myasnikovdaniil**](https://github.com/myasnikovdaniil) -* [**@sircthulhu**](https://github.com/sircthulhu) -* [**@tym83**](https://github.com/tym83) - ---- - -**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.2.0...v1.3.0-rc.1 diff --git a/docs/changelogs/v1.3.0.md b/docs/changelogs/v1.3.0.md deleted file mode 100644 index 60c800a8..00000000 --- a/docs/changelogs/v1.3.0.md +++ /dev/null @@ -1,238 +0,0 @@ - - -# Cozystack v1.3.0 - -Cozystack v1.3.0 brings **storage-aware pod scheduling** via a LINSTOR scheduler extender, a managed **LINSTOR GUI** web console with Keycloak SSO, a curated **VM Default Images** catalog for out-of-the-box virtual-machine provisioning, a new **WorkloadsReady / Events** observability surface with S3 bucket metering, and **cross-namespace VMInstance backup restore** with a full **RestoreJob dashboard** flow. The release also ships stricter tenant-name validation, VMInstance network-selector improvements, Keycloak theme injection and SMTP configuration, a host-runtime preflight check, and rolls up every fix from the v1.2.1 → v1.2.4 patch line. - -> **Note:** Items marked *(backported to v1.2.x)* were also shipped in v1.2.1, v1.2.2, v1.2.3, or v1.2.4 patch releases. - -## Feature Highlights - -### Storage-Aware Scheduling via the LINSTOR Extender - -The `cozystack-scheduler` now calls a **LINSTOR scheduler extender** for storage-locality-aware pod placement. When a pod declares both a `SchedulingClass` and LINSTOR-backed PVCs, the scheduler consults LINSTOR to prefer nodes where volume replicas already exist — reducing cross-node replication traffic and improving I/O latency for storage-heavy workloads such as databases, object stores, and VMs. - -The integration builds on the existing `SchedulingClass` tenant workload placement system introduced in v1.2.0 and requires no tenant-side configuration — workloads simply benefit once a SchedulingClass is assigned. Administrators can mix storage locality with the existing data-center / hardware-generation constraints defined on SchedulingClass CRs ([**@lllamnyp**](https://github.com/lllamnyp) in #2330). - -### LINSTOR GUI: Managed Web Console for Storage Administration - -A new opt-in `linstor-gui` system package deploys **LINBIT's linstor-gui web UI** alongside the LINSTOR controller with mTLS client authentication, non-root security context, and a ClusterIP-only service by default. When OIDC is configured on the platform, an optional **Keycloak-protected Ingress** (via oauth2-proxy) exposes the UI for browser access. Access is restricted to members of the `cozystack-cluster-admin` Keycloak group, consistent with host-cluster admin RBAC, and the gatekeeper blocks in-app LINSTOR authentication setup at the nginx proxy layer so the managed configuration cannot be subverted through the UI. - -Operators who prefer CLI access keep the existing `linstor` command; the GUI is strictly additive and stays disabled by default ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2382, #2390, #2415, #2419). - -### VM Default Images: Out-of-the-Box VM Provisioning - -The new `vm-default-images` package provides a curated set of **cluster-wide virtual-machine images** (Ubuntu, Debian, CentOS Stream, and others) as pre-populated DataVolumes, so tenants can provision VMs against well-known base images without first having to upload them. The package is opt-in via the `iaas` bundle and defaults to replicated storage for high availability. Migration 38 renames legacy `vm-image-*` DataVolumes to the new `vm-default-images-*` naming scheme, and the `vm-disk` chart gains a new "disk" source type for cloning from existing vm-disks in the same namespace ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2258). - -### Application Observability: WorkloadsReady, Events, and S3 Bucket Metering - -Applications now expose a **WorkloadsReady** condition on their status by querying associated WorkloadMonitor resources, giving operators a single place to check whether all underlying workloads (Deployments, StatefulSets, DaemonSets, PVCs) are healthy. The dashboard gains a new **Events tab** showing namespace-scoped Kubernetes events per application, with fallback to `.firstTimestamp` when `.eventTime` is absent. A long-standing bug where WorkloadMonitor's `Operational` status was never persisted is fixed in the same change ([**@lexfrei**](https://github.com/lexfrei) in #2356). - -The WorkloadMonitor reconciler is extended to track **COSI BucketClaim** objects as first-class Workloads, and the bucket controller now queries SeaweedFS logical and physical bucket-size metrics from VictoriaMetrics via a namespace-scoped monitoring endpoint, enabling S3 billing integration on par with Pods and PVCs ([**@kitsunoff**](https://github.com/kitsunoff) in #2391). Workloads are also enriched with `workloads.cozystack.io/resource-preset` and source-object labels so downstream billing pipelines can correlate monitors with the tenant preset that produced them ([**@androndo**](https://github.com/androndo) in #2416). - -### Cross-Namespace VM Backup Restore and RestoreJob Dashboard - -The backup system now supports **restoring VMInstance backups into a different namespace** (cross-namespace copy restores) with IP/MAC preservation and safe rename semantics. In-place backup and restore flows for VMDisk and VMInstance are improved: HelmReleases and DataVolumes are properly handled, and Velero failure messages are propagated to the Application status. The backup status structure has been refactored to store underlying resources as a generic opaque JSON object, enabling arbitrary application-specific metadata without status-schema churn ([**@androndo**](https://github.com/androndo) in #2251, #2319, #2329). - -The dashboard now ships a complete **RestoreJob experience**: list view, details page, create form, and sidebar entry, with a "Same as backup" fallback rendering when `spec.targetApplicationRef` is omitted. Non-CRD-backed sidebar factories (`kube-*`, `plan`, `backupjob`, `backup`, `restorejob`) are marked static so they pick up consistent managed-by labels across reconciles ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2437). - -## Major Features and Improvements - -* **[api] Reject tenant names with dashes at Create time**: Enforces alphanumeric-only naming for Tenants at the API level, preventing names with hyphens that would silently fail during Helm reconciliation. A corresponding regex tightening and regression test suite hardens the validation ([**@lexfrei**](https://github.com/lexfrei) in #2380). - -* **[platform] Validate computed tenant namespace length**: Rejects Tenant creation when the computed ancestor-chain namespace would exceed the 63-character Kubernetes namespace limit, preventing opaque HelmRelease reconcile errors downstream ([**@lexfrei**](https://github.com/lexfrei) in #2376). - -* **[vm-instance] Rename subnets to networks and add dropdown selector**: Renames the misleading `subnets` field to `networks` in VMInstance for clarity, adds a dropdown selector for available networks in the dashboard form, and includes migration 36 to copy existing `subnets` values. The old field remains supported for backward compatibility ([**@sircthulhu**](https://github.com/sircthulhu) in #2263). - -* **[keycloak] Enable injecting themes**: Cozystack administrators can now inject custom Keycloak themes via `initContainers` for UI white-labeling and customization ([**@lllamnyp**](https://github.com/lllamnyp) in #2142). - -* **[keycloak-configure] Add email verification and SMTP configuration**: Adds configurable Keycloak settings for user self-registration, email verification, and SMTP server configuration, enabling automated user onboarding flows ([**@BROngineer**](https://github.com/BROngineer) in #2318). - -* **[postgres] Pin system PostgreSQL to 17.7-standard-trixie**: Pins the PostgreSQL image for system databases (Grafana, Alerta, Harbor, Keycloak, SeaweedFS) to `17.7-standard-trixie` across chart templates and `values.yaml`, and ships migration 37 to patch existing CNPG Cluster `imageName` fields to the same variant (handling unset, any PG 17 tag, and bare-version tags). This prevents CNPG from defaulting to PostgreSQL 18 and locks system databases to the trixie variant consistent with the monitoring stack requirements *(related backports shipped in v1.2.1 via #2309 and v1.2.2 via #2364)* ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2369). - -* **[platform] Prevent installed packages deletion**: Adds the `helm.sh/resource-policy: keep` annotation to platform packages so disabling a package no longer triggers automatic Helm deletion, restoring the documented behavior where operators must explicitly delete a package *(backported to v1.2.1)* ([**@kvaps**](https://github.com/kvaps) in #2273). - -* **[mariadb] Always enable replication for consistent service naming**: MariaDB now always enables replication, creating `-primary`/`-secondary` services even for single-replica instances. This fixes dashboard visibility and backup functionality for single-replica setups ([**@sircthulhu**](https://github.com/sircthulhu) in #2279). - -* **[hack] Add host runtime preflight check**: New `check-host-runtime.sh` script and `make preflight` target that warns operators when a standalone containerd or docker runtime is running alongside the embedded k3s runtime, helping diagnose container-runtime conflicts early in an installation ([**@lexfrei**](https://github.com/lexfrei) in #2371). - -* **[hack] Add check-readiness.sh diagnostic script**: A new diagnostic script for tracking platform reconciliation by checking readiness of Packages, ArtifactGenerators, ExternalArtifacts, and HelmReleases, with support for watch mode and continuous monitoring ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2294). - -* **[platform] Add resourcePreset labels to WorkloadMonitor labels**: WorkloadMonitor labels with the `workloads.cozystack.io/` prefix are now propagated onto created Workloads; created Workloads always include the reserved `workloads.cozystack.io/monitor` label, and Helm app charts add `workloads.cozystack.io/resource-preset` metadata to WorkloadMonitor manifests, enabling downstream billing pipelines to correlate monitors with the tenant preset that produced them ([**@androndo**](https://github.com/androndo) in #2416). - -## Bug Fixes - -* **[platform] Migrate ACME HTTP-01 to ingressClassName API**: Switches ACME HTTP-01 issuance from the deprecated `acme.cert-manager.io/http01-ingress-class` annotation to the modern `ingressClassName` field on `ClusterIssuer` and solver pods. Previously, ClusterIssuers referenced a non-existent `nginx` class while each Ingress individually overrode it via annotation — producing `ingressClassName and class cannot be set at the same time` errors when tenants attempted to migrate to the modern field. The migration is atomic: both the ClusterIssuer and consuming Ingresses are updated together *(backported to v1.2.4)* ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2436). - -* **[harbor] Remove incorrect tenant module flags**: Harbor is a PaaS service, not a tenant module. Incorrect `spec.dashboard.module: true` and `internal.cozystack.io/tenantmodule` flags caused Harbor to appear in the sidebar "Modules" section and be misclassified by controllers handling tenant modules. The flags are now removed so Harbor is displayed in its proper PaaS category and is no longer treated as a tenant-scoped HelmRelease ([**@kvaps**](https://github.com/kvaps) in #2444). - -* **[kube-ovn] Resolve kubeovn-plunger RBAC forbidden on deployments**: Grants `kube-ovn-plunger` the RBAC needed to list Deployments so it can reconcile `ovn-central`, fixing `deployments.apps is forbidden` errors in `cozy-kubeovn` ([**@kvaps**](https://github.com/kvaps) in #2441). - -* **[cilium] Opt-out of cri-containerd.apparmor.d for nsenter init containers**: Opts cilium-agent init containers out of the `cri-containerd.apparmor.d` AppArmor profile on non-Talos variants (`cilium-generic`, `kubeovn-cilium-generic`), fixing `Init:CrashLoopBackOff` on Ubuntu 22.04+ and Debian where the profile denies `nsenter` namespace entry. Talos variants are untouched as Talos does not load the AppArmor LSM *(backported to v1.2.2)* ([**@lexfrei**](https://github.com/lexfrei) in #2370). - -* **[virtual-machine] Exclude external VM services from Cilium BPF LB**: Adds the `service.kubernetes.io/service-proxy-name: cozy-proxy` label to VM LoadBalancer services with `external: true`, telling Cilium to skip BPF processing entirely. Fixes inter-tenant connectivity via public LB IPs (Cilium's DNAT caused cross-tenant pod-to-pod flow classification, triggering CiliumClusterwideNetworkPolicy blocks) and restores WholeIP behavior on Cilium 1.19+ where wildcard service drop entries previously blocked traffic to LB IPs on undeclared ports *(backported to v1.2.2)* ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #2357). - -* **[monitoring] Fix infra dashboards missing in default variant**: Includes the `cozy-monitoring` namespace in the dashboard rendering condition, fixing infrastructure Grafana dashboards not rendering in the default platform variant (only the `tenant-root` namespace was previously checked) *(backported to v1.2.2)* ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #2365). - -* **[build] Filter git describe to match only v* tags**: Adds `--match 'v*'` to all `git describe` calls in `hack/common-envs.mk`, preventing the `api/apps/v1alpha1/vX.Y.Z` subtag from being picked up instead of the release tag and producing invalid Docker image tags *(backported to v1.2.2)* ([**@kvaps**](https://github.com/kvaps) in #2386). - -* **[platform] Fix resource allocation ratios not propagated to packages**: Restores propagation of `cpuAllocationRatio`, `memoryAllocationRatio`, and `ephemeralStorageAllocationRatio` from `platform/values.yaml` to the `cozystack-values` Secret that managed applications and KubeVirt read, fixing a regression introduced in the bundle restructure that silently ignored operator-configured ratios *(backported to v1.2.1)* ([**@sircthulhu**](https://github.com/sircthulhu) in #2296). - -* **[kubernetes] Set explicit ephemeral-storage on virt-launcher pods**: Sets explicit `domain.resources` ephemeral-storage on the VirtualMachine spec to prevent virt-launcher pods from being evicted because LimitRange defaults were too small for the actual emptyDisk capacity *(backported to v1.2.3)* ([**@kvaps**](https://github.com/kvaps) in #2317). - -* **[multus] Pin master CNI to 05-cilium.conflist**: Prevents a boot-time race where multus could auto-detect kube-ovn's conflist instead of Cilium's, which would cause pods to bypass the Cilium chain entirely and lose their endpoint *(backported to v1.2.1)* ([**@kvaps**](https://github.com/kvaps) in #2315). - -* **[multus] Build custom image with DEL cache fix**: Fixes sandbox cleanup deadlock when CNI ADD never completes, preventing stale sandbox name reservations from permanently blocking pod creation after a node disruption *(backported to v1.2.1)* ([**@kvaps**](https://github.com/kvaps) in #2313). - -* **[linstor] Set verify-alg to crc32c**: Prevents DRBD connection failures on kernels where `crct10dif` is unavailable (e.g., Talos v1.12.6 with kernel 6.18.18) by setting the LINSTOR verify-alg controller default to `crc32c` *(backported to v1.2.1)* ([**@kvaps**](https://github.com/kvaps) in #2303). - -* **[linstor] Preserve TCP ports during toggle-disk operations**: Saves existing TCP ports into the `LayerPayload` before `removeLayerData()` deletes them, preventing DRBD resources from entering StandAlone state when a satellite misses the resulting update *(backported to v1.2.1)* ([**@kvaps**](https://github.com/kvaps) in #2292). - -* **[linstor] Increase satellite startup probe failure threshold**: Raises the LINSTOR satellite `startupProbe` `failureThreshold` from 3 to 30 (30s → 300s) in the `LinstorSatelliteConfiguration` pod template, giving satellites with slow storage initialization enough time to come up without being killed and restarted ([**@Arsolitt**](https://github.com/Arsolitt) in #2425). - -## Security - -* **docs: add SECURITY.md**: Adds vulnerability reporting procedures, disclosure expectations, and supported release lines ([**@kvaps**](https://github.com/kvaps) in #2230). - -* **docs: add OpenSSF Best Practices badge to README**: Adds the OpenSSF Best Practices passing badge to the project README ([**@lexfrei**](https://github.com/lexfrei) in #2320). - -## Dependencies & Version Updates - -* **[kube-ovn] Bump kube-ovn to v1.15.10 with port-group regression fix**: Updates `packages/system/kubeovn` to upstream v1.15.10 (from v1.15.3) and carries a patch for `pkg/controller/pod.go` that preserves a VM LSP's port-group memberships when Kubernetes GCs a completed virt-launcher pod while another virt-launcher pod of the same VM is still running. Without the patch, the destination pod of a successful live migration lost its security groups, network policies, and node-scoped routing until `kube-ovn-controller` was restarted ([**@kvaps**](https://github.com/kvaps) in #2443). - -* **[monitoring] Upgrade victoria-metrics-operator to v0.68.4**: Bumps the vendored `victoria-metrics-operator` Helm chart from 0.59.1 to 0.61.0 (operator appVersion v0.68.1 → v0.68.4), picking up upstream fixes for `VMPodScrape` port routing on VMAgent/VLAgent and `StatefulSet` pod deletion (not eviction) when `maxUnavailable=100%` ([**@lexfrei**](https://github.com/lexfrei) in #2426). - -* **[linstor] Update piraeus-server to v1.33.2 with selected backports**: Bumps LINSTOR server from v1.33.1 to v1.33.2 with backported patches for stale bitmap adjust retry, LUKS2 header sizing, optimal I/O size detection, and the maintainer implementation. All patches verified against upstream v1.33.2 with `git apply --check` and `gradlew compileJava` *(backported to v1.2.2)* ([**@kvaps**](https://github.com/kvaps) in #2331). - -* **[kamaji] Update to 26.3.5-edge, drop upstreamed patches**: Updates Kamaji from edge-26.2.4 to 26.3.5-edge and removes two patches accepted upstream. Adds configurable probe tuning and DataStore readiness conditions ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2260). - -* **[talm] Release v0.23.0, v0.23.1, v0.24.0** (github.com/cozystack/talm): Migrates to the Talos v1.12 multi-document machine config format ([**@lexfrei**](https://github.com/lexfrei) in cozystack/talm#116); renders templates online in `apply` to resolve lookups ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/talm#119); bumps dependencies and modernizes the codebase ([**@lexfrei**](https://github.com/lexfrei) in cozystack/talm#124). - -* **[ansible-cozystack] Release v1.2.1, v1.2.2, v1.2.4** (github.com/cozystack/ansible-cozystack): Exposes `publishing.externalIPs` and tenant-root ingress via role variables ([**@lexfrei**](https://github.com/lexfrei) in cozystack/ansible-cozystack#30); adds a comprehensive node prerequisites audit ([**@lexfrei**](https://github.com/lexfrei) in cozystack/ansible-cozystack#27); replaces `ansible.utils.ipaddr` with a stdlib-based test plugin ([**@lexfrei**](https://github.com/lexfrei) in cozystack/ansible-cozystack#24); adds `v` prefix to collection version in requirements.yml examples ([**@lexfrei**](https://github.com/lexfrei) in cozystack/ansible-cozystack#23); tracks installer releases v1.2.1 through v1.2.4 ([**@app/renovate**](https://github.com/apps/renovate) in cozystack/ansible-cozystack#20, #22, #29, #31, #32). - -## Development, Testing, and CI/CD - -* **[ci] Replace cozystack-bot PAT with cozystack-ci GitHub App**: Replaces the long-lived `cozystack-bot` personal access token with short-lived, scoped tokens from the `cozystack-ci` GitHub App across all release workflows (`tags.yaml`, `auto-release.yaml`, `pull-requests-release.yaml`), improving security and auditability of CI operations ([**@tym83**](https://github.com/tym83) in #2351; [**@kvaps**](https://github.com/kvaps) in #2383, #2392). - -* **[ci] Add Gemini Code Assist and CodeRabbit configuration**: Adds repository-level configuration for AI code reviewers with ignore patterns for vendored/generated code and incremental review settings ([**@lexfrei**](https://github.com/lexfrei) in #2385). - -* **[ci] Promote next/ trunk on new minor/major releases**: Updates `update-website-docs` in `tags.yaml` to match the new docs-versioning contract — the website repo replaces the old "pre-create `vX.Y/` draft directory" scheme with a permanent `content/en/docs/next/` trunk, and released version directories are promoted explicitly by the release workflow ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2433). - -* **[tests] Fix Kafka E2E test timeout and retry race condition**: Increases Kafka E2E test timeout from 60s to 300s and fixes a retry race where `kubectl apply` could hit a still-deleting resource ([**@lexfrei**](https://github.com/lexfrei) in #2358). - -* **docs: adopt Conventional Commits for commit and PR titles**: Standardizes commit and PR title format to `type(scope): description` across all contributing docs and the PR template ([**@lexfrei**](https://github.com/lexfrei) in #2395). - -* **docs(ci): require screenshots for UI changes in PR template**: Adds a mandatory screenshots section to the PR template for UI-related changes ([**@kitsunoff**](https://github.com/kitsunoff) in #2407). - -* **chore(maintenance): add @myasnikovdaniil to CODEOWNERS**: Adds @myasnikovdaniil to the default owners in `.github/CODEOWNERS` for automatic review requests ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2434). - -## Documentation - -* **[website] Add ApplicationDefinition naming convention reference**: Documents how `cozystack-api` resolves kinds to their backing definitions ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#478). - -* **[website] Document Talos / talosctl / Cozystack version pairing**: Adds a version compatibility matrix for installation ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#484). - -* **[website] Document namespace layout and parent/child derivation**: Explains tenant namespace hierarchy and parent/child namespace derivation rules ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#479). - -* **[website] Document the checkbox-then-edit-CR customization pattern for tenants**: Describes the workflow for customizing tenant settings via the CR after initial checkbox-based creation ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#485). - -* **[website] Add custom Keycloak themes documentation**: Covers the theme image contract, configuration, `imagePullSecrets`, and theme activation in the Keycloak admin console ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#463). - -* **[website] Add bonding (LACP) configuration how-to guide**: Covers network bonding configuration for Cozystack installations ([**@sircthulhu**](https://github.com/sircthulhu) in cozystack/website#459). - -* **[website] Improve registry mirrors for tenant Kubernetes in air-gapped guide**: Improves documentation for configuring registry mirrors in air-gapped environments ([**@sircthulhu**](https://github.com/sircthulhu) in cozystack/website#461). - -* **[website] Rewrite guide for ApplicationDefinition API (external-apps)**: Comprehensive rewrite of the external apps guide using the ApplicationDefinition API with Minecraft server examples ([**@kitsunoff**](https://github.com/kitsunoff) in cozystack/website#488). - -* **[website] Add documentation for Go types usage**: Guide for using generated Go types for Cozystack managed applications as a Go module ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#465). - -* **[website] Update backup/restore documentation for VMI/VMDisk**: Updates backup documentation with VM instance and VM disk restore improvements ([**@androndo**](https://github.com/androndo) in cozystack/website#466). - -* **[website] Refactor docs versions to major.minor variants**: Moves docs to major.minor versioning for the v1.x series ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#477). - -* **[website] Trunk-based versioning with permanent next/ directory**: Replaces the old "pre-create `vX.Y/` draft directory" scheme with a permanent `content/en/docs/next/` trunk; released version directories are promoted explicitly by `hack/release_next.sh` on new minor/major releases, and routing between `next/` and `vX.Y/` is Makefile-driven ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#495). - -* **[website] Add updated OpenAPI spec**: Updates the OpenAPI specification for managed applications reference ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#469). - -* **[website] Add OpenAPI spec download to GitHub Pages build**: Fixes the GitHub Pages build to include the OpenAPI spec download ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#494). - -* **[website] Add OSS Health pages and OpenSSF badge**: Adds OSS Health section with OpenSSF Scorecard and Best Practices badges to the website ([**@tym83**](https://github.com/tym83) in cozystack/website#470). - -* **[website] Add Telemetry page under OSS Health section**: Adds the Telemetry page with initial data seeding to the OSS Health docs ([**@tym83**](https://github.com/tym83) in cozystack/website#471, cozystack/website#504). - -* **[website] Blog: OSS Health section launch announcement**: Publishes the announcement blog post for the OSS Health section ([**@tym83**](https://github.com/tym83) in cozystack/website#474). - -* **[website] Fix OpenSSF canonical status URL**: Changes the OpenSSF canonical status URL from pt-BR to en ([**@tym83**](https://github.com/tym83) in cozystack/website#475). - -* **[website] Add CozySummit Virtual 2026 program announcement**: Publishes the CozySummit Virtual 2026 program announcement blog post ([**@tym83**](https://github.com/tym83) in cozystack/website#472). - -* **[website] Add missing release announcements for v0.1–v0.41**: Backfills missing release announcement blog posts for historical Cozystack versions ([**@tym83**](https://github.com/tym83) in cozystack/website#468). - -* **[website] Blog: managed PostgreSQL with synchronous replication**: Adds a post covering the managed PostgreSQL synchronous-replication feature ([**@tym83**](https://github.com/tym83) in cozystack/website#497). - -* **[website] Blog taxonomies and client-side filter UI**: Registers article-type and topic taxonomies and adds a client-side filter on the blog list page ([**@tym83**](https://github.com/tym83) in cozystack/website#499). - -* **[website] Add images frontmatter for social preview on existing posts**: Adds images frontmatter for social preview on existing blog posts ([**@tym83**](https://github.com/tym83) in cozystack/website#498). - -* **[website] Fix broken links and stale anchors across v1 docs**: Fixes 14 broken links and stale talm anchors ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#486). - -* **[website] Prefix bundle package names with cozystack. in v1 examples**: Corrects package naming in documentation examples ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#482). - -* **[website] Finish isolated-field removal and document opt-in policy labels**: Removes the obsolete `isolated` field from tenant documentation and documents the new opt-in policy labels approach ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#481). - -* **[website] Add --take-ownership flag and describe networking.* fields**: Documents the `--take-ownership` flag and `networking.*` fields in the installation guide ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#480). - -* **[website] Fix KubeOVN MASTER_NODES example path and key in troubleshooting**: Corrects the MASTER_NODES example path and key ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#483). - -* **[website] Add CLAUDE.md for AI agent guidance**: Adds a CLAUDE.md file describing the trunk-based docs architecture for AI agent guidance ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#489). - -* **[website] Update /docs/v1/ redirect to latest v1.2**: Updates the `/docs/v1/` redirect target to point to the latest v1.2 docs on GitHub Pages ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#492). - -* **[website] Remove nbykov from CODEOWNERS and CLAUDE.md**: Cleans up CODEOWNERS and CLAUDE.md entries ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#491). - -* **[website] Add Ahrefs Analytics tracker**: Adds the Ahrefs Analytics tracker to the website ([**@tym83**](https://github.com/tym83) in cozystack/website#503). - -* **[website] Add breathing room between navbar and hero on OSS Health**: Minor styling fix for the OSS Health section ([**@tym83**](https://github.com/tym83) in cozystack/website#500). - -* **[website] Fix og social badge image and title**: Updates the social badge image and title ([**@tym83**](https://github.com/tym83) in cozystack/website#487). - -* **[website] Update managed apps reference for v1.2.1**: Automated managed-apps reference update ([**@cozystack-bot**](https://github.com/cozystack-bot) in cozystack/website#464). - -* **[external-apps-example] Replace MongoDB example with Minecraft apps**: Refactors the external apps example to use the ApplicationDefinition API with Minecraft server applications ([**@lexfrei**](https://github.com/lexfrei) in cozystack/external-apps-example#2). - -* **docs: update README introductory description**: Refines the platform positioning and improves clarity on core capabilities in the main README ([**@tym83**](https://github.com/tym83) in #2409). - -## Governance - -* **Add Mattia Eleuteri ([@mattia-eleuteri](https://github.com/mattia-eleuteri)) as Maintainer**: CSI, Storage, Networking & Security ([**@tym83**](https://github.com/tym83) in #2345). - -* **Add Matthieu Robin ([@matthieu-robin](https://github.com/matthieu-robin)) as Maintainer**: Managed applications, platform quality, and benchmarking ([**@tym83**](https://github.com/tym83) in #2346). - -## Contributors - -We'd like to thank all contributors who made this release possible: - -* [**@androndo**](https://github.com/androndo) -* [**@Arsolitt**](https://github.com/Arsolitt) -* [**@BROngineer**](https://github.com/BROngineer) -* [**@IvanHunters**](https://github.com/IvanHunters) -* [**@kitsunoff**](https://github.com/kitsunoff) -* [**@kvaps**](https://github.com/kvaps) -* [**@lexfrei**](https://github.com/lexfrei) -* [**@lllamnyp**](https://github.com/lllamnyp) -* [**@mattia-eleuteri**](https://github.com/mattia-eleuteri) -* [**@myasnikovdaniil**](https://github.com/myasnikovdaniil) -* [**@sircthulhu**](https://github.com/sircthulhu) -* [**@tym83**](https://github.com/tym83) - -### New Contributors - -We're excited to welcome our first-time contributors: - -* [**@Arsolitt**](https://github.com/Arsolitt) — First contribution! - ---- - -**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.2.0...v1.3.0 diff --git a/docs/changelogs/v1.3.1.md b/docs/changelogs/v1.3.1.md deleted file mode 100644 index 0f149d0d..00000000 --- a/docs/changelogs/v1.3.1.md +++ /dev/null @@ -1,37 +0,0 @@ - - -# v1.3.1 (2026-04-28) - -Patch release covering a TenantNamespace IDOR fix in the API, a destructive `post-upgrade` hook removed from the etcd chart, kamaji controller stability, a `linstor-csi` bump that fixes live migration on Protocol-A/B DRBD resources, the missing `linstor-gui` build wiring, and a velero RBAC fix that unblocked installs on bundles without Velero. - -## Security - -* **fix(api): prevent IDOR in TenantNamespace Get and Watch handlers**: Two IDOR (Insecure Direct Object Reference) vulnerabilities allowed authenticated users to read TenantNamespace metadata they had no RoleBinding for. The `Get` and `Watch` handlers now go through a new `hasAccessToNamespace()` helper that lists RoleBindings scoped only to the target namespace (orders of magnitude faster than the previous all-cluster scan), returns `NotFound` instead of leaking existence on unauthorized access, and applies the same check on the `Watch` filter path. Includes regression tests for the unauthorized paths. ([**@IvanHunters**](https://github.com/IvanHunters) in #2471, backport #2524) - -## Features - -* **feat(linstor): bump linstor-csi to v1.10.6 with Protocol-C dual-attach fix**: Live migration of KubeVirt VMs on Protocol-A/B (async) DRBD volumes no longer fails with `Protocol C required`. `linstor-csi` v1.10.6 now installs a `Protocol=C` override on the resource-definition during dual-attach and reverts it on detach, so `replicated-async` StorageClasses and other Protocol-A/B resource groups support live migration without manual `drbdadm adjust` intervention. ([**@kvaps**](https://github.com/kvaps) in #2496, backport #2505) - -## Fixes - -* **fix(backups): move velero-configmap Role to velero chart**: The `backupstrategy-controller` (a default package) declared a Role/RoleBinding scoped to the `cozy-velero` namespace for managing `ResourceModifier` ConfigMaps. On bundles where Velero was not enabled, that namespace did not exist and the HelmRelease failed with `namespaces "cozy-velero" not found`, blocking installation. The Role/RoleBinding now lives in the velero chart, so it is created only when velero is actually deployed. ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2459, backport #2467) - -* **fix(etcd): remove destructive post-upgrade cert-regeneration hook**: The etcd chart ran a `post-upgrade` Helm hook on every upgrade that deleted etcd TLS Secrets (`etcd-ca-tls`, `etcd-peer-ca-tls`, `etcd-client-tls`, `etcd-peer-tls`, `etcd-server-tls`) and then deleted etcd pods, forcing cert-manager to re-issue the entire etcd CA chain. On clusters with Kamaji-managed tenant control planes this put every tenant `kube-apiserver` into CrashLoopBackOff until each DataStore was manually re-reconciled. The hook was a one-shot `2.6.0 → 2.6.1` migration that became a permanent footgun once chart versioning moved to `0.0.0+` (always `< 2.6.1` per semver) and after the underlying `rotationPolicy: Always` issue was fixed in `47d81f70`. The hook is now removed entirely. ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2462, backport #2511) - -* **fix(kamaji): increase memory limits and add startup probe**: The kamaji controller frequently entered CrashLoopBackOff due to OOMKills (exit 137) within ~20–25 seconds of startup, with the readiness probe failing while the controller was still finishing initialization. Memory limit raised from 500Mi to 512Mi, request from 100Mi to 256Mi, and a 60-second startup probe (12 attempts × 5s periods) is added so the controller has room to boot before liveness/readiness probes engage. ([**@IvanHunters**](https://github.com/IvanHunters) in #2421, backport #2491) - -## Build - -* **build(linstor): include linstor-gui in root image build target**: The `linstor-gui` package (added in #2382) was never wired into the root `Makefile`'s `build:` target, so CI never built or published the image. `ghcr.io/cozystack/cozystack/linstor-gui` returned `NAME_UNKNOWN` and `values.yaml` stayed pinned to `tag: 2.3.0` without a digest. The missing build line is added so the next CI run publishes the image and the per-package `Makefile` digest-pins `values.yaml` automatically. ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2498, backport #2518) - -## Contributors - -Thanks to everyone who contributed to this patch release: - -* [**@IvanHunters**](https://github.com/IvanHunters) -* [**@kvaps**](https://github.com/kvaps) -* [**@myasnikovdaniil**](https://github.com/myasnikovdaniil) - -**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.3.0...v1.3.1 diff --git a/examples/backups/vmi/00-helpers.sh b/examples/backups/vmi/00-helpers.sh deleted file mode 100755 index 38405495..00000000 --- a/examples/backups/vmi/00-helpers.sh +++ /dev/null @@ -1,112 +0,0 @@ -#!/bin/bash -# Helper functions and variables for the VMInstance backup/restore demo -# Source this file in other scripts: source "$(dirname "$0")/00-helpers.sh" - -# ANSI color codes -export RED='\033[0;31m' -export GREEN='\033[0;32m' -export YELLOW='\033[1;33m' -export BLUE='\033[0;34m' -export MAGENTA='\033[0;35m' -export CYAN='\033[0;36m' -export WHITE='\033[1;37m' -export NC='\033[0m' # No Color -export BOLD='\033[1m' - -# Default settings -export NAMESPACE="${NAMESPACE:-tenant-root}" -export BACKUP_STORAGE_LOCATION="${BACKUP_STORAGE_LOCATION:-default}" - -# Logging functions (output to stderr to avoid polluting captured output) -log_info() { - echo -e "${BLUE}ℹ${NC} $*" >&2 -} - -log_success() { - echo -e "${GREEN}✔${NC} $*" >&2 -} - -log_warning() { - echo -e "${YELLOW}⚠${NC} $*" >&2 -} - -log_error() { - echo -e "${RED}✖${NC} $*" >&2 -} - -log_step() { - echo -e "\n${MAGENTA}${BOLD}▶ $*${NC}" >&2 -} - -log_substep() { - echo -e "${CYAN} → $*${NC}" >&2 -} - -log_command() { - echo -e "${WHITE} \$ $*${NC}" >&2 -} - -# Wait for user to press Enter -wait_for_enter() { - echo -e "\n${CYAN}Press Enter to continue...${NC}" >&2 - read -r -} - -# Check if a Kubernetes resource exists -resource_exists() { - local resource_type="$1" - local resource_name="$2" - local namespace="${3:-}" - - if [[ -n "$namespace" ]]; then - kubectl get "$resource_type" "$resource_name" -n "$namespace" &>/dev/null - else - kubectl get "$resource_type" "$resource_name" &>/dev/null - fi -} - -# Wait for a resource field to reach a desired value -wait_for_field() { - local resource_type="$1" - local resource_name="$2" - local jsonpath="$3" - local desired="$4" - local namespace="${5:-}" - local timeout="${6:-300}" - - log_substep "Waiting for $resource_type/$resource_name $jsonpath to become '$desired'..." - - local elapsed=0 - local ns_flag="" - [[ -n "$namespace" ]] && ns_flag="-n $namespace" - - while true; do - local current - # shellcheck disable=SC2086 - current=$(kubectl get "$resource_type" "$resource_name" $ns_flag -o jsonpath="$jsonpath" 2>/dev/null || true) - if [[ "$current" == "$desired" ]]; then - log_success "$resource_type/$resource_name reached '$desired'" - return 0 - fi - if [[ $elapsed -ge $timeout ]]; then - log_error "Timeout waiting for $resource_type/$resource_name (current: '$current', expected: '$desired')" - return 1 - fi - sleep 5 - elapsed=$((elapsed + 5)) - echo -n "." >&2 - done -} - -# Print a separator line -separator() { - echo -e "\n${CYAN}────────────────────────────────────────────────────────────${NC}\n" >&2 -} - -# Print script header -print_header() { - local title="$1" - echo -e "\n${MAGENTA}${BOLD}╔════════════════════════════════════════════════════════════╗${NC}" >&2 - echo -e "${MAGENTA}${BOLD}║${NC} ${WHITE}${BOLD}$title${NC}" >&2 - echo -e "${MAGENTA}${BOLD}╚════════════════════════════════════════════════════════════╝${NC}\n" >&2 -} diff --git a/examples/backups/vmi/01-create-strategies.sh b/examples/backups/vmi/01-create-strategies.sh deleted file mode 100755 index 78aa34d3..00000000 --- a/examples/backups/vmi/01-create-strategies.sh +++ /dev/null @@ -1,145 +0,0 @@ -#!/bin/bash -# Step 01: Create Velero backup/restore strategies for VMInstance and VMDisk -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/00-helpers.sh" - -print_header "Step 1: Create Velero Strategies" - -log_step "Creating VMInstance strategy..." -log_command "kubectl apply -f - (Velero strategy: vminstance-strategy)" - -kubectl apply -f - <<'EOF' -apiVersion: strategy.backups.cozystack.io/v1alpha1 -kind: Velero -metadata: - name: vminstance-strategy -spec: - template: - # Symmetric restore filters: kubevirt-velero-plugin requires launcher pods in the backup, - # but restore orLabelSelectors limit what is applied from the tarball (e.g. skip another - # VM's pod that Velero added via PVC item action). VMDisk OR branches are appended by - # the controller from backup.status.underlyingResources or the Velero backup annotation. - restoreSpec: - existingResourcePolicy: update - includedNamespaces: - - '{{ .Application.metadata.namespace }}' - orLabelSelectors: - - matchLabels: - app.kubernetes.io/instance: 'vm-instance-{{ .Application.metadata.name }}' - - matchLabels: - apps.cozystack.io/application.kind: '{{ .Application.kind }}' - apps.cozystack.io/application.name: '{{ .Application.metadata.name }}' - includedResources: - - helmreleases.helm.toolkit.fluxcd.io - - virtualmachines.kubevirt.io - - virtualmachineinstances.kubevirt.io - - pods - - persistentvolumeclaims - - configmaps - - secrets - - controllerrevisions.apps - includeClusterResources: false - excludedResources: - # Required to avoid conflict with restored DV from HR VMDisk - - datavolumes.cdi.kubevirt.io - - spec: # see https://velero.io/docs/v1.17/api-types/backup/ - includedNamespaces: - - '{{ .Application.metadata.namespace }}' - orLabelSelectors: - # VM resources (VirtualMachine, DataVolume, PVC, etc.) - - matchLabels: - app.kubernetes.io/instance: 'vm-instance-{{ .Application.metadata.name }}' - # HelmRelease (the Cozystack app object) - - matchLabels: - apps.cozystack.io/application.kind: '{{ .Application.kind }}' - apps.cozystack.io/application.name: '{{ .Application.metadata.name }}' - includedResources: - - helmreleases.helm.toolkit.fluxcd.io - - virtualmachines.kubevirt.io - - virtualmachineinstances.kubevirt.io - # Required by kubevirt-velero-plugin for running VMs ("launcher pod must be in backup"). - - pods - # Required by kubevirt-velero-plugin requires DV to be in backup of VM, but it excludes in restores - - datavolumes.cdi.kubevirt.io - - persistentvolumeclaims - - configmaps - - secrets - - controllerrevisions.apps - includeClusterResources: false - storageLocation: '{{ .Parameters.backupStorageLocationName }}' - volumeSnapshotLocations: - - '{{ .Parameters.backupStorageLocationName }}' - snapshotVolumes: true - snapshotMoveData: true - ttl: 720h0m0s - itemOperationTimeout: 24h0m0s -EOF - -log_success "VMInstance strategy created" - -separator - -log_step "Creating VMDisk strategy..." -log_command "kubectl apply -f - (Velero strategy: vmdisk-strategy)" - -kubectl apply -f - <<'EOF' -apiVersion: strategy.backups.cozystack.io/v1alpha1 -kind: Velero -metadata: - name: vmdisk-strategy -spec: - template: - restoreSpec: - existingResourcePolicy: update - includedNamespaces: - - '{{ .Application.metadata.namespace }}' - orLabelSelectors: - - matchLabels: - app.kubernetes.io/instance: 'vm-disk-{{ .Application.metadata.name }}' - - matchLabels: - apps.cozystack.io/application.kind: '{{ .Application.kind }}' - apps.cozystack.io/application.name: '{{ .Application.metadata.name }}' - includedResources: - - helmreleases.helm.toolkit.fluxcd.io - - persistentvolumeclaims - - configmaps - includeClusterResources: false - - spec: - includedNamespaces: - - '{{ .Application.metadata.namespace }}' - orLabelSelectors: - - matchLabels: - app.kubernetes.io/instance: 'vm-disk-{{ .Application.metadata.name }}' - - matchLabels: - apps.cozystack.io/application.kind: '{{ .Application.kind }}' - apps.cozystack.io/application.name: '{{ .Application.metadata.name }}' - includedResources: - - helmreleases.helm.toolkit.fluxcd.io - - persistentvolumeclaims - - configmaps - includeClusterResources: false - storageLocation: '{{ .Parameters.backupStorageLocationName }}' - volumeSnapshotLocations: - - '{{ .Parameters.backupStorageLocationName }}' - snapshotVolumes: true - snapshotMoveData: true - ttl: 720h0m0s - itemOperationTimeout: 24h0m0s -EOF - -log_success "VMDisk strategy created" - -separator - -log_step "Verifying strategies..." -log_command "kubectl get velero.strategy.backups.cozystack.io" -kubectl get velero.strategy.backups.cozystack.io - -separator - -log_success "Velero strategies are ready" -echo -e "\n${GREEN}${BOLD}Next step:${NC} ./02-create-backupclass.sh" diff --git a/examples/backups/vmi/02-create-backupclass.sh b/examples/backups/vmi/02-create-backupclass.sh deleted file mode 100755 index 08c29fc1..00000000 --- a/examples/backups/vmi/02-create-backupclass.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash -# Step 02: Create BackupClass that binds strategies to application types -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/00-helpers.sh" - -print_header "Step 2: Create BackupClass" - -log_step "Creating BackupClass 'velero'..." -log_info "BackupClass maps application kinds (VMInstance, VMDisk) to their Velero strategies" -log_command "kubectl apply -f - (BackupClass: velero)" - -kubectl apply -f - < - external: false - externalMethod: PortList - externalPorts: - - 22 -EOF - -log_success "VMInstance created" - -separator - -log_step "Verifying VMInstance..." -log_command "kubectl get vminstance test -n $NAMESPACE" -kubectl get vminstance test -n "$NAMESPACE" - -separator - -log_success "VMInstance is ready" -echo -e "\n${GREEN}${BOLD}Next step:${NC} ./05-create-backupjob.sh" diff --git a/examples/backups/vmi/05-create-backupjob.sh b/examples/backups/vmi/05-create-backupjob.sh deleted file mode 100755 index 581490a4..00000000 --- a/examples/backups/vmi/05-create-backupjob.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash -# Step 05: Create a BackupJob to back up the VMInstance -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/00-helpers.sh" - -print_header "Step 5: Create BackupJob" - -log_step "Creating BackupJob 'test-backup' in namespace $NAMESPACE..." -log_info "This triggers a Velero backup of VMInstance 'test' and all its disks" -log_command "kubectl apply -f - (BackupJob: test-backup)" - -kubectl apply -f - <-orig-`, only for in-place restore - keepOriginalIpAndMac: true # restores original IP and MAC address of VMI via OVN annotations -EOF - -log_success "RestoreJob created" - -separator - -log_step "Waiting for RestoreJob to complete..." -wait_for_field restorejob restore-in-place-test '{.status.phase}' Succeeded "$NAMESPACE" 600 - -separator - -log_step "Verifying RestoreJob result..." -log_command "kubectl get restorejob restore-in-place-test -n $NAMESPACE -o yaml" -kubectl get restorejob restore-in-place-test -n "$NAMESPACE" -o wide - -separator - -log_success "In-place restore completed successfully" -echo -e "\n${GREEN}${BOLD}Next step:${NC} ./07-restore-to-copy.sh" diff --git a/examples/backups/vmi/07-restore-to-copy.sh b/examples/backups/vmi/07-restore-to-copy.sh deleted file mode 100755 index 9b84ce55..00000000 --- a/examples/backups/vmi/07-restore-to-copy.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/bin/bash -# Step 07: Restore the VMInstance to a copy in a different namespace -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/00-helpers.sh" - -TARGET_NAMESPACE="${TARGET_NAMESPACE:-tenant-root-copy}" - -print_header "Step 7: Restore VMInstance to Copy (Cross-Namespace)" - -log_info "Restoring to the same namespace with a different app name is not supported" -log_info "due to Velero DataUpload limitations. Cross-namespace restore uses Velero's namespaceMapping." - -separator - -log_step "Ensuring target namespace '$TARGET_NAMESPACE' exists..." -kubectl create namespace "$TARGET_NAMESPACE" --dry-run=client -o yaml | kubectl apply -f - -log_success "Namespace '$TARGET_NAMESPACE' is ready" - -separator - -log_step "Creating RestoreJob 'restore-to-copy-test' in namespace $NAMESPACE..." -log_info "The backup will be restored into namespace '$TARGET_NAMESPACE' using Velero namespaceMapping" -log_command "kubectl apply -f - (RestoreJob: restore-to-copy-test)" - -kubectl apply -f - <-orig-`, only for in-place restore - keepOriginalIpAndMac: false # restores original IP and MAC address of VMI via OVN annotations -EOF - -log_success "RestoreJob created" - -separator - -log_step "Waiting for RestoreJob to complete..." -wait_for_field restorejob restore-to-copy-test '{.status.phase}' Succeeded "$NAMESPACE" 600 - -separator - -log_step "Verifying RestoreJob result..." -log_command "kubectl get restorejob restore-to-copy-test -n $NAMESPACE -o yaml" -kubectl get restorejob restore-to-copy-test -n "$NAMESPACE" -o wide - -separator - -log_step "Checking resources in target namespace..." -log_command "kubectl get all -n $TARGET_NAMESPACE" -kubectl get all -n "$TARGET_NAMESPACE" 2>/dev/null || log_warning "No resources found in $TARGET_NAMESPACE" - -separator - -log_success "Cross-namespace restore completed successfully" diff --git a/examples/backups/vmi/cleanup.sh b/examples/backups/vmi/cleanup.sh deleted file mode 100755 index 5965c3f0..00000000 --- a/examples/backups/vmi/cleanup.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/bin/bash -# Clean up all resources created by the demo -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/00-helpers.sh" - -TARGET_NAMESPACE="${TARGET_NAMESPACE:-tenant-root-copy}" - -print_header "Cleanup Demo Resources" - -log_warning "This script will delete all resources created during the demo" -echo -e "\n${YELLOW}The following will be deleted:${NC}" -echo " - RestoreJobs (restore-in-place-test, restore-to-copy-test)" -echo " - BackupJob (test-backup) and associated Backup" -echo " - VMInstance (test)" -echo " - VMDisk (ubuntu-source)" -echo " - BackupClass (velero)" -echo " - Velero strategies (vminstance-strategy, vmdisk-strategy)" -echo " - Target namespace ($TARGET_NAMESPACE)" -echo "" - -read -p "Continue? (y/N): " -n 1 -r -echo -if [[ ! $REPLY =~ ^[Yy]$ ]]; then - log_info "Cancelled" - exit 0 -fi - -separator - -log_step "Deleting RestoreJobs..." -kubectl delete restorejob restore-to-copy-test -n "$NAMESPACE" 2>/dev/null || log_warning "RestoreJob restore-to-copy-test not found" -kubectl delete restorejob restore-in-place-test -n "$NAMESPACE" 2>/dev/null || log_warning "RestoreJob restore-in-place-test not found" - -separator - -log_step "Deleting BackupJob and Backup..." -kubectl delete backupjob test-backup -n "$NAMESPACE" 2>/dev/null || log_warning "BackupJob test-backup not found" -kubectl delete backup test-backup -n "$NAMESPACE" 2>/dev/null || log_warning "Backup test-backup not found" - -separator - -log_step "Deleting VMInstance..." -kubectl delete vminstance test -n "$NAMESPACE" 2>/dev/null || log_warning "VMInstance test not found" - -log_step "Deleting VMDisk..." -kubectl delete vmdisk ubuntu-source -n "$NAMESPACE" 2>/dev/null || log_warning "VMDisk ubuntu-source not found" - -separator - -log_step "Deleting BackupClass..." -kubectl delete backupclass velero 2>/dev/null || log_warning "BackupClass velero not found" - -separator - -log_step "Deleting Velero strategies..." -kubectl delete velero.strategy.backups.cozystack.io vminstance-strategy 2>/dev/null || log_warning "Strategy vminstance-strategy not found" -kubectl delete velero.strategy.backups.cozystack.io vmdisk-strategy 2>/dev/null || log_warning "Strategy vmdisk-strategy not found" - -separator - -log_step "Deleting target namespace..." -kubectl delete namespace "$TARGET_NAMESPACE" 2>/dev/null || log_warning "Namespace $TARGET_NAMESPACE not found" - -separator - -log_success "Cleanup complete" -log_info "All demo resources have been deleted" -echo -e "\n${GREEN}${BOLD}To re-run the demo:${NC} ./01-create-strategies.sh" diff --git a/examples/backups/vmi/run-all.sh b/examples/backups/vmi/run-all.sh deleted file mode 100755 index b58f4b9d..00000000 --- a/examples/backups/vmi/run-all.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash -# Run the full VMInstance backup/restore demo sequentially -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/00-helpers.sh" - -print_header "VMInstance Backup & Restore Demo" - -log_info "This script will run all demo steps sequentially" -log_info "There will be a pause between steps for observation" -echo "" - -SCRIPTS=( - "01-create-strategies.sh" - "02-create-backupclass.sh" - "03-create-vmdisk.sh" - "04-create-vminstance.sh" - "05-create-backupjob.sh" - "06-restore-in-place.sh" - "07-restore-to-copy.sh" -) - -echo -e "${CYAN}Demo steps:${NC}" -for i in "${!SCRIPTS[@]}"; do - echo " $((i+1)). ${SCRIPTS[$i]}" -done -echo "" - -read -p "Run demo? (y/N): " -n 1 -r -echo -if [[ ! $REPLY =~ ^[Yy]$ ]]; then - log_info "Cancelled" - exit 0 -fi - -separator - -for script in "${SCRIPTS[@]}"; do - if [[ -x "$SCRIPT_DIR/$script" ]]; then - "$SCRIPT_DIR/$script" - separator - wait_for_enter - else - log_error "Script not found or not executable: $script" - exit 1 - fi -done - -print_header "Demo Complete" - -log_success "All steps completed successfully!" -echo "" -log_info "What was demonstrated:" -echo " 1. Created Velero backup strategies for VMInstance and VMDisk" -echo " 2. Created BackupClass binding strategies to application types" -echo " 3. Provisioned a VMDisk and VMInstance" -echo " 4. Created a backup of the VMInstance via BackupJob" -echo " 5. Restored the VMInstance in-place" -echo " 6. Restored the VMInstance to a copy in a different namespace" -echo "" -log_info "To clean up resources: ./cleanup.sh" diff --git a/examples/backups/vmi/scenario-admin-prepare.md b/examples/backups/vmi/scenario-admin-prepare.md deleted file mode 100644 index edfca5d1..00000000 --- a/examples/backups/vmi/scenario-admin-prepare.md +++ /dev/null @@ -1,73 +0,0 @@ -# Scenario: Cluster Administrator Configures Backups - -This scenario walks through the cluster-level setup required before users can -back up and restore their virtual machines. A cluster administrator creates -Velero strategies and a BackupClass that binds those strategies to Cozystack -application types. - -## Prerequisites - -- A running Cozystack cluster with the backup-controller installed. -- Velero deployed in the `cozy-velero` namespace with a configured - BackupStorageLocation (default: `default`). -- `kubectl` access with cluster-admin privileges. - -## Steps - -### 1. Create Velero Strategies - -```bash -./01-create-strategies.sh -``` - -This creates two cluster-scoped `Velero` strategy objects: - -| Strategy | Purpose | -|---|---| -| `vminstance-strategy` | Defines backup and restore templates for `VMInstance` applications. Includes VirtualMachine, PVCs, HelmReleases, pods (required by kubevirt-velero-plugin), and supporting resources. Uses `snapshotMoveData: true` for portable volume snapshots. | -| `vmdisk-strategy` | Defines backup and restore templates for standalone `VMDisk` applications. Covers HelmReleases, PVCs, and ConfigMaps. | - -Both strategies use Go templates with `{{ .Application.metadata.name }}`, -`{{ .Application.metadata.namespace }}`, and `{{ .Parameters.backupStorageLocationName }}` -so they can be reused across any application instance. - -**Key design details:** - -- `orLabelSelectors` scope backups to only the resources belonging to a - specific application, preventing cross-contamination between VMs in the - same namespace. -- The restore spec excludes `datavolumes.cdi.kubevirt.io` to avoid conflicts - when the HelmRelease of VMDisk recreates DataVolumes after restore. -- `existingResourcePolicy: update` allows restoring over existing resources - during in-place restore. - -### 2. Create BackupClass - -```bash -./02-create-backupclass.sh -``` - -Creates a `BackupClass` named `velero` that maps application types to their -strategies: - -| Application Kind | Strategy | Parameters | -|---|---|---| -| `VMInstance` (`apps.cozystack.io`) | `vminstance-strategy` | `backupStorageLocationName: default` | -| `VMDisk` (`apps.cozystack.io`) | `vmdisk-strategy` | `backupStorageLocationName: default` | - -The `backupStorageLocationName` parameter can be overridden via the -`BACKUP_STORAGE_LOCATION` environment variable if your Velero installation -uses a non-default storage location. - -## Configuration - -| Variable | Default | Description | -|---|---|---| -| `BACKUP_STORAGE_LOCATION` | `default` | Velero BackupStorageLocation name | - -## Result - -After completing these steps the cluster is ready for users to create backups -of their `VMInstance` and `VMDisk` applications by referencing the `velero` -BackupClass. No further admin action is required for individual backup or -restore operations. diff --git a/examples/backups/vmi/scenario-user-backup.md b/examples/backups/vmi/scenario-user-backup.md deleted file mode 100644 index 31241fd9..00000000 --- a/examples/backups/vmi/scenario-user-backup.md +++ /dev/null @@ -1,91 +0,0 @@ -# Scenario: User Creates a VM Backup - -This scenario demonstrates how a tenant user provisions a virtual machine and -creates a backup of it. The backup captures the VM definition, its disks, and -network identity so it can be restored later. - -## Prerequisites - -- Cluster administrator has completed the setup from - [scenario-admin.md](scenario-admin.md) (strategies and BackupClass exist). -- A tenant namespace (default: `tenant-root`). -- `kubectl` access scoped to the tenant namespace. - -## Steps - -### 1. Create a VMDisk - -```bash -./03-create-vmdisk.sh -``` - -Creates a `VMDisk` named `ubuntu-source` that downloads the Ubuntu Noble cloud -image and provisions a 20Gi replicated PVC. The disk serves as the boot volume -for the virtual machine. - -Wait for the image download to complete before proceeding. You can monitor -progress with: - -```bash -kubectl get vmdisk ubuntu-source -n tenant-root -w -``` - -### 2. Create a VMInstance - -```bash -./04-create-vminstance.sh -``` - -Creates a `VMInstance` named `test` that references the `ubuntu-source` disk. -The VM boots with the `ubuntu` instance profile on a `u1.medium` instance type. - -Verify the VM is running: - -```bash -kubectl get vmi -n tenant-root -``` - -### 3. Create a BackupJob - -```bash -./05-create-backupjob.sh -``` - -Creates a `BackupJob` named `test-backup` that triggers a full backup of the -`test` VMInstance. The backup controller: - -1. Resolves the `velero` BackupClass to find the matching `vminstance-strategy`. -2. Discovers underlying resources (DataVolumes, OVN IP/MAC from the VM pod). -3. Creates a Velero Backup in the `cozy-velero` namespace with label selectors - scoped to this specific VM and its disks. -4. Velero snapshots and moves the volume data to the configured storage - location. - -The script waits up to 10 minutes for the BackupJob to reach the `Succeeded` -phase. On completion, a `Backup` object is created in the same namespace -containing: - -- `spec.applicationRef` — reference to the backed-up VMInstance. -- `spec.strategyRef` — reference to the Velero strategy used. -- `spec.driverMetadata` — Velero backup name for later restore. -- `status.underlyingResources` — captured DataVolume names and OVN IP/MAC - addresses. - -You can inspect the resulting Backup: - -```bash -kubectl get backups -n tenant-root -kubectl get backup test-backup -n tenant-root -o yaml -``` - -## Configuration - -| Variable | Default | Description | -|---|---|---| -| `NAMESPACE` | `tenant-root` | Tenant namespace for all resources | - -## Result - -After completing these steps you have a `Backup` artifact that can be used to -restore the VM either in-place or to a different namespace. See -[scenario-user-restore.md](scenario-user-restore.md) for restore options. diff --git a/examples/backups/vmi/scenario-user-restore.md b/examples/backups/vmi/scenario-user-restore.md deleted file mode 100644 index 1d35b22b..00000000 --- a/examples/backups/vmi/scenario-user-restore.md +++ /dev/null @@ -1,101 +0,0 @@ -# Scenario: User Restores a VM - -This scenario demonstrates two restore methods: in-place restore (rollback the -VM to a previous state) and cross-namespace restore (create a copy of the VM in -a different namespace). - -## Prerequisites - -- A completed backup from [scenario-user-backup.md](scenario-user-backup.md) - (the `test-backup` Backup object exists in the tenant namespace). -- `kubectl` access scoped to the tenant namespace. - -## Method 1: In-Place Restore - -```bash -./06-restore-in-place.sh -``` - -Creates a `RestoreJob` that restores the `test` VMInstance back to the state -captured in the `test-backup` Backup. The `targetApplicationRef` points to the -same application as the original backup. - -The backup controller performs the following steps automatically: - -1. **Suspends HelmReleases** — prevents Flux from reconciling the VM and its - disks during restore. -2. **Halts the VirtualMachine** — sets `runStrategy: Halted` and waits for the - VirtualMachineInstance (launcher pod) to terminate. -3. **Renames existing PVCs** — moves each PVC to `-orig-` to - preserve the current disk data as a safety net. -4. **Deletes DataVolumes** — removes DVs so CDI does not recreate PVCs before - Velero restores them. -5. **Creates Velero Restore** — restores resources from the backup with: - - `existingResourcePolicy: update` to overwrite existing objects. - - Resource modifier rules that add `cdi.kubevirt.io/allowClaimAdoption=true` - to restored PVCs so CDI can adopt them. - - OVN IP/MAC annotations to preserve the VM's network identity. - -After the Velero Restore completes, the HelmReleases resume and the VM boots -with the restored disk data while retaining its original IP and MAC addresses. - -## Method 2: Cross-Namespace Restore (Copy) - -```bash -./07-restore-to-copy.sh -``` - -Creates a `RestoreJob` with `spec.options.targetNamespace` set to a different namespace -(default: `tenant-root-copy`). This creates an independent copy of the VM -without affecting the original. - -Key differences from in-place restore: - -| Aspect | In-Place | Cross-Namespace | -|---|---|---| -| Source VM affected | Yes (halted, PVCs renamed) | No (untouched) | -| Velero namespaceMapping | Not used | Maps source NS to target NS | -| OVN IP/MAC | Preserved from backup | Skipped (new IP/MAC assigned) | -| Pre-restore preparation | Full (suspend, halt, rename, delete) | Skipped | - -The script ensures the target namespace exists before creating the RestoreJob. -Velero's `namespaceMapping` redirects all resources from the source namespace to -the target namespace during restore. - -### Important limitation - -Restoring to the **same namespace** with a **different application name** is not -supported. Velero's DataUpload always writes volume data to PVCs with the -original name regardless of resource modifiers, which causes conflicts. If you -attempt this, the RestoreJob will fail with an error message suggesting -cross-namespace restore instead. - -## Configuration - -| Variable | Default | Description | -|---|---|---| -| `NAMESPACE` | `tenant-root` | Source namespace (where the Backup lives) | -| `TARGET_NAMESPACE` | `tenant-root-copy` | Target namespace for cross-namespace restore | - -## Verify - -After either restore method, verify the VM is running: - -```bash -# In-place -kubectl get vmi -n tenant-root - -# Cross-namespace copy -kubectl get vmi -n tenant-root-copy -``` - -## Cleanup - -To remove all resources created by the demo: - -```bash -./cleanup.sh -``` - -This deletes RestoreJobs, BackupJobs, Backups, VMInstance, VMDisk, BackupClass, -strategies, and the target namespace. diff --git a/go.mod b/go.mod index 7b7d5bc4..00e8e129 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,6 @@ module github.com/cozystack/cozystack go 1.25.0 require ( - github.com/cozystack/cozystack-scheduler/pkg/apis v0.1.1 github.com/emicklei/dot v1.10.0 github.com/fluxcd/helm-controller/api v1.4.3 github.com/fluxcd/source-controller/api v1.7.4 @@ -29,10 +28,8 @@ require ( k8s.io/klog/v2 v2.130.1 k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b k8s.io/utils v0.0.0-20250820121507-0af2bda4dd1d - sigs.k8s.io/container-object-storage-interface-api v0.1.0 sigs.k8s.io/controller-runtime v0.22.4 sigs.k8s.io/structured-merge-diff/v6 v6.3.0 - sigs.k8s.io/yaml v1.6.0 ) require ( @@ -45,8 +42,10 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect + github.com/cozystack/cozystack-scheduler/pkg/apis v0.1.1 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.12.2 // indirect + github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.9.11 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fluxcd/pkg/apis/acl v0.9.0 // indirect @@ -127,6 +126,7 @@ require ( sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/yaml v1.6.0 // indirect ) // See: issues.k8s.io/135537 diff --git a/go.sum b/go.sum index 81e1779f..52942562 100644 --- a/go.sum +++ b/go.sum @@ -321,8 +321,6 @@ k8s.io/utils v0.0.0-20250820121507-0af2bda4dd1d h1:wAhiDyZ4Tdtt7e46e9M5ZSAJ/MnPG k8s.io/utils v0.0.0-20250820121507-0af2bda4dd1d/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 h1:jpcvIRr3GLoUoEKRkHKSmGjxb6lWwrBlJsXc+eUYQHM= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= -sigs.k8s.io/container-object-storage-interface-api v0.1.0 h1:8tB6JFQhbQIC1hwGQ+q4+tmSSNfjKemb7bFI6C0CK/4= -sigs.k8s.io/container-object-storage-interface-api v0.1.0/go.mod h1:YiB+i/UGkzqgODDhRG3u7jkbWkQcoUeLEJ7hwOT/2Qk= sigs.k8s.io/controller-runtime v0.22.4 h1:GEjV7KV3TY8e+tJ2LCTxUTanW4z/FmNB7l327UfMq9A= sigs.k8s.io/controller-runtime v0.22.4/go.mod h1:+QX1XUpTXN4mLoblf4tqr5CQcyHPAki2HLXqQMY6vh8= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= diff --git a/hack/admin-kubeconfig-invariant.bats b/hack/admin-kubeconfig-invariant.bats deleted file mode 100644 index a4b98b00..00000000 --- a/hack/admin-kubeconfig-invariant.bats +++ /dev/null @@ -1,145 +0,0 @@ -#!/usr/bin/env bats -# ----------------------------------------------------------------------------- -# Chart-wide invariant for packages/apps/kubernetes: -# -# Every Deployment in this chart that mounts -admin-kubeconfig as a -# Secret volume MUST: -# - declare that volume optional: true (so kubelet does not FailedMount -# while Kamaji is still provisioning the Secret), AND -# - include the wait-for-kubeconfig init container (so the pod becomes -# Ready only after Kamaji publishes the Secret). -# -# The per-template unittests in packages/apps/kubernetes/tests/ lock in -# today's three Deployments (cluster-autoscaler, kccm, csi controller) by -# name. This invariant is stricter: any future Deployment added to this -# chart that mounts the same Secret but forgets the guard will fail here. -# -# Requires: helm, yq (mikefarah v4+), jq. All three are available on the -# project's CI runners and on the maintainer workstation. -# ----------------------------------------------------------------------------- - -@test "every Deployment mounting admin-kubeconfig has optional:true and wait-for-kubeconfig init" { - values_file="packages/apps/kubernetes/tests/values-ci.yaml" - [ -f "$values_file" ] - - tmp=$(mktemp -d) - trap 'rm -rf "$tmp"' EXIT - - helm template invariant packages/apps/kubernetes \ - --namespace tenant-root \ - --values "$values_file" \ - 2>/dev/null > "$tmp/rendered.yaml" - - # yq streams one JSON object per input document. jq -s slurps the stream - # into an array so we can treat all Deployments as a single collection. - yq --output-format=json eval-all '.' "$tmp/rendered.yaml" \ - | jq -s --raw-output ' - map(select(.kind == "Deployment")) | - map({ - name: .metadata.name, - volumes: (.spec.template.spec.volumes // []), - initNames: ((.spec.template.spec.initContainers // []) | map(.name)), - }) | - map( - .name as $n | - .initNames as $ins | - (.volumes[] | select(.secret.secretName | test("-admin-kubeconfig$")?)) - | { - name: $n, - optional: (.secret.optional == true), - hasInit: ($ins | index("wait-for-kubeconfig") != null), - } - ) - ' > "$tmp/summary.json" - - # At least one Deployment must match; if a refactor removes every - # admin-kubeconfig volume from this chart, the test must be updated - # deliberately rather than silently passing. - matched=$(jq 'length' "$tmp/summary.json") - [ "$matched" -ge 1 ] - - offenders=$(jq --raw-output '.[] | select(.optional != true or .hasInit != true) | .name' "$tmp/summary.json") - - if [ -n "$offenders" ]; then - echo "Deployments mounting *-admin-kubeconfig without optional:true + wait-for-kubeconfig init:" >&2 - echo "$offenders" >&2 - echo "Full summary:" >&2 - cat "$tmp/summary.json" >&2 - exit 1 - fi - - echo "Invariant holds for $matched Deployment(s)" -} - -@test "chart emits zero admin-kubeconfig Deployments when tenant has no etcd DataStore" { - # Without a DataStore (parent Tenant has not populated _namespace.etcd yet) - # the control-plane-side Deployments must NOT render at all. If they did, - # the wait-for-kubeconfig init would CrashLoopBackOff indefinitely - there - # would be no KamajiControlPlane to provision the Secret - consuming the - # HelmRelease wait budget and triggering exactly the remediation cycle the - # rest of this chart tries to avoid. This test renders the whole chart - # with etcd empty and asserts no Deployment references the admin-kubeconfig - # Secret. - tmp=$(mktemp -d) - trap 'rm -rf "$tmp"' EXIT - - helm template invariant packages/apps/kubernetes \ - --namespace tenant-root \ - --values packages/apps/kubernetes/tests/values-ci-no-etcd.yaml \ - 2>/dev/null > "$tmp/rendered.yaml" - - matched=$( - yq --output-format=json eval-all '.' "$tmp/rendered.yaml" \ - | jq -s ' - map(select(.kind == "Deployment")) | - map(select( - (.spec.template.spec.volumes // []) - | any(.secret.secretName | test("-admin-kubeconfig$")?) - )) | - length - ' - ) - - if [ "$matched" -ne 0 ]; then - echo "Expected zero Deployments mounting *-admin-kubeconfig when etcd is empty, got $matched:" >&2 - yq --output-format=json eval-all '.' "$tmp/rendered.yaml" \ - | jq -s 'map(select(.kind == "Deployment") | .metadata.name)' >&2 - exit 1 - fi - - echo "No admin-kubeconfig Deployments rendered for empty etcd (as expected)" -} - -@test "chart emits zero admin-kubeconfig HelmReleases when tenant has no etcd DataStore" { - # Same principle as the Deployment variant above, extended to every child - # HelmRelease (cilium, coredns, csi, cert-manager, ...). They reference - # *-admin-kubeconfig via kubeConfig.secretRef and would otherwise sit in - # NotReady forever on an etcd-less tenant, polluting the HelmRelease list - # the operator sees and contradicting the "awaiting-etcd beacon only" - # contract of the soft-skip path. - tmp=$(mktemp -d) - trap 'rm -rf "$tmp"' EXIT - - helm template invariant packages/apps/kubernetes \ - --namespace tenant-root \ - --values packages/apps/kubernetes/tests/values-ci-no-etcd.yaml \ - 2>/dev/null > "$tmp/rendered.yaml" - - matched=$( - yq --output-format=json eval-all '.' "$tmp/rendered.yaml" \ - | jq -s ' - map(select(.kind == "HelmRelease")) | - map(select(.spec.kubeConfig.secretRef.name | test("-admin-kubeconfig$")?)) | - length - ' - ) - - if [ "$matched" -ne 0 ]; then - echo "Expected zero HelmReleases referencing *-admin-kubeconfig when etcd is empty, got $matched:" >&2 - yq --output-format=json eval-all '.' "$tmp/rendered.yaml" \ - | jq -s 'map(select(.kind == "HelmRelease") | .metadata.name)' >&2 - exit 1 - fi - - echo "No admin-kubeconfig HelmReleases rendered for empty etcd (as expected)" -} diff --git a/hack/cdi_golden_image_create.sh b/hack/cdi_golden_image_create.sh index f19f9970..be1558d0 100644 --- a/hack/cdi_golden_image_create.sh +++ b/hack/cdi_golden_image_create.sh @@ -16,7 +16,7 @@ kubectl create -f - <:(:)+ -# where each is [a-z0-9_]. Raw DCGM metrics (DCGM_FI_*), -# kube-state-metrics (kube_*) and similar uppercase / single-word metric -# names do not match because the leading segment must be lowercase and the -# whole expression must contain at least two ':' characters. -extract_refs() { - json_file=$1 - # Prometheus convention allows 2-segment rule names (level:metric); this - # regex is tuned to the 3+ segment convention used in this repo - # (level:metric:op — e.g. cluster:gpu_count:total). Update if future - # rules use 2 segments, otherwise they will be silently skipped. - grep -hoE '[a-z][a-z0-9_]*:[a-z0-9_]+:[a-z0-9_]+' "$json_file" | sort -u -} - -# Resolve "gpu/foo" -> "$DASHBOARDS_DIR/gpu/foo.json" -list_tracked_gpu_dashboards() { - awk '/^gpu\// { print $0 ".json" }' "$DASHBOARDS_LIST" -} - -# Extract the set of DCGM_FI_* metric names declared in a CSV file. Handles -# both the upstream-style default CSV (unindented) and the ConfigMap-style -# custom CSV (YAML-indented). A declaration line starts — after any leading -# whitespace — with "DCGM_FI_," ; comment lines begin with "#" and are -# skipped. Uses POSIX awk's match()+RSTART/RLENGTH so no GNU extensions -# are required. -extract_csv_metrics() { - file=$1 - awk ' - { - line = $0 - sub(/^[[:space:]]+/, "", line) - if (line ~ /^#/) next - if (match(line, /^DCGM_FI_[A-Z0-9_]+/)) { - print substr(line, RSTART, RLENGTH) - } - } - ' "$file" | sort -u -} - -# Extract the set of DCGM_FI_* metric references from a text file (dashboard -# JSON or rules YAML). A DCGM metric name has at least two underscore-delimited -# segments after the "DCGM_FI_" prefix (e.g. DCGM_FI_DEV_GPU_UTIL, DCGM_FI_PROF_ -# PIPE_TENSOR_ACTIVE, DCGM_FI_DRIVER_VERSION). Requiring two segments keeps -# the matcher from latching onto glob stubs like "DCGM_FI_DEV_*_VIOLATION" that -# appear in comments. -extract_dcgm_refs() { - file=$1 - grep -hoE 'DCGM_FI_[A-Z0-9]+(_[A-Z0-9]+)+' "$file" | sort -u -} - -@test "every recording rule reference in tracked GPU dashboards has a matching record" { - TMP=$(mktemp -d) - trap 'rm -rf "$TMP"' EXIT - - extract_rules > "$TMP/rules.txt" - [ -s "$TMP/rules.txt" ] || { echo "no recording rules extracted from $RULES_FILE" >&2; exit 1; } - - list_tracked_gpu_dashboards > "$TMP/dashboards.txt" - [ -s "$TMP/dashboards.txt" ] || { echo "no gpu/* dashboards listed in $DASHBOARDS_LIST" >&2; exit 1; } - - failed=0 - while IFS= read -r dashboard_rel; do - dashboard="$DASHBOARDS_DIR/$dashboard_rel" - if [ ! -f "$dashboard" ]; then - echo "ERROR: dashboard listed but file missing: $dashboard" >&2 - failed=1 - continue - fi - - extract_refs "$dashboard" > "$TMP/refs.txt" - # comm -23: lines unique to refs.txt (referenced but not defined) - # Both inputs must be sorted; extract_* helpers already sort. - comm -23 "$TMP/refs.txt" "$TMP/rules.txt" > "$TMP/missing.txt" - if [ -s "$TMP/missing.txt" ]; then - echo "ERROR: $dashboard_rel references undefined recording rules:" >&2 - sed 's/^/ - /' "$TMP/missing.txt" >&2 - failed=1 - fi - done < "$TMP/dashboards.txt" - - [ "$failed" -eq 0 ] -} - -@test "every DCGM metric referenced in tracked dashboards and rules is declared" { - TMP=$(mktemp -d) - trap 'rm -rf "$TMP"' EXIT - - [ -f "$DCGM_DEFAULT_CSV" ] || { echo "missing $DCGM_DEFAULT_CSV" >&2; exit 1; } - [ -f "$DCGM_CUSTOM_CSV" ] || { echo "missing $DCGM_CUSTOM_CSV" >&2; exit 1; } - - { - extract_csv_metrics "$DCGM_DEFAULT_CSV" - extract_csv_metrics "$DCGM_CUSTOM_CSV" - } | sort -u > "$TMP/declared.txt" - [ -s "$TMP/declared.txt" ] || { echo "no DCGM metrics extracted from CSVs" >&2; exit 1; } - - list_tracked_gpu_dashboards > "$TMP/dashboards.txt" - [ -s "$TMP/dashboards.txt" ] || { echo "no gpu/* dashboards listed in $DASHBOARDS_LIST" >&2; exit 1; } - - failed=0 - - # Dashboard coverage — every dashboard's DCGM references must resolve. - while IFS= read -r dashboard_rel; do - dashboard="$DASHBOARDS_DIR/$dashboard_rel" - [ -f "$dashboard" ] || continue # handled by the existence test - extract_dcgm_refs "$dashboard" > "$TMP/refs.txt" - [ -s "$TMP/refs.txt" ] || continue # dashboard relies entirely on recording rules - comm -23 "$TMP/refs.txt" "$TMP/declared.txt" > "$TMP/missing.txt" - if [ -s "$TMP/missing.txt" ]; then - echo "ERROR: $dashboard_rel references DCGM metrics not declared in any CSV:" >&2 - sed 's/^/ - /' "$TMP/missing.txt" >&2 - failed=1 - fi - done < "$TMP/dashboards.txt" - - # Rules coverage — recording rules consume DCGM directly, so their set - # must be declared too, otherwise derived series on every dashboard - # collapse to empty. - extract_dcgm_refs "$RULES_FILE" > "$TMP/rule-refs.txt" - if [ -s "$TMP/rule-refs.txt" ]; then - comm -23 "$TMP/rule-refs.txt" "$TMP/declared.txt" > "$TMP/rule-missing.txt" - if [ -s "$TMP/rule-missing.txt" ]; then - echo "ERROR: gpu-recording.rules.yaml references DCGM metrics not declared in any CSV:" >&2 - sed 's/^/ - /' "$TMP/rule-missing.txt" >&2 - failed=1 - fi - fi - - [ "$failed" -eq 0 ] -} - -@test "every tracked GPU dashboard listed in dashboards-infra.list exists on disk" { - TMP=$(mktemp -d) - trap 'rm -rf "$TMP"' EXIT - - list_tracked_gpu_dashboards > "$TMP/dashboards.txt" - [ -s "$TMP/dashboards.txt" ] || { echo "no gpu/* dashboards listed in $DASHBOARDS_LIST" >&2; exit 1; } - - failed=0 - while IFS= read -r dashboard_rel; do - dashboard="$DASHBOARDS_DIR/$dashboard_rel" - if [ ! -f "$dashboard" ]; then - echo "ERROR: $dashboard_rel listed in dashboards-infra.list but $dashboard does not exist" >&2 - failed=1 - fi - done < "$TMP/dashboards.txt" - - [ "$failed" -eq 0 ] -} diff --git a/hack/check-host-runtime.bats b/hack/check-host-runtime.bats deleted file mode 100644 index b121b31b..00000000 --- a/hack/check-host-runtime.bats +++ /dev/null @@ -1,415 +0,0 @@ -#!/usr/bin/env bats -# ----------------------------------------------------------------------------- -# Unit tests for hack/check-host-runtime.sh -# -# The script warns when a standalone containerd.service or docker.service is -# active alongside the embedded k3s runtime on Ubuntu hosts running the -# cozystack "generic" variant. Warnings go to stderr; exit code is always 0. -# -# Test strategy: each test builds its own temporary stub directory and prepends -# it to PATH to inject a fake `systemctl` (and optionally `du`) binary. The -# script itself honors a small set of COZYSTACK_PREFLIGHT_* environment -# variables to redirect socket/dir probes into the stub tree, so tests do not -# need root privileges or a real systemd host. -# -# Each test installs a `trap 'rm -rf "$STUB_DIR"' EXIT` immediately after -# creating the stub dir so cleanup runs even when an assertion fails mid-test -# under `set -e`. cozytest.sh runs each @test in its own subshell, so traps -# scope per test and do not leak across tests. -# -# Tests are otherwise self-contained — no shared setup/teardown helpers, -# because cozytest.sh's awk parser only recognizes @test blocks and treats a -# bare `}` on its own line as the end of a test function. -# -# Title syntax constraints (inherited from cozytest.sh's awk parser): -# - Titles must be delimited by ASCII double quotes; embedded literal -# double quotes are NOT escaped and will silently truncate the title. -# - Only alphanumeric characters from the title survive into the shell -# function name (everything else becomes '_'), so titles that differ -# only in punctuation collapse to the same function name. Keep titles -# distinctive in their alphanumeric run. -# -# Run with: hack/cozytest.sh hack/check-host-runtime.bats -# (or `bats hack/check-host-runtime.bats` if the bats binary is -# installed; cozytest.sh is the CI path.) -# ----------------------------------------------------------------------------- - -@test "clean host with no runtime services exits silently" { - STUB_DIR=$(mktemp -d) - trap 'rm -rf "$STUB_DIR"' EXIT - - cat >"$STUB_DIR/systemctl" <<'STUBEOF' -#!/bin/sh -if [ "$1" = "--version" ]; then - echo "systemd stub" - exit 0 -fi -exit 1 -STUBEOF - chmod +x "$STUB_DIR/systemctl" - - STDERR_FILE="$STUB_DIR/stderr" - COZYSTACK_CONTAINERD_SOCKET="$STUB_DIR/missing-containerd.sock" \ - COZYSTACK_DOCKER_SOCKET_PATHS="$STUB_DIR/missing-docker1.sock $STUB_DIR/missing-docker2.sock" \ - COZYSTACK_CONTAINERD_DIR="$STUB_DIR/missing-containerd-dir" \ - COZYSTACK_DOCKER_DIR="$STUB_DIR/missing-docker-dir" \ - PATH="$STUB_DIR:$PATH" \ - bash hack/check-host-runtime.sh >"$STUB_DIR/stdout" 2>"$STDERR_FILE" - - [ ! -s "$STDERR_FILE" ] - [ ! -s "$STUB_DIR/stdout" ] -} - -@test "standalone containerd service active prints warning" { - STUB_DIR=$(mktemp -d) - trap 'rm -rf "$STUB_DIR"' EXIT - - cat >"$STUB_DIR/systemctl" <<'STUBEOF' -#!/bin/sh -if [ "$1" = "--version" ]; then - echo "systemd stub" - exit 0 -fi -if [ "$1" = "is-active" ] && [ "$2" = "containerd.service" ]; then - echo active - exit 0 -fi -exit 1 -STUBEOF - chmod +x "$STUB_DIR/systemctl" - - mkdir -p "$STUB_DIR/var-lib-containerd" - echo dummy >"$STUB_DIR/var-lib-containerd/dummy" - - STDERR_FILE="$STUB_DIR/stderr" - COZYSTACK_CONTAINERD_SOCKET="$STUB_DIR/missing-containerd.sock" \ - COZYSTACK_DOCKER_SOCKET_PATHS="$STUB_DIR/missing-docker.sock" \ - COZYSTACK_CONTAINERD_DIR="$STUB_DIR/var-lib-containerd" \ - COZYSTACK_DOCKER_DIR="$STUB_DIR/missing-docker-dir" \ - PATH="$STUB_DIR:$PATH" \ - bash hack/check-host-runtime.sh 2>"$STDERR_FILE" - - grep -q 'standalone containerd.service' "$STDERR_FILE" - if grep -q 'standalone docker.service' "$STDERR_FILE"; then - echo "unexpected docker warning found:" >&2 - cat "$STDERR_FILE" >&2 - exit 1 - fi - # HINT line must name only the detected service, not advise disabling - # docker.service when only containerd.service is running. The sudo - # prefix is also required — without it the command silently no-ops - # for a non-root operator, so the prefix is part of the contract. - grep -q 'sudo systemctl disable --now containerd.service' "$STDERR_FILE" - if grep -q 'systemctl disable --now.*docker' "$STDERR_FILE"; then - echo "HINT unexpectedly mentions docker:" >&2 - cat "$STDERR_FILE" >&2 - exit 1 - fi -} - -@test "standalone docker service active prints warning" { - STUB_DIR=$(mktemp -d) - trap 'rm -rf "$STUB_DIR"' EXIT - - cat >"$STUB_DIR/systemctl" <<'STUBEOF' -#!/bin/sh -if [ "$1" = "--version" ]; then - echo "systemd stub" - exit 0 -fi -if [ "$1" = "is-active" ] && [ "$2" = "docker.service" ]; then - echo active - exit 0 -fi -exit 1 -STUBEOF - chmod +x "$STUB_DIR/systemctl" - - mkdir -p "$STUB_DIR/var-lib-docker" - echo dummy >"$STUB_DIR/var-lib-docker/dummy" - - STDERR_FILE="$STUB_DIR/stderr" - COZYSTACK_CONTAINERD_SOCKET="$STUB_DIR/missing-containerd.sock" \ - COZYSTACK_DOCKER_SOCKET_PATHS="$STUB_DIR/missing-docker.sock" \ - COZYSTACK_CONTAINERD_DIR="$STUB_DIR/missing-containerd-dir" \ - COZYSTACK_DOCKER_DIR="$STUB_DIR/var-lib-docker" \ - PATH="$STUB_DIR:$PATH" \ - bash hack/check-host-runtime.sh 2>"$STDERR_FILE" - - grep -q 'standalone docker.service' "$STDERR_FILE" - if grep -q 'standalone containerd.service' "$STDERR_FILE"; then - echo "unexpected containerd warning found:" >&2 - cat "$STDERR_FILE" >&2 - exit 1 - fi - # HINT line must name only the detected service. As in the - # containerd test, the sudo prefix is part of the contract. - grep -q 'sudo systemctl disable --now docker.service' "$STDERR_FILE" - if grep -q 'systemctl disable --now.*containerd' "$STDERR_FILE"; then - echo "HINT unexpectedly mentions containerd:" >&2 - cat "$STDERR_FILE" >&2 - exit 1 - fi -} - -@test "both services active prints two warnings and the HINT block" { - STUB_DIR=$(mktemp -d) - trap 'rm -rf "$STUB_DIR"' EXIT - - cat >"$STUB_DIR/systemctl" <<'STUBEOF' -#!/bin/sh -if [ "$1" = "--version" ]; then - echo "systemd stub" - exit 0 -fi -if [ "$1" = "is-active" ]; then - case "$2" in - containerd.service|docker.service) echo active; exit 0 ;; - esac -fi -exit 1 -STUBEOF - chmod +x "$STUB_DIR/systemctl" - - mkdir -p "$STUB_DIR/var-lib-containerd" "$STUB_DIR/var-lib-docker" - - STDERR_FILE="$STUB_DIR/stderr" - # Capture exit code explicitly: the script contract says exit 0 - # unconditionally (warning, not blocker). `set -e` in the test - # function body would already fail on a nonzero exit, but an - # explicit status check locks in the contract and makes a - # regression show up as "expected 0, got N" rather than as a - # generic test failure. - status=0 - COZYSTACK_CONTAINERD_SOCKET="$STUB_DIR/missing-containerd.sock" \ - COZYSTACK_DOCKER_SOCKET_PATHS="$STUB_DIR/missing-docker.sock" \ - COZYSTACK_CONTAINERD_DIR="$STUB_DIR/var-lib-containerd" \ - COZYSTACK_DOCKER_DIR="$STUB_DIR/var-lib-docker" \ - PATH="$STUB_DIR:$PATH" \ - bash hack/check-host-runtime.sh 2>"$STDERR_FILE" || status=$? - - [ "$status" -eq 0 ] - grep -q 'standalone containerd.service' "$STDERR_FILE" - grep -q 'standalone docker.service' "$STDERR_FILE" - # HINT block must fire whenever warnings exist; otherwise a future silent - # removal of the HINT would go unnoticed. When both services fire the HINT - # must list both in a single sudo systemctl disable invocation — the sudo - # prefix is as important as the systemctl verb, otherwise the operator - # would be told to run it as a non-root user and quietly fail. - grep -q 'HINT:' "$STDERR_FILE" - grep -q 'sudo systemctl disable --now containerd.service docker.service' "$STDERR_FILE" -} - -@test "failing du does not suppress the containerd warning" { - STUB_DIR=$(mktemp -d) - trap 'rm -rf "$STUB_DIR"' EXIT - - cat >"$STUB_DIR/systemctl" <<'STUBEOF' -#!/bin/sh -if [ "$1" = "--version" ]; then - echo "systemd stub" - exit 0 -fi -if [ "$1" = "is-active" ] && [ "$2" = "containerd.service" ]; then - echo active - exit 0 -fi -exit 1 -STUBEOF - chmod +x "$STUB_DIR/systemctl" - cat >"$STUB_DIR/du" <<'DUEOF' -#!/bin/sh -exit 1 -DUEOF - chmod +x "$STUB_DIR/du" - - mkdir -p "$STUB_DIR/var-lib-containerd" - - STDERR_FILE="$STUB_DIR/stderr" - COZYSTACK_CONTAINERD_SOCKET="$STUB_DIR/missing-containerd.sock" \ - COZYSTACK_DOCKER_SOCKET_PATHS="$STUB_DIR/missing-docker.sock" \ - COZYSTACK_CONTAINERD_DIR="$STUB_DIR/var-lib-containerd" \ - COZYSTACK_DOCKER_DIR="$STUB_DIR/missing-docker-dir" \ - PATH="$STUB_DIR:$PATH" \ - bash hack/check-host-runtime.sh 2>"$STDERR_FILE" - - grep -q 'standalone containerd.service' "$STDERR_FILE" -} - -@test "containerd socket fallback fires when systemctl is unavailable" { - STUB_DIR=$(mktemp -d) - trap 'rm -rf "$STUB_DIR"' EXIT - - # The script uses `[ -e "$sock" ]`, not `[ -S ... ]`, so a regular - # file is a valid stand-in for a unix socket in tests. This also - # removes any optional runtime dependency on python3 and makes the - # test unconditional on every CI runner. - SOCK="$STUB_DIR/containerd.sock" - touch "$SOCK" - - STDERR_FILE="$STUB_DIR/stderr" - COZYSTACK_PREFLIGHT_FORCE_NO_SYSTEMCTL=1 \ - COZYSTACK_CONTAINERD_SOCKET="$SOCK" \ - COZYSTACK_DOCKER_SOCKET_PATHS="$STUB_DIR/missing-docker.sock" \ - COZYSTACK_CONTAINERD_DIR="$STUB_DIR/missing-containerd-dir" \ - COZYSTACK_DOCKER_DIR="$STUB_DIR/missing-docker-dir" \ - bash hack/check-host-runtime.sh 2>"$STDERR_FILE" - - grep -q 'standalone containerd.service' "$STDERR_FILE" -} - -@test "docker socket fallback fires when systemctl is unavailable" { - STUB_DIR=$(mktemp -d) - trap 'rm -rf "$STUB_DIR"' EXIT - - SOCK="$STUB_DIR/docker.sock" - touch "$SOCK" - - STDERR_FILE="$STUB_DIR/stderr" - COZYSTACK_PREFLIGHT_FORCE_NO_SYSTEMCTL=1 \ - COZYSTACK_CONTAINERD_SOCKET="$STUB_DIR/missing-containerd.sock" \ - COZYSTACK_DOCKER_SOCKET_PATHS="$SOCK" \ - COZYSTACK_CONTAINERD_DIR="$STUB_DIR/missing-containerd-dir" \ - COZYSTACK_DOCKER_DIR="$STUB_DIR/missing-docker-dir" \ - bash hack/check-host-runtime.sh 2>"$STDERR_FILE" - - grep -q 'standalone docker.service' "$STDERR_FILE" - if grep -q 'standalone containerd.service' "$STDERR_FILE"; then - echo "unexpected containerd warning found:" >&2 - cat "$STDERR_FILE" >&2 - exit 1 - fi -} - -@test "clean host without systemctl exits silently" { - STUB_DIR=$(mktemp -d) - trap 'rm -rf "$STUB_DIR"' EXIT - - STDERR_FILE="$STUB_DIR/stderr" - COZYSTACK_PREFLIGHT_FORCE_NO_SYSTEMCTL=1 \ - COZYSTACK_CONTAINERD_SOCKET="$STUB_DIR/missing-containerd.sock" \ - COZYSTACK_DOCKER_SOCKET_PATHS="$STUB_DIR/missing-docker1.sock $STUB_DIR/missing-docker2.sock" \ - COZYSTACK_CONTAINERD_DIR="$STUB_DIR/missing-containerd-dir" \ - COZYSTACK_DOCKER_DIR="$STUB_DIR/missing-docker-dir" \ - bash hack/check-host-runtime.sh >"$STUB_DIR/stdout" 2>"$STDERR_FILE" - - [ ! -s "$STDERR_FILE" ] - [ ! -s "$STUB_DIR/stdout" ] -} - -@test "docker service plus socket still emits exactly one warning" { - STUB_DIR=$(mktemp -d) - trap 'rm -rf "$STUB_DIR"' EXIT - - cat >"$STUB_DIR/systemctl" <<'STUBEOF' -#!/bin/sh -if [ "$1" = "--version" ]; then - echo "systemd stub" - exit 0 -fi -if [ "$1" = "is-active" ] && [ "$2" = "docker.service" ]; then - echo active - exit 0 -fi -exit 1 -STUBEOF - chmod +x "$STUB_DIR/systemctl" - - SOCK="$STUB_DIR/docker.sock" - touch "$SOCK" - - mkdir -p "$STUB_DIR/var-lib-docker" - - STDERR_FILE="$STUB_DIR/stderr" - COZYSTACK_CONTAINERD_SOCKET="$STUB_DIR/missing-containerd.sock" \ - COZYSTACK_DOCKER_SOCKET_PATHS="$SOCK" \ - COZYSTACK_CONTAINERD_DIR="$STUB_DIR/missing-containerd-dir" \ - COZYSTACK_DOCKER_DIR="$STUB_DIR/var-lib-docker" \ - PATH="$STUB_DIR:$PATH" \ - bash hack/check-host-runtime.sh 2>"$STDERR_FILE" - - count=$(grep -c 'standalone docker.service' "$STDERR_FILE") - if [ "$count" != "1" ]; then - echo "expected exactly one docker warning, got $count" >&2 - cat "$STDERR_FILE" >&2 - exit 1 - fi -} - -@test "docker socket paths with glob chars do not expand" { - STUB_DIR=$(mktemp -d) - trap 'rm -rf "$STUB_DIR"' EXIT - - # Create two directories that a naive `for sock in $PATHS` loop - # would glob-expand and treat as existing "sockets". With the - # array-based parsing the literal path "$STUB_DIR/var-lib-*" does - # not exist and no warning must fire. - mkdir -p "$STUB_DIR/var-lib-docker" "$STUB_DIR/var-lib-containerd" - - cat >"$STUB_DIR/systemctl" <<'STUBEOF' -#!/bin/sh -if [ "$1" = "--version" ]; then - echo "systemd stub" - exit 0 -fi -exit 1 -STUBEOF - chmod +x "$STUB_DIR/systemctl" - - STDERR_FILE="$STUB_DIR/stderr" - COZYSTACK_CONTAINERD_SOCKET="$STUB_DIR/missing-containerd.sock" \ - COZYSTACK_DOCKER_SOCKET_PATHS="$STUB_DIR/var-lib-*" \ - COZYSTACK_CONTAINERD_DIR="$STUB_DIR/missing-containerd-dir" \ - COZYSTACK_DOCKER_DIR="$STUB_DIR/missing-docker-dir" \ - PATH="$STUB_DIR:$PATH" \ - bash hack/check-host-runtime.sh 2>"$STDERR_FILE" - - if grep -q 'standalone docker.service' "$STDERR_FILE"; then - echo "glob pattern expanded — docker warning should not fire:" >&2 - cat "$STDERR_FILE" >&2 - exit 1 - fi -} - -@test "containerd service plus socket still emits exactly one warning" { - STUB_DIR=$(mktemp -d) - trap 'rm -rf "$STUB_DIR"' EXIT - - cat >"$STUB_DIR/systemctl" <<'STUBEOF' -#!/bin/sh -if [ "$1" = "--version" ]; then - echo "systemd stub" - exit 0 -fi -if [ "$1" = "is-active" ] && [ "$2" = "containerd.service" ]; then - echo active - exit 0 -fi -exit 1 -STUBEOF - chmod +x "$STUB_DIR/systemctl" - - # The script uses `[ -e "$sock" ]`, not `[ -S ... ]`, so a regular - # file is a valid stand-in for a unix socket in tests. This also - # removes any optional runtime dependency on python3 and makes the - # test unconditional on every CI runner. - SOCK="$STUB_DIR/containerd.sock" - touch "$SOCK" - - mkdir -p "$STUB_DIR/var-lib-containerd" - - STDERR_FILE="$STUB_DIR/stderr" - COZYSTACK_CONTAINERD_SOCKET="$SOCK" \ - COZYSTACK_DOCKER_SOCKET_PATHS="$STUB_DIR/missing-docker.sock" \ - COZYSTACK_CONTAINERD_DIR="$STUB_DIR/var-lib-containerd" \ - COZYSTACK_DOCKER_DIR="$STUB_DIR/missing-docker-dir" \ - PATH="$STUB_DIR:$PATH" \ - bash hack/check-host-runtime.sh 2>"$STDERR_FILE" - - count=$(grep -c 'standalone containerd.service' "$STDERR_FILE") - if [ "$count" != "1" ]; then - echo "expected exactly one containerd warning, got $count" >&2 - cat "$STDERR_FILE" >&2 - exit 1 - fi -} diff --git a/hack/check-host-runtime.sh b/hack/check-host-runtime.sh deleted file mode 100755 index 917453d2..00000000 --- a/hack/check-host-runtime.sh +++ /dev/null @@ -1,161 +0,0 @@ -#!/usr/bin/env bash -# ----------------------------------------------------------------------------- -# check-host-runtime.sh — operator preflight warning -# -# Purpose: -# Warn when a standalone containerd.service or docker.service is running on -# the host alongside the embedded k3s runtime. This mismatch is silent on -# day 0 (k3s uses its own containerd at /run/k3s/containerd/containerd.sock -# and /var/lib/rancher/k3s/agent/containerd) but over time the standalone -# runtime accumulates unpruned images and build cache in /var/lib/containerd -# — enough to trigger DiskPressure and crash cozystack-api with eviction -# loops. The script does NOT block install; it only prints a warning. -# -# When to run: -# Before `helm install cozy-installer` on an Ubuntu host prepared with k3s -# or kubeadm (cozystack "generic" variant). Irrelevant on Talos where the -# container runtime lifecycle is fully managed. Discoverable via -# `make preflight` from the repository root. -# -# Exit code: -# Always 0 (warning, not a blocker). Warnings go to stderr. -# -# Environment variables (test hooks — override default probe paths): -# COZYSTACK_CONTAINERD_SOCKET standalone containerd socket path -# COZYSTACK_DOCKER_SOCKET_PATHS space-separated list of docker socket paths -# COZYSTACK_CONTAINERD_DIR standalone containerd data directory -# COZYSTACK_DOCKER_DIR standalone docker data directory -# COZYSTACK_PREFLIGHT_FORCE_NO_SYSTEMCTL=1 pretend systemctl is absent -# ----------------------------------------------------------------------------- -set -euo pipefail - -if [ -t 2 ]; then - YELLOW=$'\033[1;33m' - RESET=$'\033[0m' -else - YELLOW='' - RESET='' -fi - -CONTAINERD_SOCKET=${COZYSTACK_CONTAINERD_SOCKET:-/run/containerd/containerd.sock} -DOCKER_SOCKET_PATHS=${COZYSTACK_DOCKER_SOCKET_PATHS:-/run/docker.sock /var/run/docker.sock} -CONTAINERD_DIR=${COZYSTACK_CONTAINERD_DIR:-/var/lib/containerd} -DOCKER_DIR=${COZYSTACK_DOCKER_DIR:-/var/lib/docker} - -CONTAINERD_WARN=0 -DOCKER_WARN=0 - -warn() { - printf '%sWARNING:%s %s\n' "$YELLOW" "$RESET" "$1" >&2 -} - -detect_systemctl() { - if [ "${COZYSTACK_PREFLIGHT_FORCE_NO_SYSTEMCTL:-0}" = "1" ]; then - return 1 - fi - if command -v systemctl >/dev/null 2>&1 && systemctl --version >/dev/null 2>&1; then - return 0 - fi - return 1 -} - -disk_usage() { - local path=$1 - local usage - if [ -d "$path" ]; then - # Wrap `du` in `timeout 5s` so a container data directory with - # millions of files (the exact scenario this script exists to - # warn about) cannot stall the preflight indefinitely. If the - # `timeout` binary is absent the pipeline still exits 0 via - # `|| true` and `usage` stays empty; the warning itself is - # still printed, just without the size detail. - usage=$(timeout 5s du -sh "$path" 2>/dev/null | awk '{print $1}' || true) - if [ -n "${usage:-}" ]; then - printf ' (%s uses %s)' "$path" "$usage" - fi - fi -} - -service_active() { - local service=$1 - if [ "$HAS_SYSTEMCTL" = "1" ]; then - if systemctl is-active "$service" >/dev/null 2>&1; then - return 0 - fi - fi - return 1 -} - -check_containerd() { - local detail="" - local found=0 - if service_active containerd.service; then - found=1 - fi - if [ "$found" -eq 0 ] && [ -e "$CONTAINERD_SOCKET" ]; then - found=1 - fi - if [ "$found" -eq 1 ]; then - detail=$(disk_usage "$CONTAINERD_DIR") - warn "standalone containerd.service detected alongside k3s embedded runtime${detail}" - CONTAINERD_WARN=1 - fi -} - -check_docker() { - local detail="" - local found=0 - if service_active docker.service; then - found=1 - fi - if [ "$found" -eq 0 ]; then - # DOCKER_SOCKET_PATHS is a space separated list of paths. Parse - # it into an array via `read -ra` so that word splitting is - # explicit AND glob expansion is suppressed — `for sock in - # $DOCKER_SOCKET_PATHS` would both word split and glob, so a - # path containing a literal `*` or `?` could expand into - # directory entries and produce false positives. - local -a _socks - read -ra _socks <<<"$DOCKER_SOCKET_PATHS" - for sock in "${_socks[@]}"; do - if [ -e "$sock" ]; then - found=1 - break - fi - done - fi - if [ "$found" -eq 1 ]; then - detail=$(disk_usage "$DOCKER_DIR") - warn "standalone docker.service detected alongside k3s embedded runtime${detail}" - DOCKER_WARN=1 - fi -} - -if detect_systemctl; then - HAS_SYSTEMCTL=1 -else - HAS_SYSTEMCTL=0 -fi - -check_containerd -check_docker - -if [ "$CONTAINERD_WARN" -eq 1 ] || [ "$DOCKER_WARN" -eq 1 ]; then - services="" - if [ "$CONTAINERD_WARN" -eq 1 ]; then - services="containerd.service" - fi - if [ "$DOCKER_WARN" -eq 1 ]; then - if [ -n "$services" ]; then - services="$services docker.service" - else - services="docker.service" - fi - fi - printf '%sHINT:%s cozystack runs its own containerd under k3s. To stop the shadow runtime:\n' "$YELLOW" "$RESET" >&2 - printf ' sudo systemctl disable --now %s\n' "$services" >&2 - printf 'Inspect and reclaim standalone runtime storage separately — it may contain container data\n' >&2 - printf 'that the operator still needs; do not delete it blindly.\n' >&2 -fi - -exit 0 diff --git a/hack/common-envs.mk b/hack/common-envs.mk index 9b8db5db..56eb5478 100644 --- a/hack/common-envs.mk +++ b/hack/common-envs.mk @@ -6,13 +6,13 @@ else endif REGISTRY ?= ghcr.io/cozystack/cozystack -TAG = $(shell git describe --tags --exact-match --match 'v*' 2>/dev/null || echo latest) +TAG = $(shell git describe --tags --exact-match 2>/dev/null || echo latest) PUSH := 1 LOAD := 0 BUILDER ?= PLATFORM ?= BUILDX_EXTRA_ARGS ?= -COZYSTACK_VERSION = $(patsubst v%,%,$(shell git describe --tags --match 'v*')) +COZYSTACK_VERSION = $(patsubst v%,%,$(shell git describe --tags)) BUILDX_ARGS := --provenance=false --push=$(PUSH) --load=$(LOAD) \ --label org.opencontainers.image.source=https://github.com/cozystack/cozystack \ @@ -28,6 +28,6 @@ endef ifeq ($(COZYSTACK_VERSION),) $(shell git remote add upstream https://github.com/cozystack/cozystack.git || true) $(shell git fetch upstream --tags) - COZYSTACK_VERSION = $(patsubst v%,%,$(shell git describe --tags --match 'v*')) + COZYSTACK_VERSION = $(patsubst v%,%,$(shell git describe --tags)) endif diff --git a/hack/dcgm-default-counters.csv b/hack/dcgm-default-counters.csv deleted file mode 100644 index b5e94540..00000000 --- a/hack/dcgm-default-counters.csv +++ /dev/null @@ -1,104 +0,0 @@ -# Snapshot of the upstream DCGM Exporter default-counters.csv used as a -# fixture by hack/check-gpu-recording-rules.bats. The test verifies that -# every DCGM_FI_* metric referenced by a tracked GPU dashboard is either -# declared here (upstream defaults) or in -# packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml -# (the project's custom CSV). -# -# Source: https://github.com/NVIDIA/dcgm-exporter/blob/4.1.1-4.0.4/etc/default-counters.csv -# Pinned to the DCGM Exporter image tag shipped by the gpu-operator -# chart under packages/system/gpu-operator/charts/gpu-operator/values.yaml -# (dcgmExporter.version = 4.1.1-4.0.4-ubuntu22.04). When that image is -# bumped, refresh this file from the matching tag in the NVIDIA/dcgm-exporter -# repo and re-run `./hack/cozytest.sh hack/check-gpu-recording-rules.bats`. - -# Format -# If line starts with a '#' it is considered a comment -# DCGM FIELD, Prometheus metric type, help message - -# Clocks -DCGM_FI_DEV_SM_CLOCK, gauge, SM clock frequency (in MHz). -DCGM_FI_DEV_MEM_CLOCK, gauge, Memory clock frequency (in MHz). - -# Temperature -DCGM_FI_DEV_MEMORY_TEMP, gauge, Memory temperature (in C). -DCGM_FI_DEV_GPU_TEMP, gauge, GPU temperature (in C). - -# Power -DCGM_FI_DEV_POWER_USAGE, gauge, Power draw (in W). -DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION, counter, Total energy consumption since boot (in mJ). - -# PCIE -# DCGM_FI_PROF_PCIE_TX_BYTES, counter, Total number of bytes transmitted through PCIe TX via NVML. -# DCGM_FI_PROF_PCIE_RX_BYTES, counter, Total number of bytes received through PCIe RX via NVML. -DCGM_FI_DEV_PCIE_REPLAY_COUNTER, counter, Total number of PCIe retries. - -# Utilization (the sample period varies depending on the product) -DCGM_FI_DEV_GPU_UTIL, gauge, GPU utilization (in %). -DCGM_FI_DEV_MEM_COPY_UTIL, gauge, Memory utilization (in %). -DCGM_FI_DEV_ENC_UTIL, gauge, Encoder utilization (in %). -DCGM_FI_DEV_DEC_UTIL , gauge, Decoder utilization (in %). - -# Errors and violations -DCGM_FI_DEV_XID_ERRORS, gauge, Value of the last XID error encountered. -# DCGM_FI_DEV_POWER_VIOLATION, counter, Throttling duration due to power constraints (in us). -# DCGM_FI_DEV_THERMAL_VIOLATION, counter, Throttling duration due to thermal constraints (in us). -# DCGM_FI_DEV_SYNC_BOOST_VIOLATION, counter, Throttling duration due to sync-boost constraints (in us). -# DCGM_FI_DEV_BOARD_LIMIT_VIOLATION, counter, Throttling duration due to board limit constraints (in us). -# DCGM_FI_DEV_LOW_UTIL_VIOLATION, counter, Throttling duration due to low utilization (in us). -# DCGM_FI_DEV_RELIABILITY_VIOLATION, counter, Throttling duration due to reliability constraints (in us). - -# Memory usage -DCGM_FI_DEV_FB_FREE, gauge, Framebuffer memory free (in MiB). -DCGM_FI_DEV_FB_USED, gauge, Framebuffer memory used (in MiB). - -# ECC -# DCGM_FI_DEV_ECC_SBE_VOL_TOTAL, counter, Total number of single-bit volatile ECC errors. -# DCGM_FI_DEV_ECC_DBE_VOL_TOTAL, counter, Total number of double-bit volatile ECC errors. -# DCGM_FI_DEV_ECC_SBE_AGG_TOTAL, counter, Total number of single-bit persistent ECC errors. -# DCGM_FI_DEV_ECC_DBE_AGG_TOTAL, counter, Total number of double-bit persistent ECC errors. - -# Retired pages -# DCGM_FI_DEV_RETIRED_SBE, counter, Total number of retired pages due to single-bit errors. -# DCGM_FI_DEV_RETIRED_DBE, counter, Total number of retired pages due to double-bit errors. -# DCGM_FI_DEV_RETIRED_PENDING, counter, Total number of pages pending retirement. - -# NVLink -# DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_TOTAL, counter, Total number of NVLink flow-control CRC errors. -# DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_TOTAL, counter, Total number of NVLink data CRC errors. -# DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_TOTAL, counter, Total number of NVLink retries. -# DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_TOTAL, counter, Total number of NVLink recovery errors. -DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL, counter, Total number of NVLink bandwidth counters for all lanes. -# DCGM_FI_DEV_NVLINK_BANDWIDTH_L0, counter, The number of bytes of active NVLink rx or tx data including both header and payload. - -# VGPU License status -DCGM_FI_DEV_VGPU_LICENSE_STATUS, gauge, vGPU License status - -# Remapped rows -DCGM_FI_DEV_UNCORRECTABLE_REMAPPED_ROWS, counter, Number of remapped rows for uncorrectable errors -DCGM_FI_DEV_CORRECTABLE_REMAPPED_ROWS, counter, Number of remapped rows for correctable errors -DCGM_FI_DEV_ROW_REMAP_FAILURE, gauge, Whether remapping of rows has failed - -# Static configuration information. These appear as labels on the other metrics -DCGM_FI_DRIVER_VERSION, label, Driver Version -# DCGM_FI_NVML_VERSION, label, NVML Version -# DCGM_FI_DEV_BRAND, label, Device Brand -# DCGM_FI_DEV_SERIAL, label, Device Serial Number -# DCGM_FI_DEV_OEM_INFOROM_VER, label, OEM inforom version -# DCGM_FI_DEV_ECC_INFOROM_VER, label, ECC inforom version -# DCGM_FI_DEV_POWER_INFOROM_VER, label, Power management object inforom version -# DCGM_FI_DEV_INFOROM_IMAGE_VER, label, Inforom image version -# DCGM_FI_DEV_VBIOS_VERSION, label, VBIOS version of the device - -# Datacenter Profiling (DCP) metrics -# NOTE: supported on Nvidia datacenter Volta GPUs and newer -DCGM_FI_PROF_GR_ENGINE_ACTIVE, gauge, Ratio of time the graphics engine is active. -# DCGM_FI_PROF_SM_ACTIVE, gauge, The ratio of cycles an SM has at least 1 warp assigned. -# DCGM_FI_PROF_SM_OCCUPANCY, gauge, The ratio of number of warps resident on an SM. -DCGM_FI_PROF_PIPE_TENSOR_ACTIVE, gauge, Ratio of cycles the tensor (HMMA) pipe is active. -DCGM_FI_PROF_DRAM_ACTIVE, gauge, Ratio of cycles the device memory interface is active sending or receiving data. -# DCGM_FI_PROF_PIPE_FP64_ACTIVE, gauge, Ratio of cycles the fp64 pipes are active. -# DCGM_FI_PROF_PIPE_FP32_ACTIVE, gauge, Ratio of cycles the fp32 pipes are active. -# DCGM_FI_PROF_PIPE_FP16_ACTIVE, gauge, Ratio of cycles the fp16 pipes are active. -DCGM_FI_PROF_PCIE_TX_BYTES, gauge, The rate of data transmitted over the PCIe bus - including both protocol headers and data payloads - in bytes per second. -DCGM_FI_PROF_PCIE_RX_BYTES, gauge, The rate of data received over the PCIe bus - including both protocol headers and data payloads - in bytes per second. diff --git a/hack/e2e-apps/kafka.bats b/hack/e2e-apps/kafka.bats index d85837bd..0afd374d 100644 --- a/hack/e2e-apps/kafka.bats +++ b/hack/e2e-apps/kafka.bats @@ -3,7 +3,6 @@ @test "Create Kafka" { name='test' kubectl -n tenant-test delete kafka.apps.cozystack.io $name --ignore-not-found --timeout=2m || true - kubectl -n tenant-test wait kafka.apps.cozystack.io $name --for=delete --timeout=2m 2>/dev/null || true kubectl apply -f- <}" - if [ -z "${history_statuses}" ]; then - echo "Unexpected empty .status.history on a Ready HelmRelease - Flux API shape may have changed." >&2 - kubectl -n tenant-test describe hr "kubernetes-${test_name}" >&2 - exit 1 - fi - if helmrelease_has_remediation_cycle "${history_statuses}"; then - echo "Parent HelmRelease entered remediation cycle." >&2 - kubectl -n tenant-test describe hr "kubernetes-${test_name}" >&2 - exit 1 - fi - # Clean up pkill -f "port-forward.*${port}:" 2>/dev/null || true rm -f "tenantkubeconfig-${test_name}" diff --git a/hack/e2e-install-cozystack.bats b/hack/e2e-install-cozystack.bats index cf66b73f..743782e9 100644 --- a/hack/e2e-install-cozystack.bats +++ b/hack/e2e-install-cozystack.bats @@ -200,60 +200,8 @@ EOF kubectl wait hr/keycloak hr/keycloak-configure hr/keycloak-operator -n cozy-keycloak --timeout=10m --for=condition=ready } -@test "Aggregated API rejects Tenant name with dashes" { - # Regression guard: the tenant Helm chart's tenant.name helper splits the - # Release.Name on "-" and fails unless the result is exactly - # ["tenant", ""]. The aggregated API must catch tenant names - # containing dashes up-front with a tenant-specific error, instead of - # silently accepting the Application and letting Flux fail later. - - # Defensive cleanup: if a prior regression left foo-bar in the cluster, - # remove it before exercising the validation so we are not observing - # stale state. Safe even in the happy path because of --ignore-not-found. - kubectl delete tenants.apps.cozystack.io foo-bar -n tenant-root --ignore-not-found - - # Preflight: tenant-root is created by earlier tests in this suite. Fail - # loudly if it is missing so this test does not silently trigger an - # unrelated "namespace not found" error and misreport as a pass. - kubectl get namespace tenant-root - - # --validate=ignore forces kubectl to skip client-side OpenAPI validation - # and send the payload straight to the aggregated API. This guarantees the - # server-side name check runs and the error we grep for is the tenant - # contract error, not a kubectl schema rejection. (--validate=false is the - # deprecated alias.) - local output rc - # Run the apply in its own subshell so we can capture BOTH stdout+stderr - # AND the exit code explicitly, without `|| true` swallowing a real failure - # mode (e.g. network error, auth failure) that should also fail the test. - output=$( - kubectl apply --validate=ignore -f - 2>&1 <&2 - exit 1 - fi -} - -@test "single deployed snapshot returns not-detected" { - . hack/e2e-apps/remediation-guard.sh - if helmrelease_has_remediation_cycle "deployed"; then - echo "expected not-detected for deployed-only history" >&2 - exit 1 - fi -} - -@test "deployed then superseded returns not-detected" { - . hack/e2e-apps/remediation-guard.sh - statuses=$(printf 'deployed\nsuperseded\n') - if helmrelease_has_remediation_cycle "${statuses}"; then - echo "expected not-detected for deployed+superseded history" >&2 - exit 1 - fi -} - -@test "single failed snapshot returns detected" { - . hack/e2e-apps/remediation-guard.sh - if ! helmrelease_has_remediation_cycle "failed"; then - echo "expected detected when history contains failed snapshot" >&2 - exit 1 - fi -} - -@test "single uninstalled snapshot returns detected" { - # The exact signature of the install-remediation race: the first install - # exceeded flux's wait budget, remediation uninstalled, the next retry - # eventually succeeded. History still carries the uninstalled snapshot. - . hack/e2e-apps/remediation-guard.sh - if ! helmrelease_has_remediation_cycle "uninstalled"; then - echo "expected detected when history contains uninstalled snapshot" >&2 - exit 1 - fi -} - -@test "uninstalled then deployed still returns detected" { - . hack/e2e-apps/remediation-guard.sh - statuses=$(printf 'uninstalled\ndeployed\n') - if ! helmrelease_has_remediation_cycle "${statuses}"; then - echo "expected detected despite later successful deploy" >&2 - exit 1 - fi -} - -@test "deployed then failed still returns detected" { - . hack/e2e-apps/remediation-guard.sh - statuses=$(printf 'deployed\nfailed\n') - if ! helmrelease_has_remediation_cycle "${statuses}"; then - echo "expected detected when any entry is failed" >&2 - exit 1 - fi -} - -@test "status.history extraction pins HR v2 status.history shape" { - # Pins the Flux HelmRelease v2 .status.history[].status shape that - # run-kubernetes.sh relies on. If a future flux release renames the - # field, the jsonpath returns nothing, the guard reports no cycle, - # and real remediation loops slip past the e2e assertion. This test - # uses yq to read the exact path used in the e2e script; the upstream - # Snapshot type lives at - # github.com/fluxcd/helm-controller/api/v2.Snapshot (via go.mod). - tmp=$(mktemp -d) - trap 'rm -rf "$tmp"' EXIT - - cat > "$tmp/hr.yaml" <<'YAML' -apiVersion: helm.toolkit.fluxcd.io/v2 -kind: HelmRelease -metadata: - name: kubernetes-test - namespace: tenant-test -status: - history: - - name: kubernetes-test - namespace: tenant-test - version: 1 - status: uninstalled - - name: kubernetes-test - namespace: tenant-test - version: 2 - status: deployed -YAML - - # Default yq output is yaml scalar format, which for string values emits - # bare unquoted tokens - matching what kubectl -o jsonpath produces in - # e2e. Do not switch to JSON output here; that would quote the values - # and break the loop in helmrelease_has_remediation_cycle. - statuses=$(yq '.status.history[].status' "$tmp/hr.yaml") - - [ -n "$statuses" ] - echo "$statuses" | grep --quiet '^uninstalled$' - - . hack/e2e-apps/remediation-guard.sh - if ! helmrelease_has_remediation_cycle "$statuses"; then - echo "expected detected for pinned HR snippet with uninstalled + deployed history" >&2 - exit 1 - fi -} diff --git a/internal/backupcontroller/restorejob_controller.go b/internal/backupcontroller/restorejob_controller.go index f0062064..d73b0c24 100644 --- a/internal/backupcontroller/restorejob_controller.go +++ b/internal/backupcontroller/restorejob_controller.go @@ -153,23 +153,6 @@ func (r *RestoreJobReconciler) markRestoreJobFailed(ctx context.Context, restore return ctrl.Result{}, nil } -// cleanupResourceModifierConfigMaps deletes resource modifier ConfigMaps owned -// by this RestoreJob. Called on completion (success or failure) to avoid leaking -// ConfigMaps in cozy-velero when RestoreJobs are not immediately deleted. -func (r *RestoreJobReconciler) cleanupResourceModifierConfigMaps(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob) { - logger := log.FromContext(ctx) - opts := []client.DeleteAllOfOption{ - client.InNamespace(veleroNamespace), - client.MatchingLabels{ - backupsv1alpha1.OwningJobNameLabel: restoreJob.Name, - backupsv1alpha1.OwningJobNamespaceLabel: restoreJob.Namespace, - }, - } - if err := r.DeleteAllOf(ctx, &corev1.ConfigMap{}, opts...); err != nil { - logger.Error(err, "failed to clean up resourceModifiers ConfigMap(s)") - } -} - // cleanupVeleroRestore deletes all Velero Restores and resourceModifier // ConfigMaps owned by this RestoreJob (identified by labels). func (r *RestoreJobReconciler) cleanupVeleroRestore(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob) { diff --git a/internal/backupcontroller/velerostrategy_controller.go b/internal/backupcontroller/velerostrategy_controller.go index 8b063519..d5c330ae 100644 --- a/internal/backupcontroller/velerostrategy_controller.go +++ b/internal/backupcontroller/velerostrategy_controller.go @@ -75,107 +75,6 @@ func stringPtr(s string) *string { return &s } -// boolPtr returns a pointer to a bool value. -func boolPtr(b bool) *bool { - return &b -} - -// boolDefault returns the value of a *bool pointer, or the given default if nil. -func boolDefault(p *bool, def bool) bool { - if p != nil { - return *p - } - return def -} - -// CommonRestoreOptions contains driver-agnostic restore options shared across -// all application kinds. -type CommonRestoreOptions struct { - // TargetNamespace is the namespace to restore into. When set (and differs - // from the backup namespace), a cross-namespace restore (copy) is performed - // using Velero's namespaceMapping. - TargetNamespace string `json:"targetNamespace,omitempty"` - // FailIfTargetExists makes the restore fail if the target resource already - // exists. Defaults to true when omitted. - FailIfTargetExists *bool `json:"failIfTargetExists,omitempty"` -} - -// RestoreOptions is the typed representation of RestoreJob.Spec.Options for the -// Velero driver. The struct is deserialized from runtime.RawExtension and used -// for all application kinds. VMInstance-specific fields (KeepOriginalPVC, -// KeepOriginalIpAndMac) are only effective when the application kind is VMInstance. -type RestoreOptions struct { - CommonRestoreOptions `json:",inline"` - // KeepOriginalPVC renames the original PVC to -orig- before restore. - // Only effective for in-place VMInstance restore (no targetNamespace). Defaults to true when omitted. - KeepOriginalPVC *bool `json:"keepOriginalPVC,omitempty"` - // KeepOriginalIpAndMac preserves the original IP and MAC address via OVN - // annotations. Only effective for VMInstance restores. Defaults to true when omitted. - KeepOriginalIpAndMac *bool `json:"keepOriginalIpAndMac,omitempty"` -} - -// GetFailIfTargetExists returns the effective value (default: true). -func (o *CommonRestoreOptions) GetFailIfTargetExists() bool { - return boolDefault(o.FailIfTargetExists, true) -} - -// GetKeepOriginalPVC returns the effective value (default: true). -func (o *RestoreOptions) GetKeepOriginalPVC() bool { - return boolDefault(o.KeepOriginalPVC, true) -} - -// GetKeepOriginalIpAndMac returns the effective value (default: true). -func (o *RestoreOptions) GetKeepOriginalIpAndMac() bool { - return boolDefault(o.KeepOriginalIpAndMac, true) -} - -// parseRestoreOptions deserializes RestoreJob.Spec.Options into RestoreOptions. -// Returns zero-value RestoreOptions if options is nil. -func parseRestoreOptions(opts *runtime.RawExtension) (RestoreOptions, error) { - var ro RestoreOptions - if opts == nil || len(opts.Raw) == 0 { - return ro, nil - } - if err := json.Unmarshal(opts.Raw, &ro); err != nil { - return ro, fmt.Errorf("failed to parse restore options: %w", err) - } - return ro, nil -} - -// restoreTarget holds the resolved target namespace and app identity for a restore operation. -type restoreTarget struct { - Namespace string - AppName string - AppKind string - IsCopy bool // true when targetNamespace differs from backup namespace - IsRenamed bool // true when target app name differs from source app name -} - -// resolveRestoreTarget computes the effective restore target from RestoreJob, Backup, and options. -func resolveRestoreTarget(restoreJob *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, opts RestoreOptions) restoreTarget { - targetNS := backup.Namespace - isCopy := false - if opts.TargetNamespace != "" && opts.TargetNamespace != backup.Namespace { - targetNS = opts.TargetNamespace - isCopy = true - } - targetAppName := backup.Spec.ApplicationRef.Name - if restoreJob.Spec.TargetApplicationRef != nil && restoreJob.Spec.TargetApplicationRef.Name != "" { - targetAppName = restoreJob.Spec.TargetApplicationRef.Name - } - targetAppKind := backup.Spec.ApplicationRef.Kind - if restoreJob.Spec.TargetApplicationRef != nil && restoreJob.Spec.TargetApplicationRef.Kind != "" { - targetAppKind = restoreJob.Spec.TargetApplicationRef.Kind - } - return restoreTarget{ - Namespace: targetNS, - AppName: targetAppName, - AppKind: targetAppKind, - IsCopy: isCopy, - IsRenamed: targetAppName != backup.Spec.ApplicationRef.Name, - } -} - // vmInstanceResources contains VM-specific underlying resources discovered during backup. type vmInstanceResources struct { DataVolumes []backupsv1alpha1.DataVolumeResource `json:"dataVolumes,omitempty"` @@ -576,37 +475,6 @@ func (r *RestoreJobReconciler) reconcileVeleroRestore(ctx context.Context, resto logger := getLogger(ctx) logger.Debug("reconciling Velero strategy restore", "restorejob", restoreJob.Name, "backup", backup.Name) - // Parse restore options from the opaque blob - restoreOpts, err := parseRestoreOptions(restoreJob.Spec.Options) - if err != nil { - return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("invalid restore options: %v", err)) - } - - target := resolveRestoreTarget(restoreJob, backup, restoreOpts) - logger.Debug("resolved restore target", "targetNS", target.Namespace, "targetApp", target.AppName, "isCopy", target.IsCopy) - - // Validate: target namespace must exist for cross-namespace copies - if target.IsCopy { - targetNS := &corev1.Namespace{} - if err := r.Get(ctx, client.ObjectKey{Name: target.Namespace}, targetNS); err != nil { - if errors.IsNotFound(err) { - return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf( - "target namespace %q does not exist; create it before requesting a cross-namespace restore", - target.Namespace)) - } - return ctrl.Result{}, fmt.Errorf("failed to check target namespace %q: %w", target.Namespace, err) - } - } - - // Validate: same-namespace restore with a different app name is not supported - // due to Velero DataUpload always writing to PVCs with the original name. - if !target.IsCopy && target.AppName != backup.Spec.ApplicationRef.Name { - return r.markRestoreJobFailed(ctx, restoreJob, - "restoring to the same namespace with a different application name is not supported "+ - "due to Velero DataUpload limitations: data is always uploaded to PVCs with the original name. "+ - "Use options.targetNamespace to restore into a different namespace") - } - // Step 1: On first reconcile, set startedAt and phase = Running if restoreJob.Status.StartedAt == nil { logger.Debug("setting RestoreJob StartedAt and phase to Running") @@ -657,29 +525,11 @@ func (r *RestoreJobReconciler) reconcileVeleroRestore(ctx context.Context, resto } if len(veleroRestoreList.Items) == 0 { - // For copy restores, enforce failIfTargetExists before touching anything. - // In-place restores are excluded: the source application is expected to exist - // and will be halted/overwritten deliberately. - if target.IsCopy && restoreOpts.GetFailIfTargetExists() { - targetHRName := helmReleaseNameForApp(target.AppKind, target.AppName) - exists, err := r.targetHelmReleaseExists(ctx, target.AppKind, targetHRName, target.Namespace) - if err != nil { - logger.Error(err, "failed to check whether target HelmRelease exists") - return ctrl.Result{}, err - } - if exists { - return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf( - "target application %q already exists in namespace %q; "+ - "set options.failIfTargetExists=false to overwrite", - target.AppName, target.Namespace)) - } - } - // Resolve underlying resources once; prefer Backup status, fall back to Velero annotation. ur := r.resolveUnderlyingResourcesForRestore(ctx, backup, veleroBackupName) - // Pre-restore: graceful shutdown, suspend HRs, rename PVCs (skipped for copy) - ready, result, err := r.prepareForRestore(ctx, restoreJob, backup, ur, target, restoreOpts) + // Pre-restore: graceful shutdown, suspend HRs, rename PVCs + ready, result, err := r.prepareForRestore(ctx, restoreJob, backup, ur) if err != nil { return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("pre-restore preparation failed: %v", err)) } @@ -690,7 +540,7 @@ func (r *RestoreJobReconciler) reconcileVeleroRestore(ctx context.Context, resto // Create Velero Restore logger.Debug("Velero Restore not found, creating new one") - if err := r.createVeleroRestore(ctx, restoreJob, backup, veleroStrategy, veleroBackupName, ur, target, restoreOpts); err != nil { + if err := r.createVeleroRestore(ctx, restoreJob, backup, veleroStrategy, veleroBackupName, ur); err != nil { logger.Error(err, "failed to create Velero Restore") return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("failed to create Velero Restore: %v", err)) } @@ -716,18 +566,6 @@ func (r *RestoreJobReconciler) reconcileVeleroRestore(ctx context.Context, resto // Step 4: On success if phase == "Completed" { - // Post-restore: rename resources if target app name differs from source. - // Velero resource modifiers cannot change metadata.name, so we do it after restore. - if target.IsRenamed { - if err := r.postRestoreRename(ctx, restoreJob, backup, target); err != nil { - r.cleanupResourceModifierConfigMaps(ctx, restoreJob) - return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("post-restore rename failed: %v", err)) - } - } - - // Clean up resource modifier ConfigMaps now that the restore is complete. - r.cleanupResourceModifierConfigMaps(ctx, restoreJob) - now := metav1.Now() restoreJob.Status.CompletedAt = &now restoreJob.Status.Phase = backupsv1alpha1.RestoreJobPhaseSucceeded @@ -741,7 +579,6 @@ func (r *RestoreJobReconciler) reconcileVeleroRestore(ctx context.Context, resto // Step 5: On failure if phase == "Failed" || phase == "PartiallyFailed" { - r.cleanupResourceModifierConfigMaps(ctx, restoreJob) message := fmt.Sprintf("Velero Restore failed with phase: %s", phase) if veleroRestore.Status.FailureReason != "" { message = fmt.Sprintf("%s: %s", message, veleroRestore.Status.FailureReason) @@ -763,7 +600,6 @@ type resourceModifiers struct { type resourceModifierRule struct { Conditions resourceModifierConditions `yaml:"conditions"` MergePatches []mergePatch `yaml:"mergePatches,omitempty"` - Patches []jsonPatch `yaml:"patches,omitempty"` } type resourceModifierConditions struct { @@ -776,12 +612,6 @@ type mergePatch struct { PatchData string `yaml:"patchData"` } -type jsonPatch struct { - Operation string `yaml:"operation"` - Path string `yaml:"path"` - Value string `yaml:"value,omitempty"` -} - // marshalPatchData marshals an arbitrary object to YAML for use as // mergePatch.PatchData in Velero resource modifiers. func marshalPatchData(v interface{}) (string, error) { @@ -797,10 +627,10 @@ func marshalPatchData(v interface{}) (string, error) { // - PVC adoption: always adds cdi.kubevirt.io/allowClaimAdoption=true to all // restored PVCs so CDI can adopt them when a HelmRelease of VMDisk recreates a DV. // - OVN IP/MAC: sets OVN annotations on the VirtualMachine for correct ssh access to restored VM. -func (r *RestoreJobReconciler) createResourceModifiersConfigMap(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, ur *runtime.RawExtension, target restoreTarget, opts RestoreOptions) (*corev1.ConfigMap, error) { +func (r *RestoreJobReconciler) createResourceModifiersConfigMap(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, ur *runtime.RawExtension) (*corev1.ConfigMap, error) { logger := getLogger(ctx) - targetNS := target.Namespace + targetNS := backup.Namespace var rules []resourceModifierRule @@ -824,17 +654,22 @@ func (r *RestoreJobReconciler) createResourceModifiersConfigMap(ctx context.Cont MergePatches: []mergePatch{{PatchData: pvcPatch}}, }) - // For cross-namespace restore: strip Velero's dynamic PV restore selector and - // volumeName from PVCs so the storage provisioner can dynamically provision new PVs. - // Without this, Velero's CSI PVCAction adds spec.selector with a velero.io/dynamic-pv-restore - // label that prevents dynamic provisioning when PVs are not included in the restore. - // Uses merge patch (null values) instead of JSON Patch remove to avoid RFC 6902 - // failures when the fields don't exist on the PVC (e.g. statically provisioned PVCs). - if target.IsCopy { - pvcStripPatch, err := marshalPatchData(map[string]interface{}{ + // OVN IP/MAC annotations on VirtualMachine for correct network identity after restore. + if vmRes := getVMInstanceResources(ur); vmRes != nil && (vmRes.IP != "" || vmRes.MAC != "") { + ovnAnnotations := map[string]string{} + if vmRes.IP != "" { + ovnAnnotations[ovnIPAnnotation] = vmRes.IP + } + if vmRes.MAC != "" { + ovnAnnotations[ovnMACAnnotation] = vmRes.MAC + } + vmPatch, err := marshalPatchData(map[string]interface{}{ "spec": map[string]interface{}{ - "selector": nil, - "volumeName": nil, + "template": map[string]interface{}{ + "metadata": map[string]interface{}{ + "annotations": ovnAnnotations, + }, + }, }, }) if err != nil { @@ -842,49 +677,14 @@ func (r *RestoreJobReconciler) createResourceModifiersConfigMap(ctx context.Cont } rules = append(rules, resourceModifierRule{ Conditions: resourceModifierConditions{ - GroupResource: "persistentvolumeclaims", + GroupResource: "virtualmachines.kubevirt.io", ResourceNameRegex: ".*", Namespaces: []string{targetNS}, }, - MergePatches: []mergePatch{{PatchData: pvcStripPatch}}, + MergePatches: []mergePatch{{PatchData: vmPatch}}, }) } - // OVN IP/MAC annotations on VirtualMachine for correct network identity after restore. - // Only applied when keepOriginalIpAndMac is true; for restore-to-copy the copy - // should get new IP/MAC from the network to avoid conflicts. - if opts.GetKeepOriginalIpAndMac() { - if vmRes := getVMInstanceResources(ur); vmRes != nil && (vmRes.IP != "" || vmRes.MAC != "") { - ovnAnnotations := map[string]string{} - if vmRes.IP != "" { - ovnAnnotations[ovnIPAnnotation] = vmRes.IP - } - if vmRes.MAC != "" { - ovnAnnotations[ovnMACAnnotation] = vmRes.MAC - } - vmPatch, err := marshalPatchData(map[string]interface{}{ - "spec": map[string]interface{}{ - "template": map[string]interface{}{ - "metadata": map[string]interface{}{ - "annotations": ovnAnnotations, - }, - }, - }, - }) - if err != nil { - return nil, err - } - rules = append(rules, resourceModifierRule{ - Conditions: resourceModifierConditions{ - GroupResource: "virtualmachines.kubevirt.io", - ResourceNameRegex: ".*", - Namespaces: []string{targetNS}, - }, - MergePatches: []mergePatch{{PatchData: vmPatch}}, - }) - } - } - rulesYAML, err := yaml.Marshal(resourceModifiers{ Version: "v1", ResourceModifierRules: rules, @@ -960,36 +760,6 @@ var ( dataVolumeGVR = schema.GroupVersionResource{Group: "cdi.kubevirt.io", Version: "v1beta1", Resource: "datavolumes"} ) -// helmReleaseNameForApp returns the HelmRelease name for the given application -// kind and name. Returns empty string for unsupported kinds. -func helmReleaseNameForApp(appKind, appName string) string { - switch appKind { - case vmInstanceKind: - return vmNamePrefix + appName - case vmDiskAppKind: - return vmDiskNamePrefix + appName - default: - return "" - } -} - -// targetHelmReleaseExists returns true when a HelmRelease with the given name -// already exists in namespace. Returns false for unsupported application kinds -// (those where helmReleaseNameForApp returns ""). -func (r *RestoreJobReconciler) targetHelmReleaseExists(ctx context.Context, appKind, hrName, namespace string) (bool, error) { - if hrName == "" { - return false, nil - } - _, err := r.Resource(helmReleaseGVR).Namespace(namespace).Get(ctx, hrName, metav1.GetOptions{}) - if err != nil { - if errors.IsNotFound(err) { - return false, nil - } - return false, err - } - return true, nil -} - // shortHash returns the first 4 hex characters of sha256(input). func shortHash(input string) string { h := sha256.Sum256([]byte(input)) @@ -1008,90 +778,9 @@ func shortHash(input string) string { // (e.g. restore requested when app was already deleted). Each action emits // a Kubernetes Event on the RestoreJob for observability. // -// postRestoreRename renames VMInstance HelmRelease after Velero Restore completes. -// Velero resource modifiers cannot change metadata.name, so this step creates -// a new HelmRelease with the target name and deletes the old one. -// Flux will reconcile the renamed HelmRelease and recreate downstream resources -// (VM, VMI) with the new name. -func (r *RestoreJobReconciler) postRestoreRename(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, target restoreTarget) error { - logger := getLogger(ctx) - sourceAppName := backup.Spec.ApplicationRef.Name - sourceHRName := vmNamePrefix + sourceAppName - targetHRName := vmNamePrefix + target.AppName - - logger.Debug("post-restore rename", "from", sourceHRName, "to", targetHRName, "namespace", target.Namespace) - - // Get the restored HelmRelease with the original name - hrClient := r.Resource(helmReleaseGVR).Namespace(target.Namespace) - oldHR, err := hrClient.Get(ctx, sourceHRName, metav1.GetOptions{}) - if err != nil { - if errors.IsNotFound(err) { - logger.Debug("source HelmRelease not found, skipping rename", "name", sourceHRName) - return nil - } - return fmt.Errorf("failed to get HelmRelease %s: %w", sourceHRName, err) - } - - // Create new HelmRelease with the target name - newHR := oldHR.DeepCopy() - newHR.SetName(targetHRName) - newHR.SetResourceVersion("") - newHR.SetUID("") - newHR.SetCreationTimestamp(metav1.Time{}) - newHR.SetManagedFields(nil) - newHR.SetGeneration(0) - - // Update labels - labels := newHR.GetLabels() - if labels == nil { - labels = map[string]string{} - } - labels[appNameLabel] = target.AppName - labels["app.kubernetes.io/instance"] = targetHRName - labels["helm.toolkit.fluxcd.io/name"] = targetHRName - newHR.SetLabels(labels) - - // Remove Velero restore annotations/labels that tie it to the old restore - annotations := newHR.GetAnnotations() - delete(annotations, "velero.io/restore-name") - newHR.SetAnnotations(annotations) - - // Clear status so Flux reconciles fresh - unstructured.RemoveNestedField(newHR.Object, "status") - - if _, err := hrClient.Create(ctx, newHR, metav1.CreateOptions{}); err != nil { - if errors.IsAlreadyExists(err) { - logger.Debug("target HelmRelease already exists, skipping create", "name", targetHRName) - } else { - return fmt.Errorf("failed to create renamed HelmRelease %s: %w", targetHRName, err) - } - } else { - r.Recorder.Event(restoreJob, corev1.EventTypeNormal, "PostRestoreRename", - fmt.Sprintf("Created renamed HelmRelease %s (from %s)", targetHRName, sourceHRName)) - } - - // Delete the old HelmRelease - if err := hrClient.Delete(ctx, sourceHRName, metav1.DeleteOptions{}); err != nil && !errors.IsNotFound(err) { - return fmt.Errorf("failed to delete old HelmRelease %s: %w", sourceHRName, err) - } - r.Recorder.Event(restoreJob, corev1.EventTypeNormal, "PostRestoreRename", - fmt.Sprintf("Deleted old HelmRelease %s", sourceHRName)) - - logger.Debug("post-restore rename complete", "from", sourceHRName, "to", targetHRName) - return nil -} - // Returns true when preparation is complete and the Velero Restore can be created. // Returns false (with a requeue) when still waiting for VM shutdown. -func (r *RestoreJobReconciler) prepareForRestore(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, ur *runtime.RawExtension, target restoreTarget, opts RestoreOptions) (ready bool, result ctrl.Result, err error) { - // For restore-to-copy, skip all source-app preparation. - // The source application remains untouched; we're restoring a copy into another namespace. - if target.IsCopy { - r.Recorder.Event(restoreJob, corev1.EventTypeNormal, "PrepareForRestore", - "Restore to copy: skipping source application preparation") - return true, ctrl.Result{}, nil - } - +func (r *RestoreJobReconciler) prepareForRestore(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, ur *runtime.RawExtension) (ready bool, result ctrl.Result, err error) { ns := restoreJob.Namespace appName := backup.Spec.ApplicationRef.Name appKind := backup.Spec.ApplicationRef.Kind @@ -1139,8 +828,7 @@ func (r *RestoreJobReconciler) prepareForRestore(ctx context.Context, restoreJob // --- Step 3: Rename PVCs to -orig- --- // Must happen BEFORE deleting DVs: the PVC has an ownerReference to the DV, // so deleting the DV first would cascade-delete the PVC via garbage collection. - // Only when keepOriginalPVC is true (default for in-place restore). - if opts.GetKeepOriginalPVC() && vmRes != nil { + if vmRes != nil { for _, dv := range vmRes.DataVolumes { if err := r.renamePVC(ctx, restoreJob, ns, dv.DataVolumeName, dv.DataVolumeName+origSuffix); err != nil { r.Recorder.Event(restoreJob, corev1.EventTypeWarning, "PrepareForRestore", @@ -1220,14 +908,8 @@ func (r *RestoreJobReconciler) haltVirtualMachine(ctx context.Context, ns, vmNam } // deleteDataVolume deletes a DataVolume so CDI doesn't recreate the PVC after rename. -// Uses Orphan propagation to avoid cascade-deleting the PVC that the DV owns via -// ownerReference. Without this, keepOriginalPVC=false would silently destroy the -// original PVC through garbage collection instead of leaving it for Velero to overwrite. func (r *RestoreJobReconciler) deleteDataVolume(ctx context.Context, ns, name string) error { - orphan := metav1.DeletePropagationOrphan - err := r.Resource(dataVolumeGVR).Namespace(ns).Delete(ctx, name, metav1.DeleteOptions{ - PropagationPolicy: &orphan, - }) + err := r.Resource(dataVolumeGVR).Namespace(ns).Delete(ctx, name, metav1.DeleteOptions{}) if err != nil && !errors.IsNotFound(err) { return err } @@ -1322,16 +1004,10 @@ func (r *RestoreJobReconciler) renamePVC(ctx context.Context, restoreJob *backup } // createVeleroRestore creates a Velero Restore resource. -func (r *RestoreJobReconciler) createVeleroRestore(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, strategy *strategyv1alpha1.Velero, veleroBackupName string, ur *runtime.RawExtension, target restoreTarget, opts RestoreOptions) error { +func (r *RestoreJobReconciler) createVeleroRestore(ctx context.Context, restoreJob *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, strategy *strategyv1alpha1.Velero, veleroBackupName string, ur *runtime.RawExtension) error { logger := getLogger(ctx) - logger.Debug("createVeleroRestore called", "strategy", strategy.Name, "veleroBackupName", veleroBackupName, "targetNS", target.Namespace, "isCopy", target.IsCopy) + logger.Debug("createVeleroRestore called", "strategy", strategy.Name, "veleroBackupName", veleroBackupName) - // For restore template context, always use the source (backup) namespace and app name. - // The strategy template uses includedNamespaces and orLabelSelectors to select - // resources from the backup tarball, which are stored under the source namespace - // and labeled with the source app name. - // Velero's namespaceMapping handles redirecting to the target namespace; - // resource modifiers handle renaming when the target app name differs. templateContext := map[string]interface{}{ "Application": map[string]interface{}{ "metadata": map[string]interface{}{ @@ -1358,16 +1034,6 @@ func (r *RestoreJobReconciler) createVeleroRestore(ctx context.Context, restoreJ // Set the backupName in the spec (required by Velero) veleroRestoreSpec.BackupName = veleroBackupName - // For restore-to-copy, set Velero namespaceMapping to redirect resources - // from the source namespace to the target namespace. - if target.IsCopy { - if veleroRestoreSpec.NamespaceMapping == nil { - veleroRestoreSpec.NamespaceMapping = make(map[string]string) - } - veleroRestoreSpec.NamespaceMapping[backup.Namespace] = target.Namespace - logger.Debug("set namespaceMapping on Velero Restore", "from", backup.Namespace, "to", target.Namespace) - } - // Match backup: add OR selectors for each underlying VMDisk so restore applies the same // scope as the intended backup (see createVeleroBackup). if vmRes := getVMInstanceResources(ur); vmRes != nil { @@ -1385,7 +1051,7 @@ func (r *RestoreJobReconciler) createVeleroRestore(ctx context.Context, restoreJ } // Create resourceModifiers ConfigMap - resourceModifierCM, err := r.createResourceModifiersConfigMap(ctx, restoreJob, backup, ur, target, opts) + resourceModifierCM, err := r.createResourceModifiersConfigMap(ctx, restoreJob, backup, ur) if err != nil { return fmt.Errorf("failed to create resourceModifiers ConfigMap: %w", err) } diff --git a/internal/backupcontroller/velerostrategy_controller_test.go b/internal/backupcontroller/velerostrategy_controller_test.go index 7b085dbf..a7de2076 100644 --- a/internal/backupcontroller/velerostrategy_controller_test.go +++ b/internal/backupcontroller/velerostrategy_controller_test.go @@ -2,13 +2,11 @@ package backupcontroller import ( "context" - "encoding/json" "testing" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" dynamicfake "k8s.io/client-go/dynamic/fake" @@ -208,1006 +206,3 @@ func TestCreateVeleroBackup_TemplateContext(t *testing.T) { t.Errorf("Template context Parameters.backupStorageLocationName not applied correctly. Expected 'default-storage', got '%s'", veleroBackup.Spec.StorageLocation) } } - -func TestResolveRestoreTarget_NoTargetNamespace_InPlace(t *testing.T) { - restoreJob := &backupsv1alpha1.RestoreJob{ - ObjectMeta: metav1.ObjectMeta{ - Name: "restore-test", - Namespace: "tenant-root", - }, - Spec: backupsv1alpha1.RestoreJobSpec{ - BackupRef: corev1.LocalObjectReference{Name: "my-backup"}, - TargetApplicationRef: &corev1.TypedLocalObjectReference{ - APIGroup: stringPtr("apps.cozystack.io"), - Kind: "VMInstance", - Name: "test-vm", - }, - }, - } - - backup := &backupsv1alpha1.Backup{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-backup", - Namespace: "tenant-root", - }, - Spec: backupsv1alpha1.BackupSpec{ - ApplicationRef: corev1.TypedLocalObjectReference{ - APIGroup: stringPtr("apps.cozystack.io"), - Kind: "VMInstance", - Name: "test-vm", - }, - }, - } - - // No targetNamespace in options → in-place restore - opts := RestoreOptions{} - target := resolveRestoreTarget(restoreJob, backup, opts) - - if target.IsCopy { - t.Error("expected IsCopy=false when targetNamespace is omitted, got true") - } - if target.Namespace != "tenant-root" { - t.Errorf("expected namespace 'tenant-root', got '%s'", target.Namespace) - } - if target.AppName != "test-vm" { - t.Errorf("expected appName 'test-vm', got '%s'", target.AppName) - } - if target.AppKind != "VMInstance" { - t.Errorf("expected appKind 'VMInstance', got '%s'", target.AppKind) - } -} - -func TestResolveRestoreTarget_SameTargetNamespace_InPlace(t *testing.T) { - restoreJob := &backupsv1alpha1.RestoreJob{ - ObjectMeta: metav1.ObjectMeta{ - Name: "restore-test", - Namespace: "tenant-root", - }, - Spec: backupsv1alpha1.RestoreJobSpec{ - BackupRef: corev1.LocalObjectReference{Name: "my-backup"}, - }, - } - - backup := &backupsv1alpha1.Backup{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-backup", - Namespace: "tenant-root", - }, - Spec: backupsv1alpha1.BackupSpec{ - ApplicationRef: corev1.TypedLocalObjectReference{ - APIGroup: stringPtr("apps.cozystack.io"), - Kind: "VMInstance", - Name: "test-vm", - }, - }, - } - - // targetNamespace equals backup namespace → still in-place - opts := RestoreOptions{ - CommonRestoreOptions: CommonRestoreOptions{ - TargetNamespace: "tenant-root", - }, - } - target := resolveRestoreTarget(restoreJob, backup, opts) - - if target.IsCopy { - t.Error("expected IsCopy=false when targetNamespace equals backup namespace, got true") - } - if target.Namespace != "tenant-root" { - t.Errorf("expected namespace 'tenant-root', got '%s'", target.Namespace) - } -} - -func TestResolveRestoreTarget_DifferentTargetNamespace_Copy(t *testing.T) { - restoreJob := &backupsv1alpha1.RestoreJob{ - ObjectMeta: metav1.ObjectMeta{ - Name: "restore-test", - Namespace: "tenant-root", - }, - Spec: backupsv1alpha1.RestoreJobSpec{ - BackupRef: corev1.LocalObjectReference{Name: "my-backup"}, - }, - } - - backup := &backupsv1alpha1.Backup{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-backup", - Namespace: "tenant-root", - }, - Spec: backupsv1alpha1.BackupSpec{ - ApplicationRef: corev1.TypedLocalObjectReference{ - APIGroup: stringPtr("apps.cozystack.io"), - Kind: "VMInstance", - Name: "test-vm", - }, - }, - } - - opts := RestoreOptions{ - CommonRestoreOptions: CommonRestoreOptions{ - TargetNamespace: "tenant-copy", - }, - } - target := resolveRestoreTarget(restoreJob, backup, opts) - - if !target.IsCopy { - t.Error("expected IsCopy=true when targetNamespace differs from backup namespace, got false") - } - if target.Namespace != "tenant-copy" { - t.Errorf("expected namespace 'tenant-copy', got '%s'", target.Namespace) - } -} - -// newTestRestoreJobReconcilerWithDynamic builds a RestoreJobReconciler with -// both static and dynamic fake clients. Use dynamicObjects to pre-populate -// unstructured resources (e.g. HelmReleases). -func newTestRestoreJobReconcilerWithDynamic(t *testing.T, dynamicObjects []runtime.Object, objects ...client.Object) *RestoreJobReconciler { - t.Helper() - testScheme := runtime.NewScheme() - _ = scheme.AddToScheme(testScheme) - _ = backupsv1alpha1.AddToScheme(testScheme) - _ = velerov1.AddToScheme(testScheme) - - fakeClient := clientfake.NewClientBuilder(). - WithScheme(testScheme). - WithObjects(objects...). - Build() - - dynamicClient := dynamicfake.NewSimpleDynamicClient(testScheme, dynamicObjects...) - - mapping := &meta.RESTMapping{ - Resource: schema.GroupVersionResource{Group: "", Version: "v1", Resource: "pods"}, - GroupVersionKind: schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"}, - Scope: meta.RESTScopeNamespace, - } - - return &RestoreJobReconciler{ - Client: fakeClient, - Interface: dynamicClient, - RESTMapper: &mockRESTMapper{mapping: mapping}, - Scheme: testScheme, - Recorder: record.NewFakeRecorder(100), - } -} - -// newTestRestoreJobReconciler builds a RestoreJobReconciler with fake clients for testing. -func newTestRestoreJobReconciler(t *testing.T, objects ...client.Object) *RestoreJobReconciler { - t.Helper() - testScheme := runtime.NewScheme() - _ = scheme.AddToScheme(testScheme) - _ = backupsv1alpha1.AddToScheme(testScheme) - _ = velerov1.AddToScheme(testScheme) - - fakeClient := clientfake.NewClientBuilder(). - WithScheme(testScheme). - WithObjects(objects...). - Build() - - dynamicClient := dynamicfake.NewSimpleDynamicClient(testScheme) - - mapping := &meta.RESTMapping{ - Resource: schema.GroupVersionResource{Group: "", Version: "v1", Resource: "pods"}, - GroupVersionKind: schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"}, - Scope: meta.RESTScopeNamespace, - } - - return &RestoreJobReconciler{ - Client: fakeClient, - Interface: dynamicClient, - RESTMapper: &mockRESTMapper{mapping: mapping}, - Scheme: testScheme, - Recorder: record.NewFakeRecorder(100), - } -} - -func TestPrepareForRestore_KeepOriginalPVCFalse_SkipsRename(t *testing.T) { - ns := "tenant-root" - - // Create a PVC that would be renamed if keepOriginalPVC were true - pvc := &corev1.PersistentVolumeClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: "vm-disk-ubuntu-source", - Namespace: ns, - }, - Spec: corev1.PersistentVolumeClaimSpec{}, - } - - restoreJob := &backupsv1alpha1.RestoreJob{ - ObjectMeta: metav1.ObjectMeta{ - Name: "restore-test", - Namespace: ns, - }, - Spec: backupsv1alpha1.RestoreJobSpec{ - BackupRef: corev1.LocalObjectReference{Name: "my-backup"}, - }, - } - - backup := &backupsv1alpha1.Backup{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-backup", - Namespace: ns, - }, - Spec: backupsv1alpha1.BackupSpec{ - ApplicationRef: corev1.TypedLocalObjectReference{ - APIGroup: stringPtr("apps.cozystack.io"), - Kind: "VMInstance", - Name: "test-vm", - }, - }, - } - - // underlyingResources with a DataVolume referencing the PVC - urData := vmInstanceResources{ - DataVolumes: []backupsv1alpha1.DataVolumeResource{ - {DataVolumeName: "vm-disk-ubuntu-source", ApplicationName: "ubuntu-source"}, - }, - } - urRaw, _ := json.Marshal(urData) - ur := &runtime.RawExtension{Raw: urRaw} - - target := restoreTarget{ - Namespace: ns, - AppName: "test-vm", - AppKind: "VMInstance", - IsCopy: false, - } - - // keepOriginalPVC = false → PVCs should NOT be renamed - opts := RestoreOptions{ - KeepOriginalPVC: boolPtr(false), - } - - reconciler := newTestRestoreJobReconciler(t, pvc, restoreJob, backup) - - ctx := context.Background() - ready, _, err := reconciler.prepareForRestore(ctx, restoreJob, backup, ur, target, opts) - if err != nil { - t.Fatalf("prepareForRestore() error = %v", err) - } - if !ready { - t.Fatal("expected ready=true, got false") - } - - // Verify the original PVC still exists with its original name (not renamed) - origPVC := &corev1.PersistentVolumeClaim{} - err = reconciler.Get(ctx, client.ObjectKey{Namespace: ns, Name: "vm-disk-ubuntu-source"}, origPVC) - if err != nil { - t.Errorf("original PVC should still exist with original name when keepOriginalPVC=false, got error: %v", err) - } - - // Verify no -orig PVC was created - origSuffix := "-orig-" + shortHash(restoreJob.Name) - renamedPVC := &corev1.PersistentVolumeClaim{} - err = reconciler.Get(ctx, client.ObjectKey{Namespace: ns, Name: "vm-disk-ubuntu-source" + origSuffix}, renamedPVC) - if err == nil { - t.Error("PVC should NOT have been renamed when keepOriginalPVC=false, but found renamed PVC") - } -} - -func TestResolveRestoreTarget_VMDisk_CommonRestoreOptions(t *testing.T) { - tests := []struct { - name string - targetNS string - wantIsCopy bool - wantNamespace string - }{ - { - name: "VMDisk in-place restore when targetNamespace is omitted", - targetNS: "", - wantIsCopy: false, - wantNamespace: "tenant-root", - }, - { - name: "VMDisk cross-namespace restore when targetNamespace differs", - targetNS: "tenant-copy", - wantIsCopy: true, - wantNamespace: "tenant-copy", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - restoreJob := &backupsv1alpha1.RestoreJob{ - ObjectMeta: metav1.ObjectMeta{ - Name: "restore-vmdisk", - Namespace: "tenant-root", - }, - Spec: backupsv1alpha1.RestoreJobSpec{ - BackupRef: corev1.LocalObjectReference{Name: "disk-backup"}, - TargetApplicationRef: &corev1.TypedLocalObjectReference{ - APIGroup: stringPtr("apps.cozystack.io"), - Kind: "VMDisk", - Name: "ubuntu-source", - }, - }, - } - - backup := &backupsv1alpha1.Backup{ - ObjectMeta: metav1.ObjectMeta{ - Name: "disk-backup", - Namespace: "tenant-root", - }, - Spec: backupsv1alpha1.BackupSpec{ - ApplicationRef: corev1.TypedLocalObjectReference{ - APIGroup: stringPtr("apps.cozystack.io"), - Kind: "VMDisk", - Name: "ubuntu-source", - }, - }, - } - - // VMDisk uses only CommonRestoreOptions (no VMI-specific fields) - opts := RestoreOptions{ - CommonRestoreOptions: CommonRestoreOptions{ - TargetNamespace: tt.targetNS, - }, - } - - target := resolveRestoreTarget(restoreJob, backup, opts) - - if target.IsCopy != tt.wantIsCopy { - t.Errorf("IsCopy = %v, want %v", target.IsCopy, tt.wantIsCopy) - } - if target.Namespace != tt.wantNamespace { - t.Errorf("Namespace = %q, want %q", target.Namespace, tt.wantNamespace) - } - if target.AppKind != "VMDisk" { - t.Errorf("AppKind = %q, want 'VMDisk'", target.AppKind) - } - if target.AppName != "ubuntu-source" { - t.Errorf("AppName = %q, want 'ubuntu-source'", target.AppName) - } - }) - } -} - -func TestParseRestoreOptions_VMDiskOnlyCommonFields(t *testing.T) { - // Simulate a VMDisk restore where only CommonRestoreOptions fields are set - raw, _ := json.Marshal(map[string]interface{}{ - "targetNamespace": "tenant-copy", - "failIfTargetExists": true, - }) - ext := &runtime.RawExtension{Raw: raw} - - opts, err := parseRestoreOptions(ext) - if err != nil { - t.Fatalf("parseRestoreOptions() error = %v", err) - } - - if opts.TargetNamespace != "tenant-copy" { - t.Errorf("TargetNamespace = %q, want 'tenant-copy'", opts.TargetNamespace) - } - if !opts.GetFailIfTargetExists() { - t.Error("GetFailIfTargetExists() = false, want true") - } - // VMI-specific fields should be nil (not set by VMDisk options) - if opts.KeepOriginalPVC != nil { - t.Errorf("KeepOriginalPVC should be nil for VMDisk options, got %v", *opts.KeepOriginalPVC) - } - if opts.KeepOriginalIpAndMac != nil { - t.Errorf("KeepOriginalIpAndMac should be nil for VMDisk options, got %v", *opts.KeepOriginalIpAndMac) - } -} - -func TestRestoreOptions_Defaults(t *testing.T) { - t.Run("nil options default all bools to true", func(t *testing.T) { - opts, err := parseRestoreOptions(nil) - if err != nil { - t.Fatalf("parseRestoreOptions(nil) error = %v", err) - } - - if opts.TargetNamespace != "" { - t.Errorf("TargetNamespace default should be empty, got %q", opts.TargetNamespace) - } - if !opts.GetFailIfTargetExists() { - t.Error("GetFailIfTargetExists() should default to true") - } - if !opts.GetKeepOriginalPVC() { - t.Error("GetKeepOriginalPVC() should default to true") - } - if !opts.GetKeepOriginalIpAndMac() { - t.Error("GetKeepOriginalIpAndMac() should default to true") - } - }) - - t.Run("empty JSON defaults all bools to true", func(t *testing.T) { - raw, _ := json.Marshal(map[string]interface{}{}) - opts, err := parseRestoreOptions(&runtime.RawExtension{Raw: raw}) - if err != nil { - t.Fatalf("parseRestoreOptions({}) error = %v", err) - } - - if !opts.GetFailIfTargetExists() { - t.Error("GetFailIfTargetExists() should default to true") - } - if !opts.GetKeepOriginalPVC() { - t.Error("GetKeepOriginalPVC() should default to true") - } - if !opts.GetKeepOriginalIpAndMac() { - t.Error("GetKeepOriginalIpAndMac() should default to true") - } - }) - - t.Run("explicit false overrides defaults", func(t *testing.T) { - raw, _ := json.Marshal(map[string]interface{}{ - "failIfTargetExists": false, - "keepOriginalPVC": false, - "keepOriginalIpAndMac": false, - }) - opts, err := parseRestoreOptions(&runtime.RawExtension{Raw: raw}) - if err != nil { - t.Fatalf("parseRestoreOptions() error = %v", err) - } - - if opts.GetFailIfTargetExists() { - t.Error("GetFailIfTargetExists() should be false when explicitly set") - } - if opts.GetKeepOriginalPVC() { - t.Error("GetKeepOriginalPVC() should be false when explicitly set") - } - if opts.GetKeepOriginalIpAndMac() { - t.Error("GetKeepOriginalIpAndMac() should be false when explicitly set") - } - }) -} - -func TestResolveRestoreTarget_IsRenamed(t *testing.T) { - backup := &backupsv1alpha1.Backup{ - ObjectMeta: metav1.ObjectMeta{Name: "bk", Namespace: "tenant-root"}, - Spec: backupsv1alpha1.BackupSpec{ - ApplicationRef: corev1.TypedLocalObjectReference{ - APIGroup: stringPtr("apps.cozystack.io"), - Kind: "VMInstance", - Name: "test-alpine", - }, - }, - } - - t.Run("same name is not renamed", func(t *testing.T) { - rj := &backupsv1alpha1.RestoreJob{ - ObjectMeta: metav1.ObjectMeta{Name: "rj", Namespace: "tenant-root"}, - Spec: backupsv1alpha1.RestoreJobSpec{ - BackupRef: corev1.LocalObjectReference{Name: "bk"}, - TargetApplicationRef: &corev1.TypedLocalObjectReference{ - APIGroup: stringPtr("apps.cozystack.io"), - Kind: "VMInstance", - Name: "test-alpine", - }, - }, - } - target := resolveRestoreTarget(rj, backup, RestoreOptions{ - CommonRestoreOptions: CommonRestoreOptions{TargetNamespace: "tenant-foo"}, - }) - if target.IsRenamed { - t.Error("IsRenamed should be false when names match") - } - }) - - t.Run("different name is renamed", func(t *testing.T) { - rj := &backupsv1alpha1.RestoreJob{ - ObjectMeta: metav1.ObjectMeta{Name: "rj", Namespace: "tenant-root"}, - Spec: backupsv1alpha1.RestoreJobSpec{ - BackupRef: corev1.LocalObjectReference{Name: "bk"}, - TargetApplicationRef: &corev1.TypedLocalObjectReference{ - APIGroup: stringPtr("apps.cozystack.io"), - Kind: "VMInstance", - Name: "test-new", - }, - }, - } - target := resolveRestoreTarget(rj, backup, RestoreOptions{ - CommonRestoreOptions: CommonRestoreOptions{TargetNamespace: "tenant-foo"}, - }) - if !target.IsRenamed { - t.Error("IsRenamed should be true when names differ") - } - if target.AppName != "test-new" { - t.Errorf("AppName = %q, want 'test-new'", target.AppName) - } - if !target.IsCopy { - t.Error("IsCopy should be true") - } - }) - - t.Run("no targetApplicationRef is not renamed", func(t *testing.T) { - rj := &backupsv1alpha1.RestoreJob{ - ObjectMeta: metav1.ObjectMeta{Name: "rj", Namespace: "tenant-root"}, - Spec: backupsv1alpha1.RestoreJobSpec{ - BackupRef: corev1.LocalObjectReference{Name: "bk"}, - }, - } - target := resolveRestoreTarget(rj, backup, RestoreOptions{ - CommonRestoreOptions: CommonRestoreOptions{TargetNamespace: "tenant-foo"}, - }) - if target.IsRenamed { - t.Error("IsRenamed should be false when targetApplicationRef is nil") - } - if target.AppName != "test-alpine" { - t.Errorf("AppName = %q, want 'test-alpine'", target.AppName) - } - }) -} - -// makeUnstructuredHelmRelease creates an unstructured HelmRelease for dynamic client tests. -func makeUnstructuredHelmRelease(name, namespace string, labels map[string]string) *unstructured.Unstructured { - hr := &unstructured.Unstructured{} - hr.SetAPIVersion("helm.toolkit.fluxcd.io/v2") - hr.SetKind("HelmRelease") - hr.SetName(name) - hr.SetNamespace(namespace) - hr.SetLabels(labels) - return hr -} - -func TestPostRestoreRename_RenamesHelmRelease(t *testing.T) { - ns := "tenant-foo" - sourceHR := makeUnstructuredHelmRelease("vm-instance-test-alpine", ns, map[string]string{ - appNameLabel: "test-alpine", - "app.kubernetes.io/instance": "vm-instance-test-alpine", - "helm.toolkit.fluxcd.io/name": "vm-instance-test-alpine", - }) - - restoreJob := &backupsv1alpha1.RestoreJob{ - ObjectMeta: metav1.ObjectMeta{Name: "rj-rename", Namespace: "tenant-root"}, - Spec: backupsv1alpha1.RestoreJobSpec{ - BackupRef: corev1.LocalObjectReference{Name: "bk"}, - }, - } - backup := &backupsv1alpha1.Backup{ - ObjectMeta: metav1.ObjectMeta{Name: "bk", Namespace: "tenant-root"}, - Spec: backupsv1alpha1.BackupSpec{ - ApplicationRef: corev1.TypedLocalObjectReference{ - APIGroup: stringPtr("apps.cozystack.io"), - Kind: "VMInstance", - Name: "test-alpine", - }, - }, - } - target := restoreTarget{ - Namespace: ns, - AppName: "test-new", - AppKind: "VMInstance", - IsCopy: true, - IsRenamed: true, - } - - reconciler := newTestRestoreJobReconcilerWithDynamic(t, []runtime.Object{sourceHR}, restoreJob, backup) - ctx := context.Background() - - err := reconciler.postRestoreRename(ctx, restoreJob, backup, target) - if err != nil { - t.Fatalf("postRestoreRename() error = %v", err) - } - - hrClient := reconciler.Resource(helmReleaseGVR).Namespace(ns) - - // New HelmRelease should exist with target name - newHR, err := hrClient.Get(ctx, "vm-instance-test-new", metav1.GetOptions{}) - if err != nil { - t.Fatalf("new HelmRelease vm-instance-test-new not found: %v", err) - } - - // Check labels were updated - labels := newHR.GetLabels() - if labels[appNameLabel] != "test-new" { - t.Errorf("label %s = %q, want 'test-new'", appNameLabel, labels[appNameLabel]) - } - if labels["app.kubernetes.io/instance"] != "vm-instance-test-new" { - t.Errorf("label app.kubernetes.io/instance = %q, want 'vm-instance-test-new'", labels["app.kubernetes.io/instance"]) - } - if labels["helm.toolkit.fluxcd.io/name"] != "vm-instance-test-new" { - t.Errorf("label helm.toolkit.fluxcd.io/name = %q, want 'vm-instance-test-new'", labels["helm.toolkit.fluxcd.io/name"]) - } - - // Old HelmRelease should be deleted - _, err = hrClient.Get(ctx, "vm-instance-test-alpine", metav1.GetOptions{}) - if err == nil { - t.Error("old HelmRelease vm-instance-test-alpine should have been deleted") - } -} - -func TestPostRestoreRename_SkipsWhenSourceNotFound(t *testing.T) { - restoreJob := &backupsv1alpha1.RestoreJob{ - ObjectMeta: metav1.ObjectMeta{Name: "rj-rename", Namespace: "tenant-root"}, - Spec: backupsv1alpha1.RestoreJobSpec{ - BackupRef: corev1.LocalObjectReference{Name: "bk"}, - }, - } - backup := &backupsv1alpha1.Backup{ - ObjectMeta: metav1.ObjectMeta{Name: "bk", Namespace: "tenant-root"}, - Spec: backupsv1alpha1.BackupSpec{ - ApplicationRef: corev1.TypedLocalObjectReference{ - APIGroup: stringPtr("apps.cozystack.io"), - Kind: "VMInstance", - Name: "test-alpine", - }, - }, - } - target := restoreTarget{ - Namespace: "tenant-foo", - AppName: "test-new", - AppKind: "VMInstance", - IsCopy: true, - IsRenamed: true, - } - - // No HelmRelease in dynamic client — should skip gracefully - reconciler := newTestRestoreJobReconcilerWithDynamic(t, nil, restoreJob, backup) - ctx := context.Background() - - err := reconciler.postRestoreRename(ctx, restoreJob, backup, target) - if err != nil { - t.Fatalf("postRestoreRename() should skip gracefully when source HR not found, got error: %v", err) - } -} - -func TestPostRestoreRename_IdempotentWhenTargetExists(t *testing.T) { - ns := "tenant-foo" - sourceHR := makeUnstructuredHelmRelease("vm-instance-test-alpine", ns, map[string]string{ - appNameLabel: "test-alpine", - }) - // Target already exists (e.g. from a previous reconcile) - targetHR := makeUnstructuredHelmRelease("vm-instance-test-new", ns, map[string]string{ - appNameLabel: "test-new", - }) - - restoreJob := &backupsv1alpha1.RestoreJob{ - ObjectMeta: metav1.ObjectMeta{Name: "rj-rename", Namespace: "tenant-root"}, - Spec: backupsv1alpha1.RestoreJobSpec{ - BackupRef: corev1.LocalObjectReference{Name: "bk"}, - }, - } - backup := &backupsv1alpha1.Backup{ - ObjectMeta: metav1.ObjectMeta{Name: "bk", Namespace: "tenant-root"}, - Spec: backupsv1alpha1.BackupSpec{ - ApplicationRef: corev1.TypedLocalObjectReference{ - APIGroup: stringPtr("apps.cozystack.io"), - Kind: "VMInstance", - Name: "test-alpine", - }, - }, - } - target := restoreTarget{ - Namespace: ns, - AppName: "test-new", - AppKind: "VMInstance", - IsCopy: true, - IsRenamed: true, - } - - reconciler := newTestRestoreJobReconcilerWithDynamic(t, []runtime.Object{sourceHR, targetHR}, restoreJob, backup) - ctx := context.Background() - - err := reconciler.postRestoreRename(ctx, restoreJob, backup, target) - if err != nil { - t.Fatalf("postRestoreRename() should be idempotent, got error: %v", err) - } - - hrClient := reconciler.Resource(helmReleaseGVR).Namespace(ns) - - // Target should still exist - _, err = hrClient.Get(ctx, "vm-instance-test-new", metav1.GetOptions{}) - if err != nil { - t.Fatalf("target HelmRelease should exist: %v", err) - } - - // Source should be deleted - _, err = hrClient.Get(ctx, "vm-instance-test-alpine", metav1.GetOptions{}) - if err == nil { - t.Error("source HelmRelease should have been deleted") - } -} - -// --- failIfTargetExists enforcement tests --- - -func TestTargetHelmReleaseExists_ReturnsTrueWhenPresent(t *testing.T) { - ns := "tenant-copy" - hr := makeUnstructuredHelmRelease("vm-instance-test", ns, nil) - - restoreJob := &backupsv1alpha1.RestoreJob{ - ObjectMeta: metav1.ObjectMeta{Name: "rj", Namespace: "tenant-root"}, - } - backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Name: "bk", Namespace: "tenant-root"}} - - reconciler := newTestRestoreJobReconcilerWithDynamic(t, []runtime.Object{hr}, restoreJob, backup) - ctx := context.Background() - - exists, err := reconciler.targetHelmReleaseExists(ctx, vmInstanceKind, "vm-instance-test", ns) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if !exists { - t.Error("expected exists=true when HelmRelease is present") - } -} - -func TestTargetHelmReleaseExists_ReturnsFalseWhenAbsent(t *testing.T) { - ns := "tenant-copy" - restoreJob := &backupsv1alpha1.RestoreJob{ - ObjectMeta: metav1.ObjectMeta{Name: "rj", Namespace: "tenant-root"}, - } - backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Name: "bk", Namespace: "tenant-root"}} - - reconciler := newTestRestoreJobReconcilerWithDynamic(t, nil, restoreJob, backup) - ctx := context.Background() - - exists, err := reconciler.targetHelmReleaseExists(ctx, vmInstanceKind, "vm-instance-test", ns) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if exists { - t.Error("expected exists=false when HelmRelease is absent") - } -} - -func TestTargetHelmReleaseExists_UnsupportedKindViaHelper(t *testing.T) { - ns := "tenant-copy" - // Even though a HR exists, unsupported kinds produce an empty hrName - // from helmReleaseNameForApp, which makes targetHelmReleaseExists return false. - hr := makeUnstructuredHelmRelease("vm-instance-test", ns, nil) - - restoreJob := &backupsv1alpha1.RestoreJob{ - ObjectMeta: metav1.ObjectMeta{Name: "rj", Namespace: "tenant-root"}, - } - backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Name: "bk", Namespace: "tenant-root"}} - - reconciler := newTestRestoreJobReconcilerWithDynamic(t, []runtime.Object{hr}, restoreJob, backup) - ctx := context.Background() - - // Unsupported kinds get empty hrName from the helper - hrName := helmReleaseNameForApp("MariaDB", "mydb") - exists, err := reconciler.targetHelmReleaseExists(ctx, "MariaDB", hrName, ns) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if exists { - t.Error("expected exists=false for unsupported kind (empty hrName)") - } -} - -// --- createResourceModifiersConfigMap tests --- - -func makeTestBackup(ns, appName, appKind string) *backupsv1alpha1.Backup { - return &backupsv1alpha1.Backup{ - ObjectMeta: metav1.ObjectMeta{Name: "bk", Namespace: ns}, - Spec: backupsv1alpha1.BackupSpec{ - ApplicationRef: corev1.TypedLocalObjectReference{ - APIGroup: stringPtr("apps.cozystack.io"), - Kind: appKind, - Name: appName, - }, - }, - } -} - -func makeTestRestoreJob(ns, name string) *backupsv1alpha1.RestoreJob { - return &backupsv1alpha1.RestoreJob{ - ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}, - Spec: backupsv1alpha1.RestoreJobSpec{ - BackupRef: corev1.LocalObjectReference{Name: "bk"}, - }, - } -} - -func makeVMInstanceUR(ip, mac string, dvs []backupsv1alpha1.DataVolumeResource) *runtime.RawExtension { - data := vmInstanceResources{IP: ip, MAC: mac, DataVolumes: dvs} - raw, _ := json.Marshal(data) - return &runtime.RawExtension{Raw: raw} -} - -func TestCreateResourceModifiersConfigMap_InPlace_WithOVN(t *testing.T) { - ns := "tenant-root" - restoreJob := makeTestRestoreJob(ns, "rj-inplace") - backup := makeTestBackup(ns, "test-vm", "VMInstance") - - ur := makeVMInstanceUR("10.0.0.5", "aa:bb:cc:dd:ee:ff", nil) - target := restoreTarget{Namespace: ns, AppName: "test-vm", AppKind: "VMInstance", IsCopy: false} - opts := RestoreOptions{} // defaults: keepOriginalIpAndMac=true - - reconciler := newTestRestoreJobReconciler(t, restoreJob, backup) - ctx := context.Background() - - cm, err := reconciler.createResourceModifiersConfigMap(ctx, restoreJob, backup, ur, target, opts) - if err != nil { - t.Fatalf("createResourceModifiersConfigMap() error = %v", err) - } - if cm == nil { - t.Fatal("expected non-nil ConfigMap") - } - - rulesYAML, ok := cm.Data["resource-modifier-rules.yaml"] - if !ok { - t.Fatal("ConfigMap missing resource-modifier-rules.yaml key") - } - - // Should contain PVC adoption annotation - if !containsString(rulesYAML, cdiAllowClaimAdoption) { - t.Error("expected PVC adoption annotation in rules") - } - // Should contain OVN IP annotation (keepOriginalIpAndMac defaults to true) - if !containsString(rulesYAML, ovnIPAnnotation) { - t.Error("expected OVN IP annotation in rules for in-place restore with IP") - } - // Should NOT contain selector removal (in-place, not copy) - if containsString(rulesYAML, "/spec/selector") { - t.Error("selector removal rule should not be present for in-place restore") - } -} - -func TestCreateResourceModifiersConfigMap_Copy_NoOVN(t *testing.T) { - sourceNS := "tenant-root" - targetNS := "tenant-copy" - restoreJob := makeTestRestoreJob(sourceNS, "rj-copy") - backup := makeTestBackup(sourceNS, "test-vm", "VMInstance") - - // Copy restore: keepOriginalIpAndMac=false, no OVN annotations on copy - opts := RestoreOptions{ - KeepOriginalIpAndMac: boolPtr(false), - } - ur := makeVMInstanceUR("10.0.0.5", "aa:bb:cc:dd:ee:ff", nil) - target := restoreTarget{Namespace: targetNS, AppName: "test-vm", AppKind: "VMInstance", IsCopy: true} - - reconciler := newTestRestoreJobReconciler(t, restoreJob, backup) - ctx := context.Background() - - cm, err := reconciler.createResourceModifiersConfigMap(ctx, restoreJob, backup, ur, target, opts) - if err != nil { - t.Fatalf("createResourceModifiersConfigMap() error = %v", err) - } - - rulesYAML := cm.Data["resource-modifier-rules.yaml"] - - // Should contain PVC adoption annotation - if !containsString(rulesYAML, cdiAllowClaimAdoption) { - t.Error("expected PVC adoption annotation in rules") - } - // Should contain merge patch nulling out selector and volumeName (copy restore) - if !containsString(rulesYAML, "selector: null") { - t.Error("expected selector: null merge patch for copy restore") - } - if !containsString(rulesYAML, "volumeName: null") { - t.Error("expected volumeName: null merge patch for copy restore") - } - // Should NOT contain OVN annotations (keepOriginalIpAndMac=false) - if containsString(rulesYAML, ovnIPAnnotation) { - t.Error("OVN IP annotation should not be present when keepOriginalIpAndMac=false") - } -} - -func TestCreateResourceModifiersConfigMap_AlreadyExists_IsIdempotent(t *testing.T) { - ns := "tenant-root" - restoreJob := makeTestRestoreJob(ns, "rj-idem") - backup := makeTestBackup(ns, "test-vm", "VMInstance") - - target := restoreTarget{Namespace: ns, AppName: "test-vm", AppKind: "VMInstance", IsCopy: false} - opts := RestoreOptions{} - ur := makeVMInstanceUR("", "", nil) - - reconciler := newTestRestoreJobReconciler(t, restoreJob, backup) - ctx := context.Background() - - // First call creates the ConfigMap. - cm1, err := reconciler.createResourceModifiersConfigMap(ctx, restoreJob, backup, ur, target, opts) - if err != nil { - t.Fatalf("first call error = %v", err) - } - - // Second call must not error (AlreadyExists → update path). - cm2, err := reconciler.createResourceModifiersConfigMap(ctx, restoreJob, backup, ur, target, opts) - if err != nil { - t.Fatalf("second call error = %v", err) - } - if cm1.Name != cm2.Name { - t.Errorf("ConfigMap name changed: %q → %q", cm1.Name, cm2.Name) - } -} - -// --- helmReleaseNameForApp tests --- - -func TestHelmReleaseNameForApp(t *testing.T) { - tests := []struct { - kind, name, want string - }{ - {vmInstanceKind, "test-alpine", "vm-instance-test-alpine"}, - {vmDiskAppKind, "ubuntu-source", "vm-disk-ubuntu-source"}, - {"MariaDB", "mydb", ""}, - } - for _, tt := range tests { - t.Run(tt.kind+"/"+tt.name, func(t *testing.T) { - got := helmReleaseNameForApp(tt.kind, tt.name) - if got != tt.want { - t.Errorf("helmReleaseNameForApp(%q, %q) = %q, want %q", tt.kind, tt.name, got, tt.want) - } - }) - } -} - -func TestTargetHelmReleaseExists_VMDisk(t *testing.T) { - ns := "tenant-copy" - hr := makeUnstructuredHelmRelease("vm-disk-ubuntu-source", ns, nil) - - restoreJob := &backupsv1alpha1.RestoreJob{ - ObjectMeta: metav1.ObjectMeta{Name: "rj", Namespace: "tenant-root"}, - } - backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Name: "bk", Namespace: "tenant-root"}} - - reconciler := newTestRestoreJobReconcilerWithDynamic(t, []runtime.Object{hr}, restoreJob, backup) - ctx := context.Background() - - hrName := helmReleaseNameForApp(vmDiskAppKind, "ubuntu-source") - exists, err := reconciler.targetHelmReleaseExists(ctx, vmDiskAppKind, hrName, ns) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if !exists { - t.Error("expected exists=true for VMDisk HelmRelease") - } -} - -func TestTargetHelmReleaseExists_UnsupportedKind_ReturnsFalse(t *testing.T) { - restoreJob := &backupsv1alpha1.RestoreJob{ - ObjectMeta: metav1.ObjectMeta{Name: "rj", Namespace: "tenant-root"}, - } - backup := &backupsv1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Name: "bk", Namespace: "tenant-root"}} - - reconciler := newTestRestoreJobReconcilerWithDynamic(t, nil, restoreJob, backup) - ctx := context.Background() - - // Unsupported kinds produce empty hrName which returns false - hrName := helmReleaseNameForApp("MariaDB", "mydb") - exists, err := reconciler.targetHelmReleaseExists(ctx, "MariaDB", hrName, "tenant-copy") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if exists { - t.Error("expected exists=false for unsupported kind") - } -} - -// --- ResourceModifier merge patch test for copy restore --- - -func TestCreateResourceModifiersConfigMap_Copy_UsesMergePatchForPVC(t *testing.T) { - sourceNS := "tenant-root" - targetNS := "tenant-copy" - restoreJob := makeTestRestoreJob(sourceNS, "rj-copy-merge") - backup := makeTestBackup(sourceNS, "test-vm", "VMInstance") - - opts := RestoreOptions{KeepOriginalIpAndMac: boolPtr(false)} - ur := makeVMInstanceUR("", "", nil) - target := restoreTarget{Namespace: targetNS, AppName: "test-vm", AppKind: "VMInstance", IsCopy: true} - - reconciler := newTestRestoreJobReconciler(t, restoreJob, backup) - ctx := context.Background() - - cm, err := reconciler.createResourceModifiersConfigMap(ctx, restoreJob, backup, ur, target, opts) - if err != nil { - t.Fatalf("createResourceModifiersConfigMap() error = %v", err) - } - - rulesYAML := cm.Data["resource-modifier-rules.yaml"] - - // Should use merge patch (patchData with null), NOT JSON patch (operation: remove) - if containsString(rulesYAML, "operation: remove") { - t.Error("should use merge patch instead of JSON Patch remove for PVC fields") - } - // The merge patch should null out selector and volumeName - if !containsString(rulesYAML, "selector: null") { - t.Error("expected selector: null in merge patch") - } - if !containsString(rulesYAML, "volumeName: null") { - t.Error("expected volumeName: null in merge patch") - } -} - -// containsString reports whether substr appears in s. -func containsString(s, substr string) bool { - if len(substr) == 0 || len(s) < len(substr) { - return false - } - for i := 0; i <= len(s)-len(substr); i++ { - if s[i:i+len(substr)] == substr { - return true - } - } - return false -} diff --git a/internal/controller/dashboard/customformsoverride.go b/internal/controller/dashboard/customformsoverride.go index 0a20ae71..907ebedd 100644 --- a/internal/controller/dashboard/customformsoverride.go +++ b/internal/controller/dashboard/customformsoverride.go @@ -141,10 +141,7 @@ func (m *Manager) ensureCFOMapping(ctx context.Context, crd *cozyv1alpha1.Applic } // buildMultilineStringSchema parses OpenAPI schema and creates schema with multilineString -// for all string fields inside spec that don't have enum. -// It handles two structures: -// - properties.spec.properties (most resources) -// - properties.properties (VMDisk and similar resources without spec wrapper) +// for all string fields inside spec that don't have enum func buildMultilineStringSchema(openAPISchema string) (map[string]any, error) { if openAPISchema == "" { return map[string]any{}, nil @@ -164,25 +161,15 @@ func buildMultilineStringSchema(openAPISchema string) (map[string]any, error) { "properties": map[string]any{}, } - var specProps map[string]any - var hasSpec bool - - // First try to find properties under spec - if specProp, ok := props["spec"].(map[string]any); ok { - specProps, hasSpec = specProp["properties"].(map[string]any) + // Check if there's a spec property + specProp, ok := props["spec"].(map[string]any) + if !ok { + return map[string]any{}, nil } - // If no spec wrapper, use top-level properties directly (VMDisk pattern) - if !hasSpec { - specProps = props - // Still wrap in spec for consistency with applyListInputOverrides - schemaProps := schema["properties"].(map[string]any) - specSchema := map[string]any{ - "properties": map[string]any{}, - } - schemaProps["spec"] = specSchema - processSpecProperties(specProps, specSchema["properties"].(map[string]any)) - return schema, nil + specProps, ok := specProp["properties"].(map[string]any) + if !ok { + return map[string]any{}, nil } // Create spec.properties structure in schema @@ -244,45 +231,10 @@ func applyListInputOverrides(schema map[string]any, kind string, openAPIProps ma } case "ClickHouse", "Harbor", "HTTPCache", "Kubernetes", "MariaDB", "MongoDB", - "NATS", "OpenBAO", "Postgres", "Qdrant", "RabbitMQ", "Redis": + "NATS", "OpenBAO", "Postgres", "Qdrant", "RabbitMQ", "Redis", "VMDisk": specProps := ensureSchemaPath(schema, "spec") specProps["storageClass"] = storageClassListInput() - case "VMDisk": - specProps := ensureSchemaPath(schema, "spec") - specProps["storageClass"] = storageClassListInput() - - // Override source.image.name to be an API-backed dropdown listing default images - if sourceObj, ok := specProps["source"].(map[string]any); ok { - if imgProps, ok := sourceObj["properties"].(map[string]any); ok { - if imgName, ok := imgProps["image"].(map[string]any); ok { - if imgNameProps, ok := imgName["properties"].(map[string]any); ok { - imgNameProps["name"] = map[string]any{ - "type": "listInput", - "customProps": map[string]any{ - "valueUri": "/api/clusters/{cluster}/k8s/apis/cdi.kubevirt.io/v1beta1/namespaces/cozy-public/datavolumes", - "keysToValue": []any{"metadata", "annotations", "vm-default-images.cozystack.io/name"}, - "keysToLabel": []any{"metadata", "annotations", "vm-default-images.cozystack.io/description"}, - }, - } - } - } - // Override source.disk.name to be an API-backed dropdown listing VMDisk resources - if diskName, ok := imgProps["disk"].(map[string]any); ok { - if diskNameProps, ok := diskName["properties"].(map[string]any); ok { - diskNameProps["name"] = map[string]any{ - "type": "listInput", - "customProps": map[string]any{ - "valueUri": "/api/clusters/{cluster}/k8s/apis/apps.cozystack.io/v1alpha1/namespaces/{namespace}/vmdisks", - "keysToValue": []any{"metadata", "name"}, - "keysToLabel": []any{"metadata", "name"}, - }, - } - } - } - } - } - case "FoundationDB": storageProps := ensureSchemaPath(schema, "spec", "storage") storageProps["storageClass"] = storageClassListInput() diff --git a/internal/controller/dashboard/customformsoverride_test.go b/internal/controller/dashboard/customformsoverride_test.go index 6a83d62e..7833d2e9 100644 --- a/internal/controller/dashboard/customformsoverride_test.go +++ b/internal/controller/dashboard/customformsoverride_test.go @@ -317,108 +317,6 @@ func TestApplyListInputOverrides_StorageClassFoundationDB(t *testing.T) { assertStorageClassListInput(t, sc) } -func TestApplyListInputOverrides_VMDisk_SourceFields(t *testing.T) { - openAPISchema := `{ - "properties":{ - "optical":{"type":"boolean"}, - "source":{ - "type":"object", - "properties":{ - "image":{ - "type":"object", - "properties":{ - "name":{"type":"string"} - } - }, - "disk":{ - "type":"object", - "properties":{ - "name":{"type":"string"} - } - } - } - }, - "storage":{"type":"string"}, - "storageClass":{"type":"string"} - } - }` - - schema, err := buildMultilineStringSchema(openAPISchema) - if err != nil { - t.Fatalf("buildMultilineStringSchema failed: %v", err) - } - - applyListInputOverrides(schema, "VMDisk", map[string]any{}) - - specProps := schema["properties"].(map[string]any)["spec"].(map[string]any)["properties"].(map[string]any) - - // Check storageClass - sc, ok := specProps["storageClass"].(map[string]any) - if !ok { - t.Fatal("storageClass not found in spec.properties") - } - assertStorageClassListInput(t, sc) - - // Check source.image.name listInput - // Structure: specProps["source"]["properties"]["image"]["properties"]["name"] - sourceObj, ok := specProps["source"].(map[string]any) - if !ok { - t.Fatal("source not found in spec.properties") - } - sourceObjProps, ok := sourceObj["properties"].(map[string]any) - if !ok { - t.Fatal("source.properties not found") - } - imageObj, ok := sourceObjProps["image"].(map[string]any) - if !ok { - t.Fatal("image not found in source.properties") - } - imageObjProps, ok := imageObj["properties"].(map[string]any) - if !ok { - t.Fatal("image.properties not found") - } - imgName, ok := imageObjProps["name"].(map[string]any) - if !ok { - t.Fatal("name not found in image.properties") - } - if imgName["type"] != "listInput" { - t.Errorf("expected type listInput, got %v", imgName["type"]) - } - imgNameCustomProps, ok := imgName["customProps"].(map[string]any) - if !ok { - t.Fatal("name.customProps not found") - } - expectedImageURI := "/api/clusters/{cluster}/k8s/apis/cdi.kubevirt.io/v1beta1/namespaces/cozy-public/datavolumes" - if imgNameCustomProps["valueUri"] != expectedImageURI { - t.Errorf("expected valueUri %s, got %v", expectedImageURI, imgNameCustomProps["valueUri"]) - } - - // Check source.disk.name listInput - diskObj, ok := sourceObjProps["disk"].(map[string]any) - if !ok { - t.Fatal("disk not found in source.properties") - } - diskObjProps, ok := diskObj["properties"].(map[string]any) - if !ok { - t.Fatal("disk.properties not found") - } - diskName, ok := diskObjProps["name"].(map[string]any) - if !ok { - t.Fatal("name not found in disk.properties") - } - if diskName["type"] != "listInput" { - t.Errorf("expected type listInput, got %v", diskName["type"]) - } - diskNameCustomProps, ok := diskName["customProps"].(map[string]any) - if !ok { - t.Fatal("disk name.customProps not found") - } - expectedDiskURI := "/api/clusters/{cluster}/k8s/apis/apps.cozystack.io/v1alpha1/namespaces/{namespace}/vmdisks" - if diskNameCustomProps["valueUri"] != expectedDiskURI { - t.Errorf("expected valueUri %s, got %v", expectedDiskURI, diskNameCustomProps["valueUri"]) - } -} - func TestApplyListInputOverrides_StorageClassKafka(t *testing.T) { schema := map[string]any{} applyListInputOverrides(schema, "Kafka", map[string]any{}) diff --git a/internal/controller/dashboard/factory.go b/internal/controller/dashboard/factory.go index fe0cadfd..539e503b 100644 --- a/internal/controller/dashboard/factory.go +++ b/internal/controller/dashboard/factory.go @@ -47,7 +47,6 @@ func (m *Manager) ensureFactory(ctx context.Context, crd *cozyv1alpha1.Applicati if prefix, ok := vncTabPrefix(kind); ok { tabs = append(tabs, vncTab(prefix)) } - tabs = append(tabs, eventsTab(kind)) tabs = append(tabs, yamlTab(g, v, plural)) // Use unified factory creation @@ -359,37 +358,6 @@ func secretsTab(kind string) map[string]any { } } -// eventsTab shows Kubernetes Events scoped to the application's namespace. -// Events are namespace-scoped because Kubernetes Events don't carry application -// labels and cannot be filtered by label selector. In Cozystack's multi-tenancy -// model, each tenant namespace maps to a single application scope, so namespace -// filtering provides the correct event scope. -// For Tenant applications, events are fetched from status.namespace (the tenant's -// own namespace) instead of the parent namespace where the Tenant object lives. -func eventsTab(kind string) map[string]any { - nsPlaceholder := "{3}" - if kind == "Tenant" { - nsPlaceholder = "{reqsJsonPath[0]['.status.namespace']}" - } - return map[string]any{ - "key": "events", - "label": "Events", - "children": []any{ - map[string]any{ - "type": "EnrichedTable", - "data": map[string]any{ - "id": "events-table", - "fetchUrl": "/api/clusters/{2}/k8s/api/v1/namespaces/" + nsPlaceholder + "/events", - "cluster": "{2}", - "baseprefix": "/openapi-ui", - "customizationId": "factory-details-events", - "pathToItems": []any{"items"}, - }, - }, - }, - } -} - func yamlTab(group, version, plural string) map[string]any { return map[string]any{ "key": "yaml", diff --git a/internal/controller/dashboard/factory_test.go b/internal/controller/dashboard/factory_test.go deleted file mode 100644 index b3742564..00000000 --- a/internal/controller/dashboard/factory_test.go +++ /dev/null @@ -1,60 +0,0 @@ -package dashboard - -import ( - "testing" -) - -func TestEventsTab_Structure(t *testing.T) { - tab := eventsTab("PostgreSQL") - - if tab["key"] != "events" { - t.Errorf("expected key=events, got %v", tab["key"]) - } - if tab["label"] != "Events" { - t.Errorf("expected label=Events, got %v", tab["label"]) - } - - children, ok := tab["children"].([]any) - if !ok || len(children) != 1 { - t.Fatal("expected exactly 1 child in events tab") - } - - table, ok := children[0].(map[string]any) - if !ok { - t.Fatal("child is not a map") - } - if table["type"] != "EnrichedTable" { - t.Errorf("expected type=EnrichedTable, got %v", table["type"]) - } - - data, ok := table["data"].(map[string]any) - if !ok { - t.Fatal("table data is not a map") - } - if data["id"] != "events-table" { - t.Errorf("expected id=events-table, got %v", data["id"]) - } - if data["fetchUrl"] != "/api/clusters/{2}/k8s/api/v1/namespaces/{3}/events" { - t.Errorf("unexpected fetchUrl for non-Tenant: %v", data["fetchUrl"]) - } - if data["customizationId"] != "factory-details-events" { - t.Errorf("expected customizationId=factory-details-events, got %v", data["customizationId"]) - } - - pathToItems, ok := data["pathToItems"].([]any) - if !ok || len(pathToItems) != 1 || pathToItems[0] != "items" { - t.Errorf("expected pathToItems=[items], got %v", data["pathToItems"]) - } -} - -func TestEventsTab_TenantUsesStatusNamespace(t *testing.T) { - tab := eventsTab("Tenant") - children := tab["children"].([]any) - table := children[0].(map[string]any) - data := table["data"].(map[string]any) - - expectedURL := "/api/clusters/{2}/k8s/api/v1/namespaces/{reqsJsonPath[0]['.status.namespace']}/events" - if data["fetchUrl"] != expectedURL { - t.Errorf("expected Tenant fetchUrl to use status.namespace, got %v", data["fetchUrl"]) - } -} diff --git a/internal/controller/dashboard/sidebar.go b/internal/controller/dashboard/sidebar.go index 69c933fb..90721b3d 100644 --- a/internal/controller/dashboard/sidebar.go +++ b/internal/controller/dashboard/sidebar.go @@ -144,9 +144,6 @@ func (m *Manager) ensureSidebar(ctx context.Context, crd *cozyv1alpha1.Applicati // Add sidebar for backups.cozystack.io Backup resource keysAndTags["backups"] = []any{"backup-sidebar"} - // Add sidebar for backups.cozystack.io RestoreJob resource - keysAndTags["restorejobs"] = []any{"restorejob-sidebar"} - // 3) Sort items within each category by Weight (desc), then Label (A→Z) for cat := range categories { sort.Slice(categories[cat], func(i, j int) bool { @@ -218,11 +215,6 @@ func (m *Manager) ensureSidebar(ctx context.Context, crd *cozyv1alpha1.Applicati "label": "Backups", "link": "/openapi-ui/{cluster}/{namespace}/api-table/backups.cozystack.io/v1alpha1/backups", }, - map[string]any{ - "key": "restorejobs", - "label": "RestoreJobs", - "link": "/openapi-ui/{cluster}/{namespace}/api-table/backups.cozystack.io/v1alpha1/restorejobs", - }, }, }) @@ -266,7 +258,6 @@ func (m *Manager) ensureSidebar(ctx context.Context, crd *cozyv1alpha1.Applicati "stock-project-factory-plan-details", "stock-project-factory-backupjob-details", "stock-project-factory-backup-details", - "stock-project-factory-restorejob-details", "stock-project-factory-external-ips", "stock-project-api-form", "stock-project-api-table", @@ -281,12 +272,7 @@ func (m *Manager) ensureSidebar(ctx context.Context, crd *cozyv1alpha1.Applicati "stock-instance-builtin-table", } - // Add details sidebars for all CRDs with dashboard config, and collect - // the set of IDs that are genuinely CRD-backed (dynamic). The hardcoded - // `-details` IDs above (e.g. kube-* and backup/backupjob/plan/restorejob) - // are not tied to an ApplicationDefinition and must be treated as static - // so they receive consistent labels via upsertMultipleSidebars(). - dynamicDetailsIDs := map[string]bool{} + // Add details sidebars for all CRDs with dashboard config for i := range all { def := &all[i] if def.Spec.Dashboard == nil { @@ -296,22 +282,17 @@ func (m *Manager) ensureSidebar(ctx context.Context, crd *cozyv1alpha1.Applicati lowerKind := strings.ToLower(kind) detailsID := fmt.Sprintf("stock-project-factory-%s-details", lowerKind) targetIDs = append(targetIDs, detailsID) - dynamicDetailsIDs[detailsID] = true } // 7) Upsert all target sidebars with identical menuItems and keysAndTags - return m.upsertMultipleSidebars(ctx, crd, targetIDs, dynamicDetailsIDs, keysAndTags, menuItems) + return m.upsertMultipleSidebars(ctx, crd, targetIDs, keysAndTags, menuItems) } // upsertMultipleSidebars creates/updates several Sidebar resources with the same menu spec. -// dynamicDetailsIDs identifies `stock-project-factory--details` sidebars that are -// backed by an ApplicationDefinition and should therefore be owned by that CRD. -// Any other ID is treated as a static sidebar (managed-by labels, no owner ref). func (m *Manager) upsertMultipleSidebars( ctx context.Context, crd *cozyv1alpha1.ApplicationDefinition, ids []string, - dynamicDetailsIDs map[string]bool, keysAndTags map[string]any, menuItems []any, ) error { @@ -327,10 +308,8 @@ func (m *Manager) upsertMultipleSidebars( if _, err := controllerutil.CreateOrUpdate(ctx, m.Client, obj, func() error { // Only set owner reference for dynamic sidebars (stock-project-factory-{kind}-details) - // that are actually backed by an ApplicationDefinition. Static sidebars — including - // hardcoded details sidebars for built-in/backup resources — must fall through to the - // static-label branch so they're managed consistently. - if strings.HasPrefix(id, "stock-project-factory-") && strings.HasSuffix(id, "-details") && dynamicDetailsIDs[id] { + // Static sidebars (stock-instance-*, stock-project-*) should not have owner references + if strings.HasPrefix(id, "stock-project-factory-") && strings.HasSuffix(id, "-details") { // This is a dynamic sidebar, set owner reference only if it matches the current CRD _, _, kind := pickGVK(crd) lowerKind := strings.ToLower(kind) diff --git a/internal/controller/dashboard/static_helpers.go b/internal/controller/dashboard/static_helpers.go index db37e05b..1a1018cb 100644 --- a/internal/controller/dashboard/static_helpers.go +++ b/internal/controller/dashboard/static_helpers.go @@ -650,31 +650,12 @@ func createStringColumn(name, jsonPath string) map[string]any { } } -// createTimestampColumn creates a timestamp column with custom formatting. -// Extra jsonPaths act as ordered fallbacks: the first non-null path wins, -// and `-` is used only when every path is null. Depth is capped at two -// because the template parser's non-greedy matcher cannot track more than -// one level of alternating quotes in a nested reqsJsonPath fallback. -func createTimestampColumn(name string, jsonPaths ...string) map[string]any { - if len(jsonPaths) == 0 { - panic("createTimestampColumn requires at least one jsonPath") - } - if len(jsonPaths) > 2 { - panic("createTimestampColumn supports at most two jsonPaths (primary + one fallback)") - } - - var text string - switch len(jsonPaths) { - case 1: - text = "{reqsJsonPath[0]['" + jsonPaths[0] + "']['-']}" - case 2: - text = "{reqsJsonPath[0]['" + jsonPaths[0] + "'][\"{reqsJsonPath[0]['" + jsonPaths[1] + "']['-']}\"]}" - } - +// createTimestampColumn creates a timestamp column with custom formatting +func createTimestampColumn(name, jsonPath string) map[string]any { return map[string]any{ "name": name, "type": "factory", - "jsonPath": jsonPaths[0], + "jsonPath": jsonPath, "customProps": map[string]any{ "disableEventBubbling": true, "items": []any{ @@ -692,7 +673,7 @@ func createTimestampColumn(name string, jsonPaths ...string) map[string]any { "data": map[string]any{ "formatter": "timestamp", "id": "time-value", - "text": text, + "text": "{reqsJsonPath[0]['" + jsonPath + "']['-']}", }, }, }, diff --git a/internal/controller/dashboard/static_refactored.go b/internal/controller/dashboard/static_refactored.go index 9f28c0fb..06c4239f 100644 --- a/internal/controller/dashboard/static_refactored.go +++ b/internal/controller/dashboard/static_refactored.go @@ -224,20 +224,6 @@ func CreateAllCustomColumnsOverrides() []*dashboardv1alpha1.CustomColumnsOverrid createStringColumn("Name", ".name"), }), - // Factory details events. Event Time falls back to `.firstTimestamp` - // because core/v1 Events (Helm, controller-runtime) populate only the - // legacy timestamps while events.k8s.io/v1 Events populate only - // `.eventTime`; taking the first non-null of the two covers both. - createCustomColumnsOverride("factory-details-events", []any{ - createTimestampColumn("Last Seen", ".lastTimestamp", ".eventTime"), - createTimestampColumn("Event Time", ".eventTime", ".firstTimestamp"), - createStringColumn("Type", ".type"), - createStringColumn("Reason", ".reason"), - createStringColumn("Object", ".involvedObject.kind"), - createStringColumn("Name", ".involvedObject.name"), - createStringColumn("Message", ".message"), - }), - // Factory status conditions createCustomColumnsOverride("factory-status-conditions", []any{ createStringColumn("Type", ".type"), @@ -425,14 +411,6 @@ func CreateAllCustomColumnsOverrides() []*dashboardv1alpha1.CustomColumnsOverrid createTimestampColumn("Taken At", ".spec.takenAt"), createTimestampColumn("Created", ".metadata.creationTimestamp"), }), - - // Stock namespace backups cozystack io v1alpha1 restorejobs - createCustomColumnsOverride("stock-namespace-/backups.cozystack.io/v1alpha1/restorejobs", []any{ - createCustomColumnWithJsonPath("Name", ".metadata.name", "RestoreJob", "", "/openapi-ui/{2}/{reqsJsonPath[0]['.metadata.namespace']['-']}/factory/restorejob-details/{reqsJsonPath[0]['.metadata.name']['-']}"), - createStringColumn("Phase", ".status.phase"), - createStringColumn("Backup", ".spec.backupRef.name"), - createTimestampColumn("Created", ".metadata.creationTimestamp"), - }), } } @@ -555,31 +533,6 @@ func CreateAllCustomFormsOverrides() []*dashboardv1alpha1.CustomFormsOverride { "backupClassName": listInputScemaItemBackupClass(), }), }), - - // RestoreJobs form override - backups.cozystack.io/v1alpha1 - createCustomFormsOverride("default-/backups.cozystack.io/v1alpha1/restorejobs", map[string]any{ - "formItems": []any{ - createFormItem("metadata.name", "Name", "text"), - createFormItem("metadata.namespace", "Namespace", "text"), - createFormItem("spec.backupRef.name", "Backup", "text"), - // Target application: leave empty to restore into the same application - // as referenced by the selected Backup. Fill all three to restore into - // a different application (e.g. rename, or restore into a new target). - createFormItem("spec.targetApplicationRef.apiGroup", "Target Application API Group (optional, used to restore into a different application)", "text"), - createFormItem("spec.targetApplicationRef.kind", "Target Application Kind (optional, used to restore into a different application)", "text"), - createFormItem("spec.targetApplicationRef.name", "Target Application Name (optional, used to restore into a different application)", "text"), - // Driver-specific options (key-value editor). Refer to the backup driver - // documentation for supported keys. - createFormItem("spec.options", "Options (driver-specific key/value pairs)", "object"), - }, - "schema": createSchema(map[string]any{ - "backupRef": map[string]any{ - "properties": map[string]any{ - "name": listInputSchemaItemBackup(), - }, - }, - }), - }), } } @@ -1970,177 +1923,6 @@ func CreateAllFactories() []*dashboardv1alpha1.Factory { } backupSpec := createUnifiedFactory(backupConfig, backupTabs, []any{"/api/clusters/{2}/k8s/apis/backups.cozystack.io/v1alpha1/namespaces/{3}/backups/{6}"}) - // RestoreJob details factory using unified approach - restoreJobConfig := UnifiedResourceConfig{ - Name: "restorejob-details", - ResourceType: "factory", - Kind: "RestoreJob", - Plural: "restorejobs", - Title: "restorejob", - } - restoreJobTabs := []any{ - map[string]any{ - "key": "details", - "label": "Details", - "children": []any{ - contentCard("details-card", map[string]any{ - "marginBottom": "24px", - }, []any{ - antdText("details-title", true, "RestoreJob details", map[string]any{ - "fontSize": 20, - "marginBottom": "12px", - }), - spacer("details-spacer", 16), - antdRow("details-grid", []any{48, 12}, []any{ - antdCol("col-left", 12, []any{ - antdFlexVertical("col-left-stack", 24, []any{ - antdFlexVertical("meta-name-block", 4, []any{ - antdText("meta-name-label", true, "Name", nil), - parsedText("meta-name-value", "{reqsJsonPath[0]['.metadata.name']['-']}", nil), - }), - antdFlexVertical("meta-namespace-block", 8, []any{ - antdText("meta-namespace-label", true, "Namespace", nil), - antdFlex("header-row", 6, []any{ - map[string]any{ - "type": "antdText", - "data": map[string]any{ - "id": "header-badge", - "text": "NS", - "title": "namespace", - "style": map[string]any{ - "backgroundColor": "#a25792ff", - "borderRadius": "20px", - "color": "#fff", - "display": "inline-block", - "fontFamily": "RedHatDisplay, Overpass, overpass, helvetica, arial, sans-serif", - "fontSize": "15px", - "fontWeight": 400, - "lineHeight": "24px", - "minWidth": 24, - "padding": "0 9px", - "textAlign": "center", - "whiteSpace": "nowrap", - }, - }, - }, - map[string]any{ - "type": "antdLink", - "data": map[string]any{ - "id": "namespace-link", - "text": "{reqsJsonPath[0]['.metadata.namespace']['-']}", - "href": "/openapi-ui/{2}/{reqsJsonPath[0]['.metadata.namespace']['-']}/factory/marketplace", - }, - }, - }), - }), - antdFlexVertical("meta-created-block", 4, []any{ - antdText("created-time-label", true, "Created", nil), - antdFlex("created-time-block", 6, []any{ - map[string]any{ - "type": "antdText", - "data": map[string]any{ - "id": "created-time-icon", - "text": "🌐", - }, - }, - map[string]any{ - "type": "parsedText", - "data": map[string]any{ - "formatter": "timestamp", - "id": "created-time-value", - "text": "{reqsJsonPath[0]['.metadata.creationTimestamp']['-']}", - }, - }, - }), - }), - }), - }), - antdCol("col-right", 12, []any{ - antdFlexVertical("col-right-stack", 24, []any{ - antdFlexVertical("status-phase-block", 4, []any{ - antdText("phase-label", true, "Phase", nil), - parsedText("phase-value", "{reqsJsonPath[0]['.status.phase']['-']}", nil), - }), - antdFlexVertical("spec-backup-ref-block", 4, []any{ - antdText("backup-ref-label", true, "Backup Ref", nil), - parsedText("backup-ref-value", "{reqsJsonPath[0]['.spec.backupRef.name']['-']}", nil), - }), - antdFlexVertical("spec-target-application-ref-block", 4, []any{ - antdText("target-application-ref-label", true, "Target Application", nil), - // targetApplicationRef is optional — when absent, the restore targets - // the same application as the selected Backup. Show a single friendly - // fallback in that case instead of rendering "-.-/-". - parsedText("target-application-ref-value", "{reqsJsonPath[0]['.spec.targetApplicationRef.name']['Same as backup']}", nil), - }), - antdFlexVertical("spec-options-target-namespace-block", 4, []any{ - antdText("options-target-namespace-label", true, "Target Namespace", nil), - parsedText("options-target-namespace-value", "{reqsJsonPath[0]['.spec.options.targetNamespace']['-']}", nil), - }), - antdFlexVertical("spec-options-fail-if-target-exists-block", 4, []any{ - antdText("options-fail-if-target-exists-label", true, "Fail If Target Exists", nil), - parsedText("options-fail-if-target-exists-value", "{reqsJsonPath[0]['.spec.options.failIfTargetExists']['-']}", nil), - }), - antdFlexVertical("spec-options-keep-original-pvc-block", 4, []any{ - antdText("options-keep-original-pvc-label", true, "Keep Original PVC", nil), - parsedText("options-keep-original-pvc-value", "{reqsJsonPath[0]['.spec.options.keepOriginalPVC']['-']}", nil), - }), - antdFlexVertical("spec-options-keep-original-ip-mac-block", 4, []any{ - antdText("options-keep-original-ip-mac-label", true, "Keep Original IP/MAC", nil), - parsedText("options-keep-original-ip-mac-value", "{reqsJsonPath[0]['.spec.options.keepOriginalIpAndMac']['-']}", nil), - }), - antdFlexVertical("status-started-at-block", 4, []any{ - antdText("started-at-label", true, "Started At", nil), - antdFlex("started-at-time-block", 6, []any{ - map[string]any{ - "type": "antdText", - "data": map[string]any{ - "id": "started-at-time-icon", - "text": "🌐", - }, - }, - map[string]any{ - "type": "parsedText", - "data": map[string]any{ - "formatter": "timestamp", - "id": "started-at-time-value", - "text": "{reqsJsonPath[0]['.status.startedAt']['-']}", - }, - }, - }), - }), - antdFlexVertical("status-completed-at-block", 4, []any{ - antdText("completed-at-label", true, "Completed At", nil), - antdFlex("completed-at-time-block", 6, []any{ - map[string]any{ - "type": "antdText", - "data": map[string]any{ - "id": "completed-at-time-icon", - "text": "🌐", - }, - }, - map[string]any{ - "type": "parsedText", - "data": map[string]any{ - "formatter": "timestamp", - "id": "completed-at-time-value", - "text": "{reqsJsonPath[0]['.status.completedAt']['-']}", - }, - }, - }), - }), - antdFlexVertical("status-message-block", 4, []any{ - antdText("message-label", true, "Message", nil), - parsedText("message-value", "{reqsJsonPath[0]['.status.message']['-']}", nil), - }), - }), - }), - }), - }), - }, - }, - } - restoreJobSpec := createUnifiedFactory(restoreJobConfig, restoreJobTabs, []any{"/api/clusters/{2}/k8s/apis/backups.cozystack.io/v1alpha1/namespaces/{3}/restorejobs/{6}"}) - // External IPs factory (filtered services) externalIPsTabs := []any{ map[string]any{ @@ -2193,7 +1975,6 @@ func CreateAllFactories() []*dashboardv1alpha1.Factory { createFactory("plan-details", planSpec), createFactory("backupjob-details", backupJobSpec), createFactory("backup-details", backupSpec), - createFactory("restorejob-details", restoreJobSpec), createFactory("external-ips", externalIPsSpec), } } @@ -2213,10 +1994,9 @@ func CreateAllNavigations() []*dashboardv1alpha1.Navigation { "base-factory-namespaced-api-networking.k8s.io-v1-ingresses": "kube-ingress-details", "base-factory-namespaced-api-cozystack.io-v1alpha1-workloadmonitors": "workloadmonitor-details", // Backup resources (not ApplicationDefinitions, so ensureNavigation doesn't cover them) - "base-factory-namespaced-api-backups.cozystack.io-v1alpha1-plans": "plan-details", - "base-factory-namespaced-api-backups.cozystack.io-v1alpha1-backupjobs": "backupjob-details", - "base-factory-namespaced-api-backups.cozystack.io-v1alpha1-backups": "backup-details", - "base-factory-namespaced-api-backups.cozystack.io-v1alpha1-restorejobs": "restorejob-details", + "base-factory-namespaced-api-backups.cozystack.io-v1alpha1-plans": "plan-details", + "base-factory-namespaced-api-backups.cozystack.io-v1alpha1-backupjobs": "backupjob-details", + "base-factory-namespaced-api-backups.cozystack.io-v1alpha1-backups": "backup-details", } return []*dashboardv1alpha1.Navigation{ @@ -2341,19 +2121,6 @@ func listInputScemaItemBackupClass() map[string]any { } } -// listInputSchemaItemBackup returns a listInput schema overlay for selecting a Backup -// from the current namespace (used by RestoreJob form for spec.backupRef.name). -func listInputSchemaItemBackup() map[string]any { - return map[string]any{ - "type": "listInput", - "customProps": map[string]any{ - "valueUri": "/api/clusters/{cluster}/k8s/apis/backups.cozystack.io/v1alpha1/namespaces/{namespace}/backups", - "keysToValue": []any{"metadata", "name"}, - "keysToLabel": []any{"metadata", "name"}, - }, - } -} - // backupClassSchema returns the schema for spec.backupClassName as listInput (BackupJob/Plan). func createSchema(customProps map[string]any) map[string]any { return map[string]any{ diff --git a/internal/controller/workloadmonitor_controller.go b/internal/controller/workloadmonitor_controller.go index 35906a87..a684df9b 100644 --- a/internal/controller/workloadmonitor_controller.go +++ b/internal/controller/workloadmonitor_controller.go @@ -4,13 +4,7 @@ import ( "context" "encoding/json" "fmt" - "io" - "net/http" - "net/url" "sort" - "strconv" - "strings" - "time" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" @@ -28,29 +22,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" - cosiv1alpha1 "sigs.k8s.io/container-object-storage-interface-api/apis/objectstorage/v1alpha1" ) -const ( - // namespaceMonitoringLabel is the namespace label that indicates which tenant - // namespace hosts the monitoring stack (VictoriaMetrics/Prometheus). - namespaceMonitoringLabel = "namespace.cozystack.io/monitoring" - workloadLabelPrefix = "workloads.cozystack.io/" - // workloadMonitorLabel is reserved: it names the WorkloadMonitor that owns - // the Workload and is always set by the reconciler, so it is never copied - // from monitor labels. - workloadMonitorLabel = workloadLabelPrefix + "monitor" - // vmSelectService is the well-known service name for VictoriaMetrics vmselect - // within a monitoring namespace. Port 8481, path /select/0/prometheus. - vmSelectService = "vmselect-shortterm" - vmSelectPort = "8481" - vmSelectPath = "/select/0/prometheus" -) - -// prometheusHTTPClient is a dedicated HTTP client for Prometheus queries, -// avoiding the shared http.DefaultClient global. -var prometheusHTTPClient = &http.Client{Timeout: 10 * time.Second} - // WorkloadMonitorReconciler reconciles a WorkloadMonitor object type WorkloadMonitorReconciler struct { client.Client @@ -63,13 +36,6 @@ type WorkloadMonitorReconciler struct { // +kubebuilder:rbac:groups=cozystack.io,resources=workloads/status,verbs=get;update;patch // +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch // +kubebuilder:rbac:groups=core,resources=persistentvolumeclaims,verbs=get;list;watch -// +kubebuilder:rbac:groups=core,resources=namespaces,verbs=get -// +kubebuilder:rbac:groups=objectstorage.k8s.io,resources=bucketclaims,verbs=get;list;watch - -// isBucketClaimReady checks if the BucketClaim has been provisioned. -func (r *WorkloadMonitorReconciler) isBucketClaimReady(bc *cosiv1alpha1.BucketClaim) bool { - return bc.Status.BucketReady -} // isServiceReady checks if the service has an external IP bound func (r *WorkloadMonitorReconciler) isServiceReady(svc *corev1.Service) bool { @@ -135,190 +101,6 @@ func updateOwnerReferences(obj metav1.Object, monitor client.Object) { obj.SetOwnerReferences(owners) } -// resolvePrometheusURL returns the Prometheus-compatible API base URL for the given namespace. -// It reads the namespace.cozystack.io/monitoring label to find the monitoring namespace, -// then constructs the vmselect URL. Returns empty string if monitoring is not configured. -func (r *WorkloadMonitorReconciler) resolvePrometheusURL(ctx context.Context, namespace string) string { - logger := log.FromContext(ctx) - ns := &corev1.Namespace{} - if err := r.Get(ctx, types.NamespacedName{Name: namespace}, ns); err != nil { - logger.V(1).Info("Failed to read namespace for monitoring resolution", "namespace", namespace, "error", err) - return "" - } - monitoringNS := ns.Labels[namespaceMonitoringLabel] - if monitoringNS == "" { - return "" - } - return fmt.Sprintf("http://%s.%s.svc:%s%s", vmSelectService, monitoringNS, vmSelectPort, vmSelectPath) -} - -// bucketMetrics holds size metrics for a single bucket, keyed by metric name. -type bucketMetrics struct { - LogicalSize int64 - PhysicalSize int64 - HasLogical bool - HasPhysical bool -} - -// queryAllBucketMetrics fetches SeaweedFS bucket size metrics for the given -// bucket names in a single Prometheus query and returns them keyed by bucket -// name. The query is scoped to only the requested buckets to avoid fetching -// metrics for buckets belonging to other WorkloadMonitors. -func (r *WorkloadMonitorReconciler) queryAllBucketMetrics(ctx context.Context, prometheusBaseURL string, bucketNames []string) map[string]*bucketMetrics { - result := make(map[string]*bucketMetrics) - if prometheusBaseURL == "" || len(bucketNames) == 0 { - return result - } - logger := log.FromContext(ctx) - - query := fmt.Sprintf(`{__name__=~"SeaweedFS_s3_bucket_(size|physical_size)_bytes",bucket=~"%s"}`, strings.Join(bucketNames, "|")) - u, err := url.Parse(strings.TrimRight(prometheusBaseURL, "/") + "/api/v1/query") - if err != nil { - logger.Error(err, "Failed to parse Prometheus URL") - return result - } - u.RawQuery = url.Values{"query": {query}}.Encode() - - httpCtx, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - - req, err := http.NewRequestWithContext(httpCtx, http.MethodGet, u.String(), nil) - if err != nil { - logger.Error(err, "Failed to create Prometheus request") - return result - } - - resp, err := prometheusHTTPClient.Do(req) - if err != nil { - logger.V(1).Info("Failed to query Prometheus for bucket metrics", "error", err) - return result - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - logger.V(1).Info("Prometheus returned non-OK status for bucket metrics", "status", resp.StatusCode) - return result - } - - body, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20)) - if err != nil { - logger.Error(err, "Failed to read Prometheus response") - return result - } - - var promResp struct { - Status string `json:"status"` - Data struct { - Result []struct { - Metric map[string]string `json:"metric"` - Value [2]json.RawMessage `json:"value"` - } `json:"result"` - } `json:"data"` - } - if err := json.Unmarshal(body, &promResp); err != nil { - logger.Error(err, "Failed to parse Prometheus response") - return result - } - if promResp.Status != "success" { - return result - } - - for _, r := range promResp.Data.Result { - bucket := r.Metric["bucket"] - metricName := r.Metric["__name__"] - if bucket == "" || metricName == "" { - continue - } - - var valueStr string - if err := json.Unmarshal(r.Value[1], &valueStr); err != nil { - continue - } - val, err := strconv.ParseFloat(valueStr, 64) - if err != nil { - continue - } - - bm, ok := result[bucket] - if !ok { - bm = &bucketMetrics{} - result[bucket] = bm - } - - switch metricName { - case "SeaweedFS_s3_bucket_size_bytes": - bm.LogicalSize = int64(val) - bm.HasLogical = true - case "SeaweedFS_s3_bucket_physical_size_bytes": - bm.PhysicalSize = int64(val) - bm.HasPhysical = true - } - } - - return result -} - -// reconcileBucketClaimForMonitor creates or updates a Workload object for the given BucketClaim and WorkloadMonitor. -func (r *WorkloadMonitorReconciler) reconcileBucketClaimForMonitor( - ctx context.Context, - monitor *cozyv1alpha1.WorkloadMonitor, - bc cosiv1alpha1.BucketClaim, - allMetrics map[string]*bucketMetrics, -) error { - logger := log.FromContext(ctx) - workload := &cozyv1alpha1.Workload{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("bucket-%s", bc.Name), - Namespace: bc.Namespace, - Labels: make(map[string]string, len(bc.Labels)), - }, - } - - resources := make(map[string]resource.Quantity) - - // Look up pre-fetched bucket metrics by the SeaweedFS bucket name. - // bc.Status.BucketName is the COSI Bucket name, which the COSI driver - // uses directly as the SeaweedFS bucket name. - if bm, ok := allMetrics[bc.Status.BucketName]; ok { - if bm.HasLogical { - resources["s3-storage-bytes"] = *resource.NewQuantity(bm.LogicalSize, resource.BinarySI) - } - if bm.HasPhysical { - resources["s3-physical-storage-bytes"] = *resource.NewQuantity(bm.PhysicalSize, resource.BinarySI) - } - } - - monitorLabels := r.getMonitorLabels(monitor) - _, err := ctrl.CreateOrUpdate(ctx, r.Client, workload, func() error { - updateOwnerReferences(workload.GetObjectMeta(), &bc) - - if workload.Labels == nil { - workload.Labels = make(map[string]string) - } - // Apply monitor-level labels first so source-object labels can override on conflict - for k, v := range monitorLabels { - workload.Labels[k] = v - } - for k, v := range bc.Labels { - workload.Labels[k] = v - } - workload.Labels[workloadMonitorLabel] = monitor.Name - - workload.Status.Kind = monitor.Spec.Kind - workload.Status.Type = monitor.Spec.Type - workload.Status.Resources = resources - workload.Status.Operational = r.isBucketClaimReady(&bc) - - return nil - }) - if err != nil { - logger.Error(err, "Failed to CreateOrUpdate Workload", "workload", workload.Name) - return err - } - - return nil -} - // reconcileServiceForMonitor creates or updates a Workload object for the given Service and WorkloadMonitor. func (r *WorkloadMonitorReconciler) reconcileServiceForMonitor( ctx context.Context, @@ -355,22 +137,14 @@ func (r *WorkloadMonitorReconciler) reconcileServiceForMonitor( resourceLabel = fmt.Sprintf("%s.ipaddresspool.metallb.io/requests.ipaddresses", resourceLabel) resources[resourceLabel] = quantity - monitorLabels := r.getMonitorLabels(monitor) _, err := ctrl.CreateOrUpdate(ctx, r.Client, workload, func() error { // Update owner references with the new monitor updateOwnerReferences(workload.GetObjectMeta(), &svc) - // Apply monitor-level labels first so source-object labels can override on conflict - if workload.Labels == nil { - workload.Labels = make(map[string]string) - } - for k, v := range monitorLabels { - workload.Labels[k] = v - } for k, v := range svc.Labels { workload.Labels[k] = v } - workload.Labels[workloadMonitorLabel] = monitor.Name + workload.Labels["workloads.cozystack.io/monitor"] = monitor.Name // Fill Workload status fields: workload.Status.Kind = monitor.Spec.Kind @@ -414,22 +188,14 @@ func (r *WorkloadMonitorReconciler) reconcilePVCForMonitor( resources[resourceLabel] = resourceQuantity } - monitorLabels := r.getMonitorLabels(monitor) _, err := ctrl.CreateOrUpdate(ctx, r.Client, workload, func() error { // Update owner references with the new monitor updateOwnerReferences(workload.GetObjectMeta(), &pvc) - // Apply monitor-level labels first so source-object labels can override on conflict - if workload.Labels == nil { - workload.Labels = make(map[string]string) - } - for k, v := range monitorLabels { - workload.Labels[k] = v - } for k, v := range pvc.Labels { workload.Labels[k] = v } - workload.Labels[workloadMonitorLabel] = monitor.Name + workload.Labels["workloads.cozystack.io/monitor"] = monitor.Name // Fill Workload status fields: workload.Status.Kind = monitor.Spec.Kind @@ -496,22 +262,14 @@ func (r *WorkloadMonitorReconciler) reconcilePodForMonitor( } metaLabels := r.getWorkloadMetadata(&pod) - monitorLabels := r.getMonitorLabels(monitor) _, err := ctrl.CreateOrUpdate(ctx, r.Client, workload, func() error { // Update owner references with the new monitor updateOwnerReferences(workload.GetObjectMeta(), &pod) - // Apply monitor-level labels first so source-object labels can override on conflict - if workload.Labels == nil { - workload.Labels = make(map[string]string) - } - for k, v := range monitorLabels { - workload.Labels[k] = v - } for k, v := range pod.Labels { workload.Labels[k] = v } - workload.Labels[workloadMonitorLabel] = monitor.Name + workload.Labels["workloads.cozystack.io/monitor"] = monitor.Name // Add workload meta to labels for k, v := range metaLabels { @@ -617,34 +375,6 @@ func (r *WorkloadMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ } } - bucketClaimList := &cosiv1alpha1.BucketClaimList{} - if err := r.List( - ctx, - bucketClaimList, - client.InNamespace(monitor.Namespace), - client.MatchingLabels(monitor.Spec.Selector), - ); err != nil { - logger.Error(err, "Unable to list BucketClaims for WorkloadMonitor", "monitor", monitor.Name) - return ctrl.Result{}, err - } - - if len(bucketClaimList.Items) > 0 { - bucketPromURL := r.resolvePrometheusURL(ctx, monitor.Namespace) - var bucketNames []string - for _, bc := range bucketClaimList.Items { - if bc.Status.BucketName != "" { - bucketNames = append(bucketNames, bc.Status.BucketName) - } - } - allBucketMetrics := r.queryAllBucketMetrics(ctx, bucketPromURL, bucketNames) - for _, bc := range bucketClaimList.Items { - if err := r.reconcileBucketClaimForMonitor(ctx, monitor, bc, allBucketMetrics); err != nil { - logger.Error(err, "Failed to reconcile Workload for BucketClaim", "BucketClaim", bc.Name) - continue - } - } - } - // Update WorkloadMonitor status based on observed pods monitor.Status.ObservedReplicas = observedReplicas monitor.Status.AvailableReplicas = availableReplicas @@ -658,12 +388,10 @@ func (r *WorkloadMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ fresh.Status.ObservedReplicas = observedReplicas fresh.Status.AvailableReplicas = availableReplicas - // Default to operational = true, but check MinReplicas if set. - // Use fresh.Spec to avoid making decisions based on a stale cached copy - // when the spec was updated between the initial read and this retry. - fresh.Status.Operational = pointer.Bool(true) - if fresh.Spec.MinReplicas != nil && availableReplicas < *fresh.Spec.MinReplicas { - fresh.Status.Operational = pointer.Bool(false) + // Default to operational = true, but check MinReplicas if set + monitor.Status.Operational = pointer.Bool(true) + if monitor.Spec.MinReplicas != nil && availableReplicas < *monitor.Spec.MinReplicas { + monitor.Status.Operational = pointer.Bool(false) } return r.Status().Update(ctx, fresh) }) @@ -672,12 +400,7 @@ func (r *WorkloadMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ return ctrl.Result{}, err } - // Requeue periodically if there are BucketClaims to keep sizes up to date. - // Bucket sizes come from Prometheus metrics that update every 60s. - if len(bucketClaimList.Items) > 0 { - return ctrl.Result{RequeueAfter: 60 * time.Second}, nil - } - + // Return without requeue if we want purely event-driven reconciliations return ctrl.Result{}, nil } @@ -696,11 +419,6 @@ func (r *WorkloadMonitorReconciler) SetupWithManager(mgr ctrl.Manager) error { &corev1.PersistentVolumeClaim{}, handler.EnqueueRequestsFromMapFunc(mapObjectToMonitor(&corev1.PersistentVolumeClaim{}, r.Client)), ). - // Watch BucketClaims for S3 bucket billing - Watches( - &cosiv1alpha1.BucketClaim{}, - handler.EnqueueRequestsFromMapFunc(mapObjectToMonitor(&cosiv1alpha1.BucketClaim{}, r.Client)), - ). // Watch for changes to Workload objects we create (owned by WorkloadMonitor) Owns(&cozyv1alpha1.Workload{}). Complete(r) @@ -754,21 +472,3 @@ func (r *WorkloadMonitorReconciler) getWorkloadMetadata(obj client.Object) map[s } return labels } - -// getMonitorLabels extracts workloads.cozystack.io/* labels from a WorkloadMonitor -// so they can be propagated onto Workload objects created for pods, PVCs, services, -// or bucket claims. The monitor label "workloads.cozystack.io/monitor" is reserved -// and set separately per Workload, so it is excluded here. -func (r *WorkloadMonitorReconciler) getMonitorLabels(monitor *cozyv1alpha1.WorkloadMonitor) map[string]string { - labels := make(map[string]string) - for k, v := range monitor.GetLabels() { - if !strings.HasPrefix(k, workloadLabelPrefix) { - continue - } - if k == workloadMonitorLabel { - continue - } - labels[k] = v - } - return labels -} diff --git a/internal/controller/workloadmonitor_controller_test.go b/internal/controller/workloadmonitor_controller_test.go deleted file mode 100644 index 355b0379..00000000 --- a/internal/controller/workloadmonitor_controller_test.go +++ /dev/null @@ -1,846 +0,0 @@ -package controller - -import ( - "context" - "fmt" - "net/http" - "net/http/httptest" - "strings" - "testing" - - cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - cosiv1alpha1 "sigs.k8s.io/container-object-storage-interface-api/apis/objectstorage/v1alpha1" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - "sigs.k8s.io/controller-runtime/pkg/reconcile" -) - -func TestReconcile_OperationalStatusPersisted(t *testing.T) { - scheme := runtime.NewScheme() - _ = cozyv1alpha1.AddToScheme(scheme) - _ = corev1.AddToScheme(scheme) - _ = cosiv1alpha1.AddToScheme(scheme) - - minReplicas := int32(2) - monitor := &cozyv1alpha1.WorkloadMonitor{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-monitor", - Namespace: "default", - }, - Spec: cozyv1alpha1.WorkloadMonitorSpec{ - Selector: map[string]string{"app": "test"}, - MinReplicas: &minReplicas, - }, - } - - // Create one pod that is ready — availableReplicas=1 < minReplicas=2, so Operational should be false - pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod-1", - Namespace: "default", - Labels: map[string]string{"app": "test"}, - }, - Status: corev1.PodStatus{ - Conditions: []corev1.PodCondition{ - {Type: corev1.PodReady, Status: corev1.ConditionTrue}, - }, - }, - } - - fakeClient := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(monitor, pod). - WithStatusSubresource(monitor). - Build() - - reconciler := &WorkloadMonitorReconciler{Client: fakeClient, Scheme: scheme} - req := reconcile.Request{NamespacedName: types.NamespacedName{Name: "test-monitor", Namespace: "default"}} - - _, err := reconciler.Reconcile(context.TODO(), req) - if err != nil { - t.Fatalf("Reconcile returned error: %v", err) - } - - // Fetch the monitor back from fake client and check Operational is persisted - updated := &cozyv1alpha1.WorkloadMonitor{} - if err := fakeClient.Get(context.TODO(), req.NamespacedName, updated); err != nil { - t.Fatalf("Failed to get updated WorkloadMonitor: %v", err) - } - - if updated.Status.Operational == nil { - t.Fatal("Expected Operational to be set, got nil") - } - if *updated.Status.Operational { - t.Error("Expected Operational=false (1 available < 2 minReplicas), got true") - } - if updated.Status.ObservedReplicas != 1 { - t.Errorf("Expected ObservedReplicas=1, got %d", updated.Status.ObservedReplicas) - } - if updated.Status.AvailableReplicas != 1 { - t.Errorf("Expected AvailableReplicas=1, got %d", updated.Status.AvailableReplicas) - } -} - -func TestReconcile_OperationalTrue_WhenEnoughReplicas(t *testing.T) { - scheme := runtime.NewScheme() - _ = cozyv1alpha1.AddToScheme(scheme) - _ = corev1.AddToScheme(scheme) - _ = cosiv1alpha1.AddToScheme(scheme) - - minReplicas := int32(1) - monitor := &cozyv1alpha1.WorkloadMonitor{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-monitor", - Namespace: "default", - }, - Spec: cozyv1alpha1.WorkloadMonitorSpec{ - Selector: map[string]string{"app": "test"}, - MinReplicas: &minReplicas, - }, - } - - pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod-1", - Namespace: "default", - Labels: map[string]string{"app": "test"}, - }, - Status: corev1.PodStatus{ - Conditions: []corev1.PodCondition{ - {Type: corev1.PodReady, Status: corev1.ConditionTrue}, - }, - }, - } - - fakeClient := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(monitor, pod). - WithStatusSubresource(monitor). - Build() - - reconciler := &WorkloadMonitorReconciler{Client: fakeClient, Scheme: scheme} - req := reconcile.Request{NamespacedName: types.NamespacedName{Name: "test-monitor", Namespace: "default"}} - - _, err := reconciler.Reconcile(context.TODO(), req) - if err != nil { - t.Fatalf("Reconcile returned error: %v", err) - } - - updated := &cozyv1alpha1.WorkloadMonitor{} - if err := fakeClient.Get(context.TODO(), req.NamespacedName, updated); err != nil { - t.Fatalf("Failed to get updated WorkloadMonitor: %v", err) - } - - if updated.Status.Operational == nil { - t.Fatal("Expected Operational to be set, got nil") - } - if !*updated.Status.Operational { - t.Error("Expected Operational=true (1 available >= 1 minReplicas), got false") - } -} - -func TestGetMonitorLabels(t *testing.T) { - tests := []struct { - name string - labels map[string]string - expected map[string]string - }{ - { - name: "nil labels", - labels: nil, - expected: map[string]string{}, - }, - { - name: "only workloads.cozystack.io/* labels are propagated", - labels: map[string]string{ - "workloads.cozystack.io/resource-preset": "medium", - "app.kubernetes.io/name": "postgres", - "custom.example.com/team": "platform", - }, - expected: map[string]string{ - "workloads.cozystack.io/resource-preset": "medium", - }, - }, - { - name: "monitor label is reserved and excluded", - labels: map[string]string{ - "workloads.cozystack.io/resource-preset": "small", - "workloads.cozystack.io/monitor": "should-be-dropped", - }, - expected: map[string]string{ - "workloads.cozystack.io/resource-preset": "small", - }, - }, - { - name: "multiple workloads.cozystack.io labels propagate", - labels: map[string]string{ - "workloads.cozystack.io/resource-preset": "large", - "workloads.cozystack.io/tier": "db", - }, - expected: map[string]string{ - "workloads.cozystack.io/resource-preset": "large", - "workloads.cozystack.io/tier": "db", - }, - }, - { - name: "no matching labels returns empty map", - labels: map[string]string{ - "app.kubernetes.io/name": "postgres", - }, - expected: map[string]string{}, - }, - } - - r := &WorkloadMonitorReconciler{} - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - monitor := &cozyv1alpha1.WorkloadMonitor{ - ObjectMeta: metav1.ObjectMeta{Labels: tc.labels}, - } - got := r.getMonitorLabels(monitor) - if len(got) != len(tc.expected) { - t.Fatalf("expected %d labels, got %d (%v)", len(tc.expected), len(got), got) - } - for k, v := range tc.expected { - if gv, ok := got[k]; !ok || gv != v { - t.Errorf("expected label %q=%q, got %q", k, v, gv) - } - } - }) - } -} - -func TestReconcile_MonitorLabelsPropagatedToPodWorkload(t *testing.T) { - scheme := runtime.NewScheme() - _ = cozyv1alpha1.AddToScheme(scheme) - _ = corev1.AddToScheme(scheme) - - monitor := &cozyv1alpha1.WorkloadMonitor{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-monitor", - Namespace: "default", - Labels: map[string]string{ - "workloads.cozystack.io/resource-preset": "medium", - "app.kubernetes.io/name": "ignored-not-propagated", - }, - }, - Spec: cozyv1alpha1.WorkloadMonitorSpec{ - Selector: map[string]string{"app": "test"}, - Kind: "postgres", - Type: "postgres", - }, - } - - pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod-1", - Namespace: "default", - Labels: map[string]string{ - "app": "test", - "app.kubernetes.io/name": "pod-wins-on-conflict", - }, - }, - Status: corev1.PodStatus{ - Conditions: []corev1.PodCondition{ - {Type: corev1.PodReady, Status: corev1.ConditionTrue}, - }, - }, - } - - fakeClient := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(monitor, pod). - WithStatusSubresource(monitor). - Build() - - reconciler := &WorkloadMonitorReconciler{Client: fakeClient, Scheme: scheme} - req := reconcile.Request{NamespacedName: types.NamespacedName{Name: "test-monitor", Namespace: "default"}} - if _, err := reconciler.Reconcile(context.TODO(), req); err != nil { - t.Fatalf("Reconcile returned error: %v", err) - } - - workload := &cozyv1alpha1.Workload{} - if err := fakeClient.Get(context.TODO(), types.NamespacedName{Name: "pod-test-pod-1", Namespace: "default"}, workload); err != nil { - t.Fatalf("Failed to get Workload: %v", err) - } - - if got := workload.Labels["workloads.cozystack.io/resource-preset"]; got != "medium" { - t.Errorf("expected monitor label propagated, got %q", got) - } - // Non-workloads.cozystack.io monitor labels must not be copied - if _, ok := workload.Labels["app.kubernetes.io/name"]; !ok { - t.Error("expected pod label to be present on Workload") - } - // Source-object label takes precedence on conflict - if got := workload.Labels["app.kubernetes.io/name"]; got != "pod-wins-on-conflict" { - t.Errorf("expected pod label to win on conflict, got %q", got) - } - // Reserved monitor label is always set from the monitor name - if got := workload.Labels["workloads.cozystack.io/monitor"]; got != "test-monitor" { - t.Errorf("expected monitor-name label, got %q", got) - } -} - -func TestReconcile_BackwardCompat_NoMonitorLabels(t *testing.T) { - scheme := runtime.NewScheme() - _ = cozyv1alpha1.AddToScheme(scheme) - _ = corev1.AddToScheme(scheme) - - monitor := &cozyv1alpha1.WorkloadMonitor{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-monitor", - Namespace: "default", - }, - Spec: cozyv1alpha1.WorkloadMonitorSpec{ - Selector: map[string]string{"app": "test"}, - }, - } - - pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod-1", - Namespace: "default", - Labels: map[string]string{"app": "test"}, - }, - Status: corev1.PodStatus{ - Conditions: []corev1.PodCondition{ - {Type: corev1.PodReady, Status: corev1.ConditionTrue}, - }, - }, - } - - fakeClient := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(monitor, pod). - WithStatusSubresource(monitor). - Build() - - reconciler := &WorkloadMonitorReconciler{Client: fakeClient, Scheme: scheme} - req := reconcile.Request{NamespacedName: types.NamespacedName{Name: "test-monitor", Namespace: "default"}} - if _, err := reconciler.Reconcile(context.TODO(), req); err != nil { - t.Fatalf("Reconcile returned error: %v", err) - } - - workload := &cozyv1alpha1.Workload{} - if err := fakeClient.Get(context.TODO(), types.NamespacedName{Name: "pod-test-pod-1", Namespace: "default"}, workload); err != nil { - t.Fatalf("Failed to get Workload: %v", err) - } - for k := range workload.Labels { - if strings.HasPrefix(k, "workloads.cozystack.io/") && k != "workloads.cozystack.io/monitor" { - t.Errorf("unexpected workload label present: %q", k) - } - } -} - -func TestReconcile_OperationalTrue_WhenNoMinReplicas(t *testing.T) { - scheme := runtime.NewScheme() - _ = cozyv1alpha1.AddToScheme(scheme) - _ = corev1.AddToScheme(scheme) - _ = cosiv1alpha1.AddToScheme(scheme) - - monitor := &cozyv1alpha1.WorkloadMonitor{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-monitor", - Namespace: "default", - }, - Spec: cozyv1alpha1.WorkloadMonitorSpec{ - Selector: map[string]string{"app": "test"}, - // No MinReplicas — should default to operational=true - }, - } - - fakeClient := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(monitor). - WithStatusSubresource(monitor). - Build() - - reconciler := &WorkloadMonitorReconciler{Client: fakeClient, Scheme: scheme} - req := reconcile.Request{NamespacedName: types.NamespacedName{Name: "test-monitor", Namespace: "default"}} - - _, err := reconciler.Reconcile(context.TODO(), req) - if err != nil { - t.Fatalf("Reconcile returned error: %v", err) - } - - updated := &cozyv1alpha1.WorkloadMonitor{} - if err := fakeClient.Get(context.TODO(), req.NamespacedName, updated); err != nil { - t.Fatalf("Failed to get updated WorkloadMonitor: %v", err) - } - - if updated.Status.Operational == nil { - t.Fatal("Expected Operational to be set, got nil") - } - if !*updated.Status.Operational { - t.Error("Expected Operational=true (no MinReplicas constraint), got false") - } -} - -func newTestScheme() *runtime.Scheme { - s := runtime.NewScheme() - _ = cozyv1alpha1.AddToScheme(s) - _ = corev1.AddToScheme(s) - _ = cosiv1alpha1.AddToScheme(s) - return s -} - -func TestReconcileBucketClaimCreatesWorkload(t *testing.T) { - s := newTestScheme() - - monitor := &cozyv1alpha1.WorkloadMonitor{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-bucket", - Namespace: "tenant-demo", - }, - Spec: cozyv1alpha1.WorkloadMonitorSpec{ - Kind: "bucket", - Type: "s3", - Selector: map[string]string{ - "app.kubernetes.io/instance": "my-bucket", - }, - }, - } - - bc := &cosiv1alpha1.BucketClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-bucket", - Namespace: "tenant-demo", - Labels: map[string]string{ - "app.kubernetes.io/instance": "my-bucket", - }, - }, - Spec: cosiv1alpha1.BucketClaimSpec{ - BucketClassName: "seaweedfs", - Protocols: []cosiv1alpha1.Protocol{cosiv1alpha1.ProtocolS3}, - }, - Status: cosiv1alpha1.BucketClaimStatus{ - BucketReady: true, - BucketName: "cosi-abc123", - }, - } - - fakeClient := fake.NewClientBuilder(). - WithScheme(s). - WithObjects(monitor, bc). - WithStatusSubresource(monitor). - Build() - - reconciler := &WorkloadMonitorReconciler{Client: fakeClient, Scheme: s} - req := reconcile.Request{NamespacedName: types.NamespacedName{ - Name: "my-bucket", - Namespace: "tenant-demo", - }} - - _, err := reconciler.Reconcile(context.TODO(), req) - if err != nil { - t.Fatalf("Reconcile returned error: %v", err) - } - - workload := &cozyv1alpha1.Workload{} - err = fakeClient.Get(context.TODO(), types.NamespacedName{ - Name: "bucket-my-bucket", - Namespace: "tenant-demo", - }, workload) - if err != nil { - t.Fatalf("expected Workload to be created, got error: %v", err) - } - - if workload.Status.Kind != "bucket" { - t.Errorf("expected Kind=bucket, got %q", workload.Status.Kind) - } - if workload.Status.Type != "s3" { - t.Errorf("expected Type=s3, got %q", workload.Status.Type) - } - if !workload.Status.Operational { - t.Error("expected Operational=true for ready BucketClaim") - } -} - -func TestReconcileBucketClaimNotReady(t *testing.T) { - s := newTestScheme() - - monitor := &cozyv1alpha1.WorkloadMonitor{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-bucket", - Namespace: "tenant-demo", - }, - Spec: cozyv1alpha1.WorkloadMonitorSpec{ - Kind: "bucket", - Type: "s3", - Selector: map[string]string{ - "app.kubernetes.io/instance": "my-bucket", - }, - }, - } - - bc := &cosiv1alpha1.BucketClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-bucket", - Namespace: "tenant-demo", - Labels: map[string]string{ - "app.kubernetes.io/instance": "my-bucket", - }, - }, - Spec: cosiv1alpha1.BucketClaimSpec{ - BucketClassName: "seaweedfs", - Protocols: []cosiv1alpha1.Protocol{cosiv1alpha1.ProtocolS3}, - }, - Status: cosiv1alpha1.BucketClaimStatus{ - BucketReady: false, - }, - } - - fakeClient := fake.NewClientBuilder(). - WithScheme(s). - WithObjects(monitor, bc). - WithStatusSubresource(monitor). - Build() - - reconciler := &WorkloadMonitorReconciler{Client: fakeClient, Scheme: s} - req := reconcile.Request{NamespacedName: types.NamespacedName{ - Name: "my-bucket", - Namespace: "tenant-demo", - }} - - _, err := reconciler.Reconcile(context.TODO(), req) - if err != nil { - t.Fatalf("Reconcile returned error: %v", err) - } - - workload := &cozyv1alpha1.Workload{} - err = fakeClient.Get(context.TODO(), types.NamespacedName{ - Name: "bucket-my-bucket", - Namespace: "tenant-demo", - }, workload) - if err != nil { - t.Fatalf("expected Workload to be created, got error: %v", err) - } - - if workload.Status.Operational { - t.Error("expected Operational=false for not-ready BucketClaim") - } -} - -func TestReconcile_MonitorLabelsPropagatedToBucketClaimWorkload(t *testing.T) { - s := newTestScheme() - - monitor := &cozyv1alpha1.WorkloadMonitor{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-bucket", - Namespace: "tenant-demo", - Labels: map[string]string{ - "workloads.cozystack.io/resource-preset": "medium", - "app.kubernetes.io/name": "ignored-not-propagated", - }, - }, - Spec: cozyv1alpha1.WorkloadMonitorSpec{ - Kind: "bucket", - Type: "s3", - Selector: map[string]string{ - "app.kubernetes.io/instance": "my-bucket", - }, - }, - } - - bc := &cosiv1alpha1.BucketClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-bucket", - Namespace: "tenant-demo", - Labels: map[string]string{ - "app.kubernetes.io/instance": "my-bucket", - "app.kubernetes.io/name": "bucket-wins-on-conflict", - }, - }, - Spec: cosiv1alpha1.BucketClaimSpec{ - BucketClassName: "seaweedfs", - Protocols: []cosiv1alpha1.Protocol{cosiv1alpha1.ProtocolS3}, - }, - Status: cosiv1alpha1.BucketClaimStatus{ - BucketReady: true, - BucketName: "cosi-abc123", - }, - } - - fakeClient := fake.NewClientBuilder(). - WithScheme(s). - WithObjects(monitor, bc). - WithStatusSubresource(monitor). - Build() - - reconciler := &WorkloadMonitorReconciler{Client: fakeClient, Scheme: s} - req := reconcile.Request{NamespacedName: types.NamespacedName{ - Name: "my-bucket", - Namespace: "tenant-demo", - }} - if _, err := reconciler.Reconcile(context.TODO(), req); err != nil { - t.Fatalf("Reconcile returned error: %v", err) - } - - workload := &cozyv1alpha1.Workload{} - if err := fakeClient.Get(context.TODO(), types.NamespacedName{ - Name: "bucket-my-bucket", - Namespace: "tenant-demo", - }, workload); err != nil { - t.Fatalf("Failed to get Workload: %v", err) - } - - if got := workload.Labels["workloads.cozystack.io/resource-preset"]; got != "medium" { - t.Errorf("expected monitor label propagated, got %q", got) - } - // Source-object label takes precedence on conflict - if got := workload.Labels["app.kubernetes.io/name"]; got != "bucket-wins-on-conflict" { - t.Errorf("expected bucket claim label to win on conflict, got %q", got) - } - // Reserved monitor label is always set from the monitor name - if got := workload.Labels["workloads.cozystack.io/monitor"]; got != "my-bucket" { - t.Errorf("expected monitor-name label, got %q", got) - } -} - -func TestReconcileNoBucketClaimSkips(t *testing.T) { - s := newTestScheme() - - monitor := &cozyv1alpha1.WorkloadMonitor{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-postgres", - Namespace: "tenant-demo", - }, - Spec: cozyv1alpha1.WorkloadMonitorSpec{ - Kind: "postgres", - Type: "postgres", - Selector: map[string]string{ - "app.kubernetes.io/instance": "my-postgres", - }, - }, - } - - fakeClient := fake.NewClientBuilder(). - WithScheme(s). - WithObjects(monitor). - WithStatusSubresource(monitor). - Build() - - reconciler := &WorkloadMonitorReconciler{Client: fakeClient, Scheme: s} - req := reconcile.Request{NamespacedName: types.NamespacedName{ - Name: "my-postgres", - Namespace: "tenant-demo", - }} - - _, err := reconciler.Reconcile(context.TODO(), req) - if err != nil { - t.Fatalf("Reconcile returned error: %v", err) - } - - workloadList := &cozyv1alpha1.WorkloadList{} - err = fakeClient.List(context.TODO(), workloadList) - if err != nil { - t.Fatalf("List returned error: %v", err) - } - - for _, w := range workloadList.Items { - if w.Status.Kind == "bucket" { - t.Error("expected no bucket workloads to be created for postgres monitor") - } - } -} - -func TestQueryAllBucketMetrics(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[ - {"metric":{"__name__":"SeaweedFS_s3_bucket_size_bytes","bucket":"bucket-aaa"},"value":[1713000000,"10485864"]}, - {"metric":{"__name__":"SeaweedFS_s3_bucket_physical_size_bytes","bucket":"bucket-aaa"},"value":[1713000000,"20971728"]}, - {"metric":{"__name__":"SeaweedFS_s3_bucket_size_bytes","bucket":"bucket-bbb"},"value":[1713000000,"0"]} - ]}}`) - })) - defer srv.Close() - - reconciler := &WorkloadMonitorReconciler{} - metrics := reconciler.queryAllBucketMetrics(context.TODO(), srv.URL, []string{"bucket-aaa", "bucket-bbb"}) - - bm, ok := metrics["bucket-aaa"] - if !ok { - t.Fatal("expected bucket-aaa in metrics") - } - if !bm.HasLogical || bm.LogicalSize != 10485864 { - t.Errorf("expected logical=10485864, got %d", bm.LogicalSize) - } - if !bm.HasPhysical || bm.PhysicalSize != 20971728 { - t.Errorf("expected physical=20971728, got %d", bm.PhysicalSize) - } - - bm2, ok := metrics["bucket-bbb"] - if !ok { - t.Fatal("expected bucket-bbb in metrics") - } - if !bm2.HasLogical || bm2.LogicalSize != 0 { - t.Errorf("expected logical=0 for empty bucket, got %d", bm2.LogicalSize) - } - if bm2.HasPhysical { - t.Error("expected no physical size for bucket-bbb") - } -} - -func TestQueryAllBucketMetricsEmpty(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[]}}`) - })) - defer srv.Close() - - reconciler := &WorkloadMonitorReconciler{} - metrics := reconciler.queryAllBucketMetrics(context.TODO(), srv.URL, []string{"bucket-aaa", "bucket-bbb"}) - if len(metrics) != 0 { - t.Errorf("expected empty metrics, got %d", len(metrics)) - } -} - -func TestQueryAllBucketMetricsServerError(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusInternalServerError) - })) - defer srv.Close() - - reconciler := &WorkloadMonitorReconciler{} - metrics := reconciler.queryAllBucketMetrics(context.TODO(), srv.URL, []string{"bucket-aaa", "bucket-bbb"}) - if len(metrics) != 0 { - t.Errorf("expected empty metrics on error, got %d", len(metrics)) - } -} - -func TestQueryAllBucketMetricsNoURL(t *testing.T) { - reconciler := &WorkloadMonitorReconciler{} - metrics := reconciler.queryAllBucketMetrics(context.TODO(), "", nil) - if len(metrics) != 0 { - t.Errorf("expected empty metrics when URL is empty, got %d", len(metrics)) - } -} - -func TestResolvePrometheusURL(t *testing.T) { - s := newTestScheme() - - ns := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: "tenant-demo", - Labels: map[string]string{ - "namespace.cozystack.io/monitoring": "tenant-root", - }, - }, - } - - fakeClient := fake.NewClientBuilder(). - WithScheme(s). - WithObjects(ns). - Build() - - reconciler := &WorkloadMonitorReconciler{Client: fakeClient, Scheme: s} - url := reconciler.resolvePrometheusURL(context.TODO(), "tenant-demo") - - expected := "http://vmselect-shortterm.tenant-root.svc:8481/select/0/prometheus" - if url != expected { - t.Errorf("expected %q, got %q", expected, url) - } -} - -func TestResolvePrometheusURLNoLabel(t *testing.T) { - s := newTestScheme() - - ns := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: "tenant-demo", - }, - } - - fakeClient := fake.NewClientBuilder(). - WithScheme(s). - WithObjects(ns). - Build() - - reconciler := &WorkloadMonitorReconciler{Client: fakeClient, Scheme: s} - url := reconciler.resolvePrometheusURL(context.TODO(), "tenant-demo") - - if url != "" { - t.Errorf("expected empty URL when no monitoring label, got %q", url) - } -} - -func TestReconcileBucketClaimRequeuesWhenBucketsExist(t *testing.T) { - s := newTestScheme() - - ns := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: "tenant-demo", - }, - } - - monitor := &cozyv1alpha1.WorkloadMonitor{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-bucket", - Namespace: "tenant-demo", - }, - Spec: cozyv1alpha1.WorkloadMonitorSpec{ - Kind: "bucket", - Type: "s3", - Selector: map[string]string{ - "app.kubernetes.io/instance": "my-bucket", - }, - }, - } - - bc := &cosiv1alpha1.BucketClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-bucket", - Namespace: "tenant-demo", - Labels: map[string]string{ - "app.kubernetes.io/instance": "my-bucket", - }, - }, - Spec: cosiv1alpha1.BucketClaimSpec{ - BucketClassName: "seaweedfs", - Protocols: []cosiv1alpha1.Protocol{cosiv1alpha1.ProtocolS3}, - }, - Status: cosiv1alpha1.BucketClaimStatus{ - BucketReady: true, - BucketName: "cosi-abc123", - }, - } - - fakeClient := fake.NewClientBuilder(). - WithScheme(s). - WithObjects(ns, monitor, bc). - WithStatusSubresource(monitor). - Build() - - reconciler := &WorkloadMonitorReconciler{Client: fakeClient, Scheme: s} - req := reconcile.Request{NamespacedName: types.NamespacedName{ - Name: "my-bucket", - Namespace: "tenant-demo", - }} - - result, err := reconciler.Reconcile(context.TODO(), req) - if err != nil { - t.Fatalf("Reconcile returned error: %v", err) - } - - if result.RequeueAfter == 0 { - t.Error("expected RequeueAfter > 0 when buckets exist") - } - - workload := &cozyv1alpha1.Workload{} - err = fakeClient.Get(context.TODO(), types.NamespacedName{ - Name: "bucket-my-bucket", - Namespace: "tenant-demo", - }, workload) - if err != nil { - t.Fatalf("expected Workload to be created, got error: %v", err) - } - - // Without monitoring label on namespace, no size metrics should be set - if _, ok := workload.Status.Resources["s3-storage-bytes"]; ok { - t.Error("expected no s3-storage-bytes when monitoring is not configured") - } - if len(workload.Status.Resources) != 0 { - t.Errorf("expected empty resources without monitoring, got %v", workload.Status.Resources) - } -} diff --git a/internal/crdinstall/manifests/cozystack.io_packagesources.yaml b/internal/crdinstall/manifests/cozystack.io_packagesources.yaml index 1d0037df..0acfcdd2 100644 --- a/internal/crdinstall/manifests/cozystack.io_packagesources.yaml +++ b/internal/crdinstall/manifests/cozystack.io_packagesources.yaml @@ -118,19 +118,6 @@ spec: ReleaseName is the name of the HelmRelease resource that will be created If not specified, defaults to the component Name field type: string - upgradeCRDs: - description: |- - UpgradeCRDs controls how CRDs from the chart's crds/ directory are - handled on HelmRelease upgrades. Maps to HelmRelease.Spec.Upgrade.CRDs. - Empty string (default) preserves the helm-controller default (Skip). - Use "CreateReplace" for operators that evolve their CRD set between - versions. Warning: CreateReplace overwrites CRDs and may cause data - loss if upstream drops fields from a CRD with live objects. - enum: - - Skip - - Create - - CreateReplace - type: string type: object libraries: description: |- diff --git a/internal/operator/package_reconciler.go b/internal/operator/package_reconciler.go index 0e724e49..32e667e4 100644 --- a/internal/operator/package_reconciler.go +++ b/internal/operator/package_reconciler.go @@ -45,16 +45,6 @@ const ( SecretCozystackValues = "cozystack-values" ) -// parseCRDPolicy maps ComponentInstall.UpgradeCRDs to a helmv2.CRDsPolicy. -// Empty / nil preserves the helm-controller default (Skip on upgrade); -// the CRD enum marker restricts the string to Skip/Create/CreateReplace. -func parseCRDPolicy(install *cozyv1alpha1.ComponentInstall) helmv2.CRDsPolicy { - if install == nil || install.UpgradeCRDs == "" { - return "" - } - return helmv2.CRDsPolicy(install.UpgradeCRDs) -} - // PackageReconciler reconciles Package resources type PackageReconciler struct { client.Client @@ -231,7 +221,6 @@ func (r *PackageReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct Remediation: &helmv2.UpgradeRemediation{ Retries: -1, }, - CRDs: parseCRDPolicy(component.Install), }, }, } diff --git a/internal/operator/package_reconciler_test.go b/internal/operator/package_reconciler_test.go deleted file mode 100644 index f0ee2c19..00000000 --- a/internal/operator/package_reconciler_test.go +++ /dev/null @@ -1,138 +0,0 @@ -/* -Copyright 2025 The Cozystack Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package operator - -import ( - "encoding/json" - "os" - "path/filepath" - "testing" - - cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" - helmv2 "github.com/fluxcd/helm-controller/api/v2" - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - "sigs.k8s.io/yaml" -) - -func TestParseCRDPolicy(t *testing.T) { - tests := []struct { - name string - install *cozyv1alpha1.ComponentInstall - want helmv2.CRDsPolicy - }{ - { - name: "nil install leaves flux default", - install: nil, - want: "", - }, - { - name: "empty upgradeCRDs leaves flux default", - install: &cozyv1alpha1.ComponentInstall{}, - want: "", - }, - { - name: "Skip is passed through", - install: &cozyv1alpha1.ComponentInstall{UpgradeCRDs: "Skip"}, - want: helmv2.Skip, - }, - { - name: "Create is passed through", - install: &cozyv1alpha1.ComponentInstall{UpgradeCRDs: "Create"}, - want: helmv2.Create, - }, - { - name: "CreateReplace is passed through", - install: &cozyv1alpha1.ComponentInstall{UpgradeCRDs: "CreateReplace"}, - want: helmv2.CreateReplace, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - got := parseCRDPolicy(tc.install) - if got != tc.want { - t.Errorf("parseCRDPolicy() = %q, want %q", got, tc.want) - } - }) - } -} - -// TestPackageSourceCRDHasUpgradeCRDsEnum guards the generated CRD schema: the -// invalid-value case from the spec is enforced at the API server via a -// kubebuilder enum marker, not in the reconciler. If someone drops the marker -// and forgets to regenerate, this test catches it. -func TestPackageSourceCRDHasUpgradeCRDsEnum(t *testing.T) { - path := filepath.Join("..", "crdinstall", "manifests", "cozystack.io_packagesources.yaml") - data, err := os.ReadFile(path) - if err != nil { - t.Fatalf("read %s: %v", path, err) - } - - var crd apiextensionsv1.CustomResourceDefinition - if err := yaml.Unmarshal(data, &crd); err != nil { - t.Fatalf("unmarshal CRD: %v", err) - } - - var field *apiextensionsv1.JSONSchemaProps - for i := range crd.Spec.Versions { - v := &crd.Spec.Versions[i] - if v.Schema == nil || v.Schema.OpenAPIV3Schema == nil { - continue - } - spec, ok := v.Schema.OpenAPIV3Schema.Properties["spec"] - if !ok { - continue - } - variants, ok := spec.Properties["variants"] - if !ok || variants.Items == nil || variants.Items.Schema == nil { - continue - } - components, ok := variants.Items.Schema.Properties["components"] - if !ok || components.Items == nil || components.Items.Schema == nil { - continue - } - install, ok := components.Items.Schema.Properties["install"] - if !ok { - continue - } - f, ok := install.Properties["upgradeCRDs"] - if !ok { - continue - } - field = &f - break - } - - if field == nil { - t.Fatal("upgradeCRDs field not found in PackageSource CRD schema") - } - - got := map[string]bool{} - for _, e := range field.Enum { - var s string - if err := json.Unmarshal(e.Raw, &s); err != nil { - t.Fatalf("unmarshal enum value %q: %v", e.Raw, err) - } - got[s] = true - } - - for _, want := range []string{"Skip", "Create", "CreateReplace"} { - if !got[want] { - t.Errorf("enum value %q missing from upgradeCRDs; got %v", want, got) - } - } -} diff --git a/packages/apps/bucket/templates/bucketclaim.yaml b/packages/apps/bucket/templates/bucketclaim.yaml index 0639916f..2f57fd70 100644 --- a/packages/apps/bucket/templates/bucketclaim.yaml +++ b/packages/apps/bucket/templates/bucketclaim.yaml @@ -4,8 +4,6 @@ apiVersion: objectstorage.k8s.io/v1alpha1 kind: BucketClaim metadata: name: {{ .Release.Name }} - labels: - app.kubernetes.io/instance: {{ .Release.Name }} spec: bucketClassName: {{ $seaweedfs }}{{- if $pool }}-{{ $pool }}{{- end }}{{- if .Values.locking }}-lock{{- end }} protocols: diff --git a/packages/apps/bucket/templates/workloadmonitor.yaml b/packages/apps/bucket/templates/workloadmonitor.yaml deleted file mode 100644 index a23f3147..00000000 --- a/packages/apps/bucket/templates/workloadmonitor.yaml +++ /dev/null @@ -1,13 +0,0 @@ ---- -apiVersion: cozystack.io/v1alpha1 -kind: WorkloadMonitor -metadata: - name: {{ $.Release.Name }} -spec: - replicas: 0 - minReplicas: 0 - kind: bucket - type: s3 - selector: - app.kubernetes.io/instance: {{ $.Release.Name }} - version: {{ $.Chart.Version }} diff --git a/packages/apps/clickhouse/templates/workloadmonitor.yaml b/packages/apps/clickhouse/templates/workloadmonitor.yaml index 62c951c2..9020ad41 100644 --- a/packages/apps/clickhouse/templates/workloadmonitor.yaml +++ b/packages/apps/clickhouse/templates/workloadmonitor.yaml @@ -3,8 +3,6 @@ apiVersion: cozystack.io/v1alpha1 kind: WorkloadMonitor metadata: name: {{ $.Release.Name }} - labels: - workloads.cozystack.io/resource-preset: {{ .Values.resourcesPreset | quote }} spec: replicas: {{ .Values.replicas }} minReplicas: 1 @@ -19,8 +17,6 @@ apiVersion: cozystack.io/v1alpha1 kind: WorkloadMonitor metadata: name: {{ $.Release.Name }}-keeper - labels: - workloads.cozystack.io/resource-preset: {{ .Values.clickhouseKeeper.resourcesPreset | quote }} spec: replicas: {{ .Values.clickhouseKeeper.replicas }} minReplicas: 1 diff --git a/packages/apps/foundationdb/templates/workloadmonitor.yaml b/packages/apps/foundationdb/templates/workloadmonitor.yaml index 2e2a3d19..306797f3 100644 --- a/packages/apps/foundationdb/templates/workloadmonitor.yaml +++ b/packages/apps/foundationdb/templates/workloadmonitor.yaml @@ -8,7 +8,6 @@ metadata: app.kubernetes.io/name: foundationdb app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} - workloads.cozystack.io/resource-preset: {{ .Values.resourcesPreset | quote }} spec: replicas: {{ .Values.cluster.processCounts.storage }} minReplicas: {{ include "foundationdb.minReplicas" . }} diff --git a/packages/apps/harbor/templates/harbor.yaml b/packages/apps/harbor/templates/harbor.yaml index bf780967..ff8e88ad 100644 --- a/packages/apps/harbor/templates/harbor.yaml +++ b/packages/apps/harbor/templates/harbor.yaml @@ -158,8 +158,6 @@ apiVersion: cozystack.io/v1alpha1 kind: WorkloadMonitor metadata: name: {{ $.Release.Name }}-core - labels: - workloads.cozystack.io/resource-preset: {{ .Values.core.resourcesPreset | quote }} spec: replicas: 1 minReplicas: 1 @@ -176,8 +174,6 @@ apiVersion: cozystack.io/v1alpha1 kind: WorkloadMonitor metadata: name: {{ $.Release.Name }}-registry - labels: - workloads.cozystack.io/resource-preset: {{ .Values.registry.resourcesPreset | quote }} spec: replicas: 1 minReplicas: 1 diff --git a/packages/apps/harbor/templates/ingress.yaml b/packages/apps/harbor/templates/ingress.yaml index 70933f5f..bb6ef07c 100644 --- a/packages/apps/harbor/templates/ingress.yaml +++ b/packages/apps/harbor/templates/ingress.yaml @@ -15,7 +15,7 @@ metadata: nginx.ingress.kubernetes.io/ssl-redirect: "true" nginx.ingress.kubernetes.io/backend-protocol: "HTTP" {{- if eq $solver "http01" }} - acme.cert-manager.io/http01-ingress-ingressclassname: {{ $ingress }} + acme.cert-manager.io/http01-ingress-class: {{ $ingress }} {{- end }} cert-manager.io/cluster-issuer: {{ $clusterIssuer }} spec: diff --git a/packages/apps/http-cache/images/nginx-cache.tag b/packages/apps/http-cache/images/nginx-cache.tag index b08a374e..1da929ec 100644 --- a/packages/apps/http-cache/images/nginx-cache.tag +++ b/packages/apps/http-cache/images/nginx-cache.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/nginx-cache:0.0.0@sha256:d397781152ab9123b11b8191d92eba7a0d2faa376aa2c15ddeb67842a9b59bab +ghcr.io/cozystack/cozystack/nginx-cache:0.0.0@sha256:2f987017ff95d3c782e16ef0d99928831f520daf154d08a2fa38c2d68363e036 diff --git a/packages/apps/http-cache/templates/workloadmonitor.yaml b/packages/apps/http-cache/templates/workloadmonitor.yaml index a38a395a..150d8bbe 100644 --- a/packages/apps/http-cache/templates/workloadmonitor.yaml +++ b/packages/apps/http-cache/templates/workloadmonitor.yaml @@ -3,8 +3,6 @@ apiVersion: cozystack.io/v1alpha1 kind: WorkloadMonitor metadata: name: {{ $.Release.Name }}-haproxy - labels: - workloads.cozystack.io/resource-preset: {{ .Values.haproxy.resourcesPreset | quote }} spec: replicas: {{ .Values.haproxy.replicas }} minReplicas: 1 @@ -18,8 +16,6 @@ apiVersion: cozystack.io/v1alpha1 kind: WorkloadMonitor metadata: name: {{ $.Release.Name }}-nginx - labels: - workloads.cozystack.io/resource-preset: {{ .Values.nginx.resourcesPreset | quote }} spec: replicas: {{ .Values.nginx.replicas }} minReplicas: 1 diff --git a/packages/apps/kafka/templates/workloadmonitor.yaml b/packages/apps/kafka/templates/workloadmonitor.yaml index c31eb425..4b161b04 100644 --- a/packages/apps/kafka/templates/workloadmonitor.yaml +++ b/packages/apps/kafka/templates/workloadmonitor.yaml @@ -3,8 +3,6 @@ apiVersion: cozystack.io/v1alpha1 kind: WorkloadMonitor metadata: name: {{ $.Release.Name }} - labels: - workloads.cozystack.io/resource-preset: {{ .Values.kafka.resourcesPreset | quote }} spec: replicas: {{ .Values.replicas }} minReplicas: 1 @@ -21,8 +19,6 @@ apiVersion: cozystack.io/v1alpha1 kind: WorkloadMonitor metadata: name: {{ $.Release.Name }}-zookeeper - labels: - workloads.cozystack.io/resource-preset: {{ .Values.zookeeper.resourcesPreset | quote }} spec: replicas: {{ .Values.replicas }} minReplicas: 1 diff --git a/packages/apps/kubernetes/Makefile b/packages/apps/kubernetes/Makefile index 4fea42ef..01cf736d 100644 --- a/packages/apps/kubernetes/Makefile +++ b/packages/apps/kubernetes/Makefile @@ -4,9 +4,6 @@ KUBERNETES_PKG_TAG = $(shell awk '$$1 == "version:" {print $$2}' Chart.yaml) include ../../../hack/common-envs.mk include ../../../hack/package.mk -test: - helm unittest . - generate: cozyvalues-gen -m 'kubernetes' -v values.yaml -s values.schema.json -r README.md -g ../../../api/apps/v1alpha1/kubernetes/types.go ../../../hack/update-crd.sh @@ -70,4 +67,3 @@ image-cluster-autoscaler: echo "$(REGISTRY)/cluster-autoscaler:$(call settag,$(KUBERNETES_PKG_TAG))@$$(yq e '."containerimage.digest"' images/cluster-autoscaler.json -o json -r)" \ > images/cluster-autoscaler.tag rm -f images/cluster-autoscaler.json - diff --git a/packages/apps/kubernetes/README.md b/packages/apps/kubernetes/README.md index bed1117a..d62d3a39 100644 --- a/packages/apps/kubernetes/README.md +++ b/packages/apps/kubernetes/README.md @@ -128,9 +128,6 @@ See the reference for components utilized in this service: | `addons.gpuOperator` | NVIDIA GPU Operator. | `object` | `{}` | | `addons.gpuOperator.enabled` | Enable GPU Operator. | `bool` | `false` | | `addons.gpuOperator.valuesOverride` | Custom Helm values overrides. | `object` | `{}` | -| `addons.hami` | HAMi GPU virtualization middleware. | `object` | `{}` | -| `addons.hami.enabled` | Enable HAMi (requires GPU Operator). | `bool` | `false` | -| `addons.hami.valuesOverride` | Custom Helm values overrides. | `object` | `{}` | | `addons.fluxcd` | FluxCD GitOps operator. | `object` | `{}` | | `addons.fluxcd.enabled` | Enable FluxCD. | `bool` | `false` | | `addons.fluxcd.valuesOverride` | Custom Helm values overrides. | `object` | `{}` | @@ -148,33 +145,31 @@ See the reference for components utilized in this service: ### Kubernetes Control Plane Configuration -| Name | Description | Type | Value | -| --------------------------------------------------- | --------------------------------------------------------------------------------------------- | ---------- | ------- | -| `controlPlane` | Kubernetes control-plane configuration. | `object` | `{}` | -| `controlPlane.replicas` | Number of control-plane replicas. | `int` | `2` | -| `controlPlane.apiServer` | API Server configuration. | `object` | `{}` | -| `controlPlane.apiServer.resources` | CPU and memory resources for API Server. | `object` | `{}` | -| `controlPlane.apiServer.resources.cpu` | CPU available. | `quantity` | `""` | -| `controlPlane.apiServer.resources.memory` | Memory (RAM) available. | `quantity` | `""` | -| `controlPlane.apiServer.resourcesPreset` | Preset if `resources` omitted. | `string` | `large` | -| `controlPlane.controllerManager` | Controller Manager configuration. | `object` | `{}` | -| `controlPlane.controllerManager.resources` | CPU and memory resources for Controller Manager. | `object` | `{}` | -| `controlPlane.controllerManager.resources.cpu` | CPU available. | `quantity` | `""` | -| `controlPlane.controllerManager.resources.memory` | Memory (RAM) available. | `quantity` | `""` | -| `controlPlane.controllerManager.resourcesPreset` | Preset if `resources` omitted. | `string` | `micro` | -| `controlPlane.scheduler` | Scheduler configuration. | `object` | `{}` | -| `controlPlane.scheduler.resources` | CPU and memory resources for Scheduler. | `object` | `{}` | -| `controlPlane.scheduler.resources.cpu` | CPU available. | `quantity` | `""` | -| `controlPlane.scheduler.resources.memory` | Memory (RAM) available. | `quantity` | `""` | -| `controlPlane.scheduler.resourcesPreset` | Preset if `resources` omitted. | `string` | `micro` | -| `controlPlane.konnectivity` | Konnectivity configuration. | `object` | `{}` | -| `controlPlane.konnectivity.server` | Konnectivity Server configuration. | `object` | `{}` | -| `controlPlane.konnectivity.server.resources` | CPU and memory resources for Konnectivity. | `object` | `{}` | -| `controlPlane.konnectivity.server.resources.cpu` | CPU available. | `quantity` | `""` | -| `controlPlane.konnectivity.server.resources.memory` | Memory (RAM) available. | `quantity` | `""` | -| `controlPlane.konnectivity.server.resourcesPreset` | Preset if `resources` omitted. | `string` | `micro` | -| `images` | Optional image overrides for air-gapped or rate-limited registries. | `object` | `{}` | -| `images.waitForKubeconfig` | Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag. | `string` | `""` | +| Name | Description | Type | Value | +| --------------------------------------------------- | ------------------------------------------------ | ---------- | ------- | +| `controlPlane` | Kubernetes control-plane configuration. | `object` | `{}` | +| `controlPlane.replicas` | Number of control-plane replicas. | `int` | `2` | +| `controlPlane.apiServer` | API Server configuration. | `object` | `{}` | +| `controlPlane.apiServer.resources` | CPU and memory resources for API Server. | `object` | `{}` | +| `controlPlane.apiServer.resources.cpu` | CPU available. | `quantity` | `""` | +| `controlPlane.apiServer.resources.memory` | Memory (RAM) available. | `quantity` | `""` | +| `controlPlane.apiServer.resourcesPreset` | Preset if `resources` omitted. | `string` | `large` | +| `controlPlane.controllerManager` | Controller Manager configuration. | `object` | `{}` | +| `controlPlane.controllerManager.resources` | CPU and memory resources for Controller Manager. | `object` | `{}` | +| `controlPlane.controllerManager.resources.cpu` | CPU available. | `quantity` | `""` | +| `controlPlane.controllerManager.resources.memory` | Memory (RAM) available. | `quantity` | `""` | +| `controlPlane.controllerManager.resourcesPreset` | Preset if `resources` omitted. | `string` | `micro` | +| `controlPlane.scheduler` | Scheduler configuration. | `object` | `{}` | +| `controlPlane.scheduler.resources` | CPU and memory resources for Scheduler. | `object` | `{}` | +| `controlPlane.scheduler.resources.cpu` | CPU available. | `quantity` | `""` | +| `controlPlane.scheduler.resources.memory` | Memory (RAM) available. | `quantity` | `""` | +| `controlPlane.scheduler.resourcesPreset` | Preset if `resources` omitted. | `string` | `micro` | +| `controlPlane.konnectivity` | Konnectivity configuration. | `object` | `{}` | +| `controlPlane.konnectivity.server` | Konnectivity Server configuration. | `object` | `{}` | +| `controlPlane.konnectivity.server.resources` | CPU and memory resources for Konnectivity. | `object` | `{}` | +| `controlPlane.konnectivity.server.resources.cpu` | CPU available. | `quantity` | `""` | +| `controlPlane.konnectivity.server.resources.memory` | Memory (RAM) available. | `quantity` | `""` | +| `controlPlane.konnectivity.server.resourcesPreset` | Preset if `resources` omitted. | `string` | `micro` | ## Parameter examples and reference diff --git a/packages/apps/kubernetes/images/busybox.tag b/packages/apps/kubernetes/images/busybox.tag deleted file mode 100644 index 39de220a..00000000 --- a/packages/apps/kubernetes/images/busybox.tag +++ /dev/null @@ -1 +0,0 @@ -docker.io/library/busybox:1.37.0@sha256:1487d0af5f52b4ba31c7e465126ee2123fe3f2305d638e7827681e7cf6c83d5e diff --git a/packages/apps/kubernetes/images/kubevirt-csi-driver.tag b/packages/apps/kubernetes/images/kubevirt-csi-driver.tag index 00aa2caa..e623b778 100644 --- a/packages/apps/kubernetes/images/kubevirt-csi-driver.tag +++ b/packages/apps/kubernetes/images/kubevirt-csi-driver.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/kubevirt-csi-driver:0.0.0@sha256:72154a97054e16cdf3dea6129d962b8d7e86b55cf9386095e8ac2ce7c8b69172 +ghcr.io/cozystack/cozystack/kubevirt-csi-driver:0.0.0@sha256:5b209248acba3d8cf0d999edd8a915915a2e565a1c8ac1f5dde0dffec20fa02e diff --git a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.30.tag b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.30.tag index f49fffe6..2d1e4d52 100644 --- a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.30.tag +++ b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.30.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.30@sha256:f460ed1fd5a721aed423e6c3b8be0105d7f693cd86ad992d9c15fd4b27e58cec +ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.30@sha256:51e272db23d64eb20bc44e38cdbb70199fd00a49a958b76c2a89d4a46fec9256 diff --git a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.31.tag b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.31.tag index b5bdc79a..9698df5c 100644 --- a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.31.tag +++ b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.31.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.31@sha256:aaa2dfa8ee53ae26295f44d2491330f51412457bbe4aa6ba256297cc0bd8a0da +ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.31@sha256:31c3290305159fe484caf6c5780960cf071dc3939528295336da4abfa731a81b diff --git a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.32.tag b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.32.tag index ca285644..90545820 100644 --- a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.32.tag +++ b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.32.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.32@sha256:7acb4728ef6b9c0ff3344bf486bb00f2a083f2098452344a362242235013db01 +ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.32@sha256:9efb1fe18cffb045530b43ac4b39910f1802d3c16798d733165dcd1c67de9237 diff --git a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.33.tag b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.33.tag index bb58690a..64345fab 100644 --- a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.33.tag +++ b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.33.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.33@sha256:a2024339ab9edb980a96b43ad2744b7aa31afaef0983ddbc6a546068b4cfa87a +ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.33@sha256:a0afca1d6f720cfff764a1bc912bfe404b5e97ff731f6c126a7abdb51ed23f85 diff --git a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.34.tag b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.34.tag index 36eed7a7..f6916755 100644 --- a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.34.tag +++ b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.34.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.34@sha256:97bcf946a5687889c6a421d14e5adece11f0978151b2165dc87b28f77f7607db +ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.34@sha256:51dfe5ce3e2f8765fde9422ae7e9fba677016ec5e2be41559c5f11a56958b0e6 diff --git a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.35.tag b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.35.tag index faae83d2..b80ce821 100644 --- a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.35.tag +++ b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.35.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.35@sha256:364c6d454891f1eb1a598fddb69cf328a14dbc451a8ac65812b038a7756da60a +ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.35@sha256:c4ae418b2a2c139794cd9630c3b2c2171870cc7748ac200344d2c4ed4283cf0b diff --git a/packages/apps/kubernetes/templates/_helpers.tpl b/packages/apps/kubernetes/templates/_helpers.tpl index 40a8ae7a..36c06b64 100644 --- a/packages/apps/kubernetes/templates/_helpers.tpl +++ b/packages/apps/kubernetes/templates/_helpers.tpl @@ -49,52 +49,3 @@ Selector labels app.kubernetes.io/name: {{ include "kubernetes.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} - -{{/* -wait-for-kubeconfig init container shared by the control-plane-side -Deployments (cluster-autoscaler, kccm, kcsi-controller) that mount the -*-admin-kubeconfig Secret provisioned asynchronously by Kamaji. The -Secret volume is declared optional so kubelet does not FailedMount while -Kamaji is still bootstrapping; this container polls the mounted path and -exits only when super-admin.svc appears, which happens after kubelet's -optional-Secret refresh cycle. - -The 10m deadline stays strictly below the 15m HelmRelease -Install.Timeout set by cozystack-api for the Kubernetes kind (via the -release.cozystack.io/helm-install-timeout annotation) so the -CrashLoopBackOff surfaces before flux remediation fires and uninstalls -the Cluster CR. - -The default image lives in images/busybox.tag and points directly at -docker.io by digest (not mirrored to ghcr.io like the other .tag files -here): the payload is a one-shot sh loop and the digest pin makes the -pull immutable. Operators in air-gapped or rate-limited environments -can override it via .Values.images.waitForKubeconfig (any registry -reference kubelet can pull). When the value is empty the chart falls -back to the bundled digest pin, preserving the prior default. - -Call site owns the surrounding volumes block; the kubeconfig volume -must exist on the pod and mount at /etc/kubernetes/kubeconfig. -*/}} -{{- define "kubernetes.waitForAdminKubeconfig" -}} -- name: wait-for-kubeconfig - image: "{{ default (.Files.Get "images/busybox.tag" | trim) .Values.images.waitForKubeconfig }}" - command: - - sh - - -c - - | - set -eu - deadline=$(( $(date +%s) + 600 )) - until [ -s /etc/kubernetes/kubeconfig/super-admin.svc ]; do - if [ "$(date +%s)" -ge "$deadline" ]; then - echo "admin kubeconfig was not provisioned within 10m; exiting so the pod goes CrashLoopBackOff and surfaces in dashboards" >&2 - exit 1 - fi - echo "waiting for admin kubeconfig (provisioned by Kamaji, visible after kubelet Secret refresh)..." - sleep 5 - done - volumeMounts: - - name: kubeconfig - mountPath: /etc/kubernetes/kubeconfig - readOnly: true -{{- end }} diff --git a/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml b/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml index 298d86db..a00e0155 100644 --- a/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml +++ b/packages/apps/kubernetes/templates/cluster-autoscaler/deployment.yaml @@ -1,14 +1,3 @@ -{{- /* - Gate the control-plane-side workloads on the parent tenant having an etcd - DataStore. Without it no KamajiControlPlane is ever created, Kamaji never - provisions -admin-kubeconfig, and rendering these Deployments would cause - the wait-for-kubeconfig init to CrashLoopBackOff indefinitely, consuming - the parent HelmRelease install timeout and triggering the very uninstall - remediation cycle this chart is supposed to avoid. Rendering them only - when $etcd is set keeps the HelmRelease Ready while flux retries on its - interval and picks up the DataStore as soon as the Tenant chart finishes. -*/}} -{{- if .Values._namespace.etcd }} --- apiVersion: apps/v1 kind: Deployment @@ -34,8 +23,6 @@ spec: - key: node-role.kubernetes.io/control-plane operator: Exists effect: "NoSchedule" - initContainers: - {{- include "kubernetes.waitForAdminKubeconfig" $ | nindent 6 }} containers: - image: "{{ $.Files.Get "images/cluster-autoscaler.tag" | trim }}" name: cluster-autoscaler @@ -69,7 +56,6 @@ spec: name: cloud-config - secret: secretName: {{ .Release.Name }}-admin-kubeconfig - optional: true name: kubeconfig serviceAccountName: {{ .Release.Name }}-cluster-autoscaler terminationGracePeriodSeconds: 10 @@ -119,4 +105,3 @@ rules: - list - update - watch -{{- end }} diff --git a/packages/apps/kubernetes/templates/cluster.yaml b/packages/apps/kubernetes/templates/cluster.yaml index e1494e64..10d6fd80 100644 --- a/packages/apps/kubernetes/templates/cluster.yaml +++ b/packages/apps/kubernetes/templates/cluster.yaml @@ -1,15 +1,4 @@ {{- $etcd := .Values._namespace.etcd }} -{{- /* - When $etcd is empty, the parent Tenant application has not populated - _namespace.etcd in cozystack-values yet - either the operator forgot to - set etcd: true on an ancestor Tenant, or the Tenant HelmRelease is still - reconciling. Either way, rendering a KamajiControlPlane with an empty - dataStoreName would be rejected by Kamaji's admission webhook and the - HelmRelease would fail to install, triggering remediation. Instead, emit - a single ConfigMap as a user-visible status beacon and skip the rest so - flux marks the HelmRelease Ready and retries its 5m reconcile loop until - the Tenant chart catches up. -*/}} {{- $ingress := .Values._namespace.ingress }} {{- $host := .Values._namespace.host }} {{- $kubevirtmachinetemplateNames := list }} @@ -95,26 +84,6 @@ spec: - name: default pod: {} {{- end }} -{{- if not $etcd }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ .Release.Name }}-awaiting-etcd - namespace: {{ .Release.Namespace }} - labels: - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/name: kubernetes - app.kubernetes.io/instance: {{ .Release.Name }} -data: - status: "awaiting-etcd" - message: | - No DataStore is available for this tenant Kubernetes cluster. The parent - Tenant application has not populated _namespace.etcd. Set spec.etcd: true - on an ancestor Tenant (usually tenant-root) and wait for its HelmRelease - to reconcile - this HelmRelease will pick up the DataStore on its next - 5m reconcile loop and provision the cluster. -{{- else }} --- apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster @@ -435,4 +404,3 @@ metadata: spec: {{- .spec | toYaml | nindent 2 }} {{- end }} -{{- end }} diff --git a/packages/apps/kubernetes/templates/csi/deploy.yaml b/packages/apps/kubernetes/templates/csi/deploy.yaml index de62104c..938b6d67 100644 --- a/packages/apps/kubernetes/templates/csi/deploy.yaml +++ b/packages/apps/kubernetes/templates/csi/deploy.yaml @@ -1,4 +1,3 @@ -{{- if .Values._namespace.etcd }} kind: Deployment apiVersion: apps/v1 metadata: @@ -25,8 +24,6 @@ spec: - key: node-role.kubernetes.io/control-plane operator: Exists effect: "NoSchedule" - initContainers: - {{- include "kubernetes.waitForAdminKubeconfig" $ | nindent 6 }} containers: - name: csi-driver imagePullPolicy: Always @@ -237,6 +234,4 @@ spec: emptyDir: {} - secret: secretName: {{ .Release.Name }}-admin-kubeconfig - optional: true name: kubeconfig -{{- end }} diff --git a/packages/apps/kubernetes/templates/helmreleases/cert-manager-crds.yaml b/packages/apps/kubernetes/templates/helmreleases/cert-manager-crds.yaml index fd9dac7e..be07a8b9 100644 --- a/packages/apps/kubernetes/templates/helmreleases/cert-manager-crds.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/cert-manager-crds.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.addons.certManager.enabled .Values._namespace.etcd }} +{{- if .Values.addons.certManager.enabled }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/packages/apps/kubernetes/templates/helmreleases/cert-manager.yaml b/packages/apps/kubernetes/templates/helmreleases/cert-manager.yaml index 700b666e..6857581a 100644 --- a/packages/apps/kubernetes/templates/helmreleases/cert-manager.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/cert-manager.yaml @@ -8,7 +8,7 @@ cert-manager: {{- end }} {{- end }} -{{- if and .Values.addons.certManager.enabled .Values._namespace.etcd }} +{{- if .Values.addons.certManager.enabled }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/packages/apps/kubernetes/templates/helmreleases/cilium.yaml b/packages/apps/kubernetes/templates/helmreleases/cilium.yaml index d8c90cbf..d032f5b6 100644 --- a/packages/apps/kubernetes/templates/helmreleases/cilium.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/cilium.yaml @@ -14,7 +14,6 @@ cilium: {{- end }} {{- end }} -{{- if .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: @@ -56,4 +55,3 @@ spec: - name: {{ .Release.Name }}-gateway-api-crds namespace: {{ .Release.Namespace }} {{- end }} -{{- end }} diff --git a/packages/apps/kubernetes/templates/helmreleases/coredns.yaml b/packages/apps/kubernetes/templates/helmreleases/coredns.yaml index 0711a51d..bdb6c682 100644 --- a/packages/apps/kubernetes/templates/helmreleases/coredns.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/coredns.yaml @@ -4,7 +4,6 @@ coredns: clusterIP: "10.95.0.10" {{- end }} -{{- if .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: @@ -43,4 +42,3 @@ spec: {{- end }} - name: {{ .Release.Name }}-cilium namespace: {{ .Release.Namespace }} -{{- end }} diff --git a/packages/apps/kubernetes/templates/helmreleases/csi.yaml b/packages/apps/kubernetes/templates/helmreleases/csi.yaml index 109d78e1..dd2c69a6 100644 --- a/packages/apps/kubernetes/templates/helmreleases/csi.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/csi.yaml @@ -1,4 +1,3 @@ -{{- if .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: @@ -40,4 +39,3 @@ spec: - name: {{ .Release.Name }} namespace: {{ .Release.Namespace }} {{- end }} -{{- end }} diff --git a/packages/apps/kubernetes/templates/helmreleases/fluxcd.yaml b/packages/apps/kubernetes/templates/helmreleases/fluxcd.yaml index 25fff01c..76499dfe 100644 --- a/packages/apps/kubernetes/templates/helmreleases/fluxcd.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/fluxcd.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.addons.fluxcd.enabled .Values._namespace.etcd }} +{{- if .Values.addons.fluxcd.enabled }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/packages/apps/kubernetes/templates/helmreleases/gateway-api-crds.yaml b/packages/apps/kubernetes/templates/helmreleases/gateway-api-crds.yaml index b4172ed1..2bcc8d4d 100644 --- a/packages/apps/kubernetes/templates/helmreleases/gateway-api-crds.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/gateway-api-crds.yaml @@ -1,4 +1,4 @@ -{{- if and $.Values.addons.gatewayAPI.enabled $.Values._namespace.etcd }} +{{- if $.Values.addons.gatewayAPI.enabled }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml b/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml index 655dc868..5ef48912 100644 --- a/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/gpu-operator.yaml @@ -1,12 +1,4 @@ -{{- define "cozystack.defaultGpuOperatorValues" -}} -{{- if .Values.addons.hami.enabled }} -gpu-operator: - devicePlugin: - enabled: false -{{- end }} -{{- end }} - -{{- if and .Values.addons.gpuOperator.enabled .Values._namespace.etcd }} +{{- if .Values.addons.gpuOperator.enabled }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: @@ -37,12 +29,9 @@ spec: force: true remediation: retries: -1 - {{- $defaults := fromYaml (include "cozystack.defaultGpuOperatorValues" .) }} - {{- $overrides := deepCopy (default (dict) .Values.addons.gpuOperator.valuesOverride) }} - {{- $merged := mergeOverwrite (default (dict) $defaults) $overrides }} - {{- if $merged }} + {{- with .Values.addons.gpuOperator.valuesOverride }} values: - {{- toYaml $merged | nindent 4 }} + {{- toYaml . | nindent 4 }} {{- end }} dependsOn: diff --git a/packages/apps/kubernetes/templates/helmreleases/hami.yaml b/packages/apps/kubernetes/templates/helmreleases/hami.yaml deleted file mode 100644 index f1538c7c..00000000 --- a/packages/apps/kubernetes/templates/helmreleases/hami.yaml +++ /dev/null @@ -1,49 +0,0 @@ -{{- if and .Values.addons.hami.enabled .Values._namespace.etcd }} -{{- if not .Values.addons.gpuOperator.enabled }} -{{- fail "addons.hami requires addons.gpuOperator to be enabled" }} -{{- end }} -apiVersion: helm.toolkit.fluxcd.io/v2 -kind: HelmRelease -metadata: - name: {{ .Release.Name }}-hami - labels: - cozystack.io/repository: system - cozystack.io/target-cluster-name: {{ .Release.Name }} - sharding.fluxcd.io/key: tenants -spec: - releaseName: hami - chartRef: - kind: ExternalArtifact - name: cozystack-kubernetes-application-kubevirt-kubernetes-hami - namespace: cozy-system - kubeConfig: - secretRef: - name: {{ .Release.Name }}-admin-kubeconfig - key: super-admin.svc - targetNamespace: cozy-hami - storageNamespace: cozy-hami - interval: 5m - timeout: 10m - install: - createNamespace: true - remediation: - retries: -1 - upgrade: - force: true - remediation: - retries: -1 - {{- with .Values.addons.hami.valuesOverride }} - values: - {{- toYaml . | nindent 4 }} - {{- end }} - - dependsOn: - {{- if lookup "helm.toolkit.fluxcd.io/v2" "HelmRelease" .Release.Namespace .Release.Name }} - - name: {{ .Release.Name }} - namespace: {{ .Release.Namespace }} - {{- end }} - - name: {{ .Release.Name }}-cilium - namespace: {{ .Release.Namespace }} - - name: {{ .Release.Name }}-gpu-operator - namespace: {{ .Release.Namespace }} -{{- end }} diff --git a/packages/apps/kubernetes/templates/helmreleases/ingress-nginx.yaml b/packages/apps/kubernetes/templates/helmreleases/ingress-nginx.yaml index 6e2183d3..5cafff90 100644 --- a/packages/apps/kubernetes/templates/helmreleases/ingress-nginx.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/ingress-nginx.yaml @@ -20,7 +20,7 @@ ingress-nginx: node-role.kubernetes.io/ingress-nginx: "" {{- end }} -{{- if and .Values.addons.ingressNginx.enabled .Values._namespace.etcd }} +{{- if .Values.addons.ingressNginx.enabled }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/packages/apps/kubernetes/templates/helmreleases/metrics-server.yaml b/packages/apps/kubernetes/templates/helmreleases/metrics-server.yaml index 3e6f9660..3cc81a14 100644 --- a/packages/apps/kubernetes/templates/helmreleases/metrics-server.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/metrics-server.yaml @@ -1,4 +1,3 @@ -{{- if .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: @@ -37,4 +36,3 @@ spec: namespace: {{ .Release.Namespace }} - name: {{ .Release.Name }}-prometheus-operator-crds namespace: {{ .Release.Namespace }} -{{- end }} diff --git a/packages/apps/kubernetes/templates/helmreleases/monitoring-agents.yaml b/packages/apps/kubernetes/templates/helmreleases/monitoring-agents.yaml index a811f7dd..ea84dec0 100644 --- a/packages/apps/kubernetes/templates/helmreleases/monitoring-agents.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/monitoring-agents.yaml @@ -1,6 +1,6 @@ {{- $targetTenant := .Values._namespace.monitoring }} {{- $clusterDomain := (index .Values._cluster "cluster-domain") | default "cozy.local" }} -{{- if and .Values.addons.monitoringAgents.enabled .Values._namespace.etcd }} +{{- if .Values.addons.monitoringAgents.enabled }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/packages/apps/kubernetes/templates/helmreleases/prometheus-operator-crds.yaml b/packages/apps/kubernetes/templates/helmreleases/prometheus-operator-crds.yaml index 3038a058..600a7994 100644 --- a/packages/apps/kubernetes/templates/helmreleases/prometheus-operator-crds.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/prometheus-operator-crds.yaml @@ -1,4 +1,3 @@ -{{- if .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: @@ -32,4 +31,3 @@ spec: - name: {{ .Release.Name }} namespace: {{ .Release.Namespace }} {{- end }} -{{- end }} diff --git a/packages/apps/kubernetes/templates/helmreleases/velero.yaml b/packages/apps/kubernetes/templates/helmreleases/velero.yaml index 781b9c49..ad236d53 100644 --- a/packages/apps/kubernetes/templates/helmreleases/velero.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/velero.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.addons.velero.enabled .Values._namespace.etcd }} +{{- if .Values.addons.velero.enabled }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler-crds.yaml b/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler-crds.yaml index 55a5faac..a3b7a9b4 100644 --- a/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler-crds.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler-crds.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.addons.monitoringAgents.enabled .Values._namespace.etcd }} +{{- if .Values.addons.monitoringAgents.enabled }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler.yaml b/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler.yaml index 74fb5a39..178df3e3 100644 --- a/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/vertical-pod-autoscaler.yaml @@ -24,7 +24,7 @@ vertical-pod-autoscaler: memory: 1600Mi {{- end }} -{{- if and .Values.addons.monitoringAgents.enabled .Values._namespace.etcd }} +{{- if .Values.addons.monitoringAgents.enabled }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/packages/apps/kubernetes/templates/helmreleases/victoria-metrics-operator.yaml b/packages/apps/kubernetes/templates/helmreleases/victoria-metrics-operator.yaml index 7302f8f5..99744277 100644 --- a/packages/apps/kubernetes/templates/helmreleases/victoria-metrics-operator.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/victoria-metrics-operator.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.addons.monitoringAgents.enabled .Values._namespace.etcd }} +{{- if .Values.addons.monitoringAgents.enabled }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/packages/apps/kubernetes/templates/helmreleases/volumesnapshot-crd.yaml b/packages/apps/kubernetes/templates/helmreleases/volumesnapshot-crd.yaml index d50fd93c..025f01b7 100644 --- a/packages/apps/kubernetes/templates/helmreleases/volumesnapshot-crd.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/volumesnapshot-crd.yaml @@ -1,4 +1,3 @@ -{{- if .Values._namespace.etcd }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: @@ -34,4 +33,3 @@ spec: - name: {{ .Release.Name }} namespace: {{ .Release.Namespace }} {{- end }} -{{- end }} diff --git a/packages/apps/kubernetes/templates/kccm/manager.yaml b/packages/apps/kubernetes/templates/kccm/manager.yaml index bd9e2798..81426d4e 100644 --- a/packages/apps/kubernetes/templates/kccm/manager.yaml +++ b/packages/apps/kubernetes/templates/kccm/manager.yaml @@ -1,4 +1,3 @@ -{{- if .Values._namespace.etcd }} apiVersion: apps/v1 kind: Deployment metadata: @@ -23,8 +22,6 @@ spec: - key: node-role.kubernetes.io/control-plane operator: Exists effect: "NoSchedule" - initContainers: - {{- include "kubernetes.waitForAdminKubeconfig" $ | nindent 6 }} containers: - name: kubevirt-cloud-controller-manager args: @@ -58,7 +55,5 @@ spec: name: cloud-config - secret: secretName: {{ .Release.Name }}-admin-kubeconfig - optional: true name: kubeconfig serviceAccountName: {{ .Release.Name }}-kccm -{{- end }} diff --git a/packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml b/packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml deleted file mode 100644 index 507e00af..00000000 --- a/packages/apps/kubernetes/tests/admin_kubeconfig_wait_test.yaml +++ /dev/null @@ -1,155 +0,0 @@ -suite: admin-kubeconfig wait guards - -release: - name: test - namespace: tenant-root - -values: - - values-ci.yaml - -tests: - - it: cluster-autoscaler mounts admin-kubeconfig as optional - template: templates/cluster-autoscaler/deployment.yaml - documentSelector: - path: kind - value: Deployment - asserts: - - equal: - path: spec.template.spec.volumes[?(@.name=="kubeconfig")].secret.secretName - value: test-admin-kubeconfig - - equal: - path: spec.template.spec.volumes[?(@.name=="kubeconfig")].secret.optional - value: true - - - it: cluster-autoscaler waits for admin-kubeconfig via initContainer - template: templates/cluster-autoscaler/deployment.yaml - documentSelector: - path: kind - value: Deployment - asserts: - - equal: - path: spec.template.spec.initContainers[0].name - value: wait-for-kubeconfig - - contains: - path: spec.template.spec.initContainers[0].volumeMounts - content: - name: kubeconfig - mountPath: /etc/kubernetes/kubeconfig - readOnly: true - - - it: kccm mounts admin-kubeconfig as optional - template: templates/kccm/manager.yaml - documentSelector: - path: kind - value: Deployment - asserts: - - equal: - path: spec.template.spec.volumes[?(@.name=="kubeconfig")].secret.secretName - value: test-admin-kubeconfig - - equal: - path: spec.template.spec.volumes[?(@.name=="kubeconfig")].secret.optional - value: true - - - it: kccm waits for admin-kubeconfig via initContainer - template: templates/kccm/manager.yaml - documentSelector: - path: kind - value: Deployment - asserts: - - equal: - path: spec.template.spec.initContainers[0].name - value: wait-for-kubeconfig - - contains: - path: spec.template.spec.initContainers[0].volumeMounts - content: - name: kubeconfig - mountPath: /etc/kubernetes/kubeconfig - readOnly: true - - - it: csi controller mounts admin-kubeconfig as optional - template: templates/csi/deploy.yaml - documentSelector: - path: kind - value: Deployment - asserts: - - equal: - path: spec.template.spec.volumes[?(@.name=="kubeconfig")].secret.secretName - value: test-admin-kubeconfig - - equal: - path: spec.template.spec.volumes[?(@.name=="kubeconfig")].secret.optional - value: true - - - it: csi controller waits for admin-kubeconfig via initContainer - template: templates/csi/deploy.yaml - documentSelector: - path: kind - value: Deployment - asserts: - - equal: - path: spec.template.spec.initContainers[0].name - value: wait-for-kubeconfig - - contains: - path: spec.template.spec.initContainers[0].volumeMounts - content: - name: kubeconfig - mountPath: /etc/kubernetes/kubeconfig - readOnly: true - - - it: wait-for-kubeconfig defaults to bundled busybox digest pin - template: templates/cluster-autoscaler/deployment.yaml - documentSelector: - path: kind - value: Deployment - asserts: - - matchRegex: - path: spec.template.spec.initContainers[0].image - pattern: '^docker\.io/library/busybox:[^@]+@sha256:[0-9a-f]{64}$' - - - it: wait-for-kubeconfig honours images.waitForKubeconfig override - template: templates/cluster-autoscaler/deployment.yaml - documentSelector: - path: kind - value: Deployment - set: - images: - waitForKubeconfig: "registry.example.test/library/busybox:1.37.0@sha256:0000000000000000000000000000000000000000000000000000000000000000" - asserts: - - equal: - path: spec.template.spec.initContainers[0].image - value: "registry.example.test/library/busybox:1.37.0@sha256:0000000000000000000000000000000000000000000000000000000000000000" - - - it: cluster.yaml renders and wires dataStoreName when tenant has etcd - template: templates/cluster.yaml - documentSelector: - path: kind - value: KamajiControlPlane - asserts: - - equal: - path: spec.dataStoreName - value: tenant-root - - - it: cluster.yaml skips Cluster resources when tenant has no etcd DataStore - # Must NOT fail rendering - the parent Tenant chart populates - # _namespace.etcd asynchronously, so rendering failures here would cause - # flux install remediation on every cold bootstrap. Instead, emit only a - # ConfigMap status beacon so the HelmRelease reports Ready while flux - # retries on its interval until the DataStore appears. - template: templates/cluster.yaml - set: - _namespace: - etcd: "" - monitoring: "" - ingress: "" - seaweedfs: "" - host: "" - asserts: - - hasDocuments: - count: 1 - - isKind: - of: ConfigMap - - equal: - path: metadata.name - value: test-awaiting-etcd - - equal: - path: data.status - value: awaiting-etcd diff --git a/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml b/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml deleted file mode 100644 index 44528470..00000000 --- a/packages/apps/kubernetes/tests/gpu_operator_hami_test.yaml +++ /dev/null @@ -1,99 +0,0 @@ -suite: GPU Operator HelmRelease HAMi integration tests -templates: - - templates/helmreleases/gpu-operator.yaml -values: - - values-ci.yaml -tests: - - it: should disable devicePlugin when hami is enabled - set: - addons: - gpuOperator: - enabled: true - valuesOverride: {} - hami: - enabled: true - valuesOverride: {} - asserts: - - equal: - path: spec.values.gpu-operator.devicePlugin.enabled - value: false - - - it: should not have values when hami is disabled and no overrides - set: - addons: - gpuOperator: - enabled: true - valuesOverride: {} - hami: - enabled: false - valuesOverride: {} - asserts: - - notExists: - path: spec.values - - - it: should apply hami defaults when valuesOverride key is omitted - set: - addons: - gpuOperator: - enabled: true - hami: - enabled: true - asserts: - - equal: - path: spec.values.gpu-operator.devicePlugin.enabled - value: false - - - it: should allow user overrides to merge with hami defaults - set: - addons: - gpuOperator: - enabled: true - valuesOverride: - gpu-operator: - driver: - enabled: false - hami: - enabled: true - valuesOverride: {} - asserts: - - equal: - path: spec.values.gpu-operator.devicePlugin.enabled - value: false - - equal: - path: spec.values.gpu-operator.driver.enabled - value: false - - - it: should let user explicitly override devicePlugin.enabled to true with hami enabled - set: - addons: - gpuOperator: - enabled: true - valuesOverride: - gpu-operator: - devicePlugin: - enabled: true - driver: - enabled: false - hami: - enabled: true - valuesOverride: {} - asserts: - - equal: - path: spec.values.gpu-operator.devicePlugin.enabled - value: true - - equal: - path: spec.values.gpu-operator.driver.enabled - value: false - - - it: should not render when gpuOperator is disabled - set: - addons: - gpuOperator: - enabled: false - valuesOverride: {} - hami: - enabled: false - valuesOverride: {} - asserts: - - hasDocuments: - count: 0 diff --git a/packages/apps/kubernetes/tests/hami_test.yaml b/packages/apps/kubernetes/tests/hami_test.yaml deleted file mode 100644 index 27f7c75b..00000000 --- a/packages/apps/kubernetes/tests/hami_test.yaml +++ /dev/null @@ -1,153 +0,0 @@ -suite: HAMi HelmRelease tests -templates: - - templates/helmreleases/hami.yaml -values: - - values-ci.yaml -tests: - - it: should not render when hami is disabled - set: - addons: - hami: - enabled: false - valuesOverride: {} - gpuOperator: - enabled: true - valuesOverride: {} - asserts: - - hasDocuments: - count: 0 - - - it: should render HelmRelease when hami is enabled - set: - addons: - hami: - enabled: true - valuesOverride: {} - gpuOperator: - enabled: true - valuesOverride: {} - asserts: - - hasDocuments: - count: 1 - - isKind: - of: HelmRelease - - - it: should fail when gpuOperator is not enabled - set: - addons: - hami: - enabled: true - valuesOverride: {} - gpuOperator: - enabled: false - valuesOverride: {} - asserts: - - failedTemplate: - errorMessage: "addons.hami requires addons.gpuOperator to be enabled" - - - it: should have correct metadata labels - set: - addons: - hami: - enabled: true - valuesOverride: {} - gpuOperator: - enabled: true - valuesOverride: {} - asserts: - - equal: - path: metadata.labels["cozystack.io/repository"] - value: system - - equal: - path: metadata.labels["sharding.fluxcd.io/key"] - value: tenants - - - it: should use ExternalArtifact chartRef - set: - addons: - hami: - enabled: true - valuesOverride: {} - gpuOperator: - enabled: true - valuesOverride: {} - asserts: - - equal: - path: spec.chartRef.kind - value: ExternalArtifact - - equal: - path: spec.chartRef.namespace - value: cozy-system - - - it: should target cozy-hami namespace - set: - addons: - hami: - enabled: true - valuesOverride: {} - gpuOperator: - enabled: true - valuesOverride: {} - asserts: - - equal: - path: spec.targetNamespace - value: cozy-hami - - equal: - path: spec.storageNamespace - value: cozy-hami - - - it: should depend on gpu-operator and cilium - release: - name: test - namespace: test-ns - set: - addons: - hami: - enabled: true - valuesOverride: {} - gpuOperator: - enabled: true - valuesOverride: {} - asserts: - - contains: - path: spec.dependsOn - content: - name: test-cilium - namespace: test-ns - - contains: - path: spec.dependsOn - content: - name: test-gpu-operator - namespace: test-ns - - - it: should not render spec.values when valuesOverride is empty - set: - addons: - hami: - enabled: true - valuesOverride: {} - gpuOperator: - enabled: true - valuesOverride: {} - asserts: - - hasDocuments: - count: 1 - - notExists: - path: spec.values - - - it: should pass through valuesOverride - set: - addons: - hami: - enabled: true - valuesOverride: - hami: - devicePlugin: - deviceSplitCount: 5 - gpuOperator: - enabled: true - valuesOverride: {} - asserts: - - equal: - path: spec.values.hami.devicePlugin.deviceSplitCount - value: 5 diff --git a/packages/apps/kubernetes/tests/values-ci-no-etcd.yaml b/packages/apps/kubernetes/tests/values-ci-no-etcd.yaml deleted file mode 100644 index c7c8196f..00000000 --- a/packages/apps/kubernetes/tests/values-ci-no-etcd.yaml +++ /dev/null @@ -1,9 +0,0 @@ -_namespace: - etcd: "" - monitoring: "" - ingress: "" - seaweedfs: "" - host: "" -_cluster: - cluster-domain: cozy.local -nodeGroups: null diff --git a/packages/apps/kubernetes/tests/values-ci.yaml b/packages/apps/kubernetes/tests/values-ci.yaml deleted file mode 100644 index 13365e8c..00000000 --- a/packages/apps/kubernetes/tests/values-ci.yaml +++ /dev/null @@ -1,9 +0,0 @@ -_namespace: - etcd: tenant-root - monitoring: "" - ingress: "" - seaweedfs: "" - host: "" -_cluster: - cluster-domain: cozy.local -nodeGroups: null diff --git a/packages/apps/kubernetes/values.schema.json b/packages/apps/kubernetes/values.schema.json index a43deaec..1beeff9c 100644 --- a/packages/apps/kubernetes/values.schema.json +++ b/packages/apps/kubernetes/values.schema.json @@ -149,7 +149,6 @@ "fluxcd", "gatewayAPI", "gpuOperator", - "hami", "ingressNginx", "monitoringAgents", "velero", @@ -269,28 +268,6 @@ } } }, - "hami": { - "description": "HAMi GPU virtualization middleware.", - "type": "object", - "default": {}, - "required": [ - "enabled", - "valuesOverride" - ], - "properties": { - "enabled": { - "description": "Enable HAMi (requires GPU Operator).", - "type": "boolean", - "default": false - }, - "valuesOverride": { - "description": "Custom Helm values overrides.", - "type": "object", - "default": {}, - "x-kubernetes-preserve-unknown-fields": true - } - } - }, "ingressNginx": { "description": "Ingress-NGINX controller.", "type": "object", @@ -653,18 +630,6 @@ } } } - }, - "images": { - "description": "Optional image overrides for air-gapped or rate-limited registries.", - "type": "object", - "default": {}, - "properties": { - "waitForKubeconfig": { - "description": "Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag.", - "type": "string", - "default": "" - } - } } } } diff --git a/packages/apps/kubernetes/values.yaml b/packages/apps/kubernetes/values.yaml index d609476c..a67b5d69 100644 --- a/packages/apps/kubernetes/values.yaml +++ b/packages/apps/kubernetes/values.yaml @@ -94,10 +94,6 @@ host: "" ## @field {bool} enabled - Enable FluxCD. ## @field {object} valuesOverride - Custom Helm values overrides. -## @typedef {struct} HAMiAddon - HAMi GPU virtualization middleware. -## @field {bool} enabled - Enable HAMi (requires GPU Operator). -## @field {object} valuesOverride - Custom Helm values overrides. - ## @typedef {struct} MonitoringAgentsAddon - Monitoring agents (Fluent Bit, VMAgents). ## @field {bool} enabled - Enable monitoring agents. ## @field {object} valuesOverride - Custom Helm values overrides. @@ -118,7 +114,6 @@ host: "" ## @field {GatewayAPIAddon} gatewayAPI - Gateway API addon. ## @field {IngressNginxAddon} ingressNginx - Ingress-NGINX controller. ## @field {GPUOperatorAddon} gpuOperator - NVIDIA GPU Operator. -## @field {HAMiAddon} hami - HAMi GPU virtualization middleware. ## @field {FluxCDAddon} fluxcd - FluxCD GitOps operator. ## @field {MonitoringAgentsAddon} monitoringAgents - Monitoring agents. ## @field {VerticalPodAutoscalerAddon} verticalPodAutoscaler - Vertical Pod Autoscaler. @@ -142,9 +137,6 @@ addons: gpuOperator: enabled: false valuesOverride: {} - hami: - enabled: false - valuesOverride: {} fluxcd: enabled: false valuesOverride: {} @@ -205,10 +197,3 @@ controlPlane: server: resources: {} resourcesPreset: "micro" - -## @typedef {struct} Images - Optional image overrides for chart-internal helpers. -## @field {string} [waitForKubeconfig] - Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag. - -## @param {Images} images - Optional image overrides for air-gapped or rate-limited registries. -images: - waitForKubeconfig: "" diff --git a/packages/apps/mariadb/images/mariadb-backup.tag b/packages/apps/mariadb/images/mariadb-backup.tag index 6c830892..1e381661 100644 --- a/packages/apps/mariadb/images/mariadb-backup.tag +++ b/packages/apps/mariadb/images/mariadb-backup.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/mariadb-backup:0.0.0@sha256:3841eb171416711977dea0cf8cd45d32344caac9727af760c37d5e1dd41ee4bb +ghcr.io/cozystack/cozystack/mariadb-backup:0.0.0@sha256:0ddbbec0568dcb9fbc317cd9cc654e826dbe88ba3f184fa9b6b58aacb93b4570 diff --git a/packages/apps/mariadb/templates/workloadmonitor.yaml b/packages/apps/mariadb/templates/workloadmonitor.yaml index 36cb59f0..b69139bf 100644 --- a/packages/apps/mariadb/templates/workloadmonitor.yaml +++ b/packages/apps/mariadb/templates/workloadmonitor.yaml @@ -3,8 +3,6 @@ apiVersion: cozystack.io/v1alpha1 kind: WorkloadMonitor metadata: name: {{ $.Release.Name }} - labels: - workloads.cozystack.io/resource-preset: {{ .Values.resourcesPreset | quote }} spec: replicas: {{ .Values.replicas }} minReplicas: 1 diff --git a/packages/apps/mongodb/templates/mongodb.yaml b/packages/apps/mongodb/templates/mongodb.yaml index 9273e506..67969758 100644 --- a/packages/apps/mongodb/templates/mongodb.yaml +++ b/packages/apps/mongodb/templates/mongodb.yaml @@ -169,8 +169,6 @@ apiVersion: cozystack.io/v1alpha1 kind: WorkloadMonitor metadata: name: {{ .Release.Name }} - labels: - workloads.cozystack.io/resource-preset: {{ .Values.resourcesPreset | quote }} spec: {{- if .Values.sharding }} {{- $totalReplicas := 0 }} diff --git a/packages/apps/nats/templates/workloadmonitor.yaml b/packages/apps/nats/templates/workloadmonitor.yaml index cb20b4a6..43d64a46 100644 --- a/packages/apps/nats/templates/workloadmonitor.yaml +++ b/packages/apps/nats/templates/workloadmonitor.yaml @@ -3,8 +3,6 @@ apiVersion: cozystack.io/v1alpha1 kind: WorkloadMonitor metadata: name: {{ $.Release.Name }} - labels: - workloads.cozystack.io/resource-preset: {{ .Values.resourcesPreset | quote }} spec: replicas: {{ .Values.replicas }} minReplicas: 1 diff --git a/packages/apps/openbao/templates/workloadmonitor.yaml b/packages/apps/openbao/templates/workloadmonitor.yaml index 56f9ec8e..0a9acf76 100644 --- a/packages/apps/openbao/templates/workloadmonitor.yaml +++ b/packages/apps/openbao/templates/workloadmonitor.yaml @@ -3,8 +3,6 @@ apiVersion: cozystack.io/v1alpha1 kind: WorkloadMonitor metadata: name: {{ $.Release.Name }} - labels: - workloads.cozystack.io/resource-preset: {{ .Values.resourcesPreset | quote }} spec: replicas: {{ .Values.replicas }} minReplicas: 1 diff --git a/packages/apps/opensearch/templates/opensearch.yaml b/packages/apps/opensearch/templates/opensearch.yaml index 836a3d4b..fcf431ac 100644 --- a/packages/apps/opensearch/templates/opensearch.yaml +++ b/packages/apps/opensearch/templates/opensearch.yaml @@ -93,8 +93,6 @@ apiVersion: cozystack.io/v1alpha1 kind: WorkloadMonitor metadata: name: {{ .Release.Name }} - labels: - workloads.cozystack.io/resource-preset: {{ .Values.resourcesPreset | quote }} spec: replicas: {{ .Values.replicas }} minReplicas: 1 diff --git a/packages/apps/postgres/README.md b/packages/apps/postgres/README.md index 4cda7284..5a550f7a 100644 --- a/packages/apps/postgres/README.md +++ b/packages/apps/postgres/README.md @@ -133,13 +133,12 @@ See: ### Bootstrap (recovery) parameters -| Name | Description | Type | Value | -| ------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------- | -| `bootstrap` | Bootstrap configuration. | `object` | `{}` | -| `bootstrap.enabled` | Whether to restore from a backup. | `bool` | `false` | -| `bootstrap.recoveryTime` | Timestamp (RFC3339) for point-in-time recovery; empty means latest. | `string` | `""` | -| `bootstrap.oldName` | Previous cluster name before deletion. | `string` | `""` | -| `bootstrap.serverName` | Barman server name (S3 path prefix) used by the original cluster when writing backups. Set this only when the original cluster had an explicit barmanObjectStore.serverName that differed from its Kubernetes resource name. | `string` | `""` | +| Name | Description | Type | Value | +| ------------------------ | ------------------------------------------------------------------- | -------- | ------- | +| `bootstrap` | Bootstrap configuration. | `object` | `{}` | +| `bootstrap.enabled` | Whether to restore from a backup. | `bool` | `false` | +| `bootstrap.recoveryTime` | Timestamp (RFC3339) for point-in-time recovery; empty means latest. | `string` | `""` | +| `bootstrap.oldName` | Previous cluster name before deletion. | `string` | `""` | ## Parameter examples and reference diff --git a/packages/apps/postgres/templates/db.yaml b/packages/apps/postgres/templates/db.yaml index 2cdaec1d..7557c436 100644 --- a/packages/apps/postgres/templates/db.yaml +++ b/packages/apps/postgres/templates/db.yaml @@ -32,9 +32,6 @@ spec: - name: {{ .Values.bootstrap.oldName }} barmanObjectStore: destinationPath: {{ .Values.backup.destinationPath }} - {{- if .Values.bootstrap.serverName }} - serverName: {{ .Values.bootstrap.serverName }} - {{- end }} endpointURL: {{ .Values.backup.endpointURL }} s3Credentials: accessKeyId: @@ -87,8 +84,6 @@ apiVersion: cozystack.io/v1alpha1 kind: WorkloadMonitor metadata: name: {{ $.Release.Name }} - labels: - workloads.cozystack.io/resource-preset: {{ .Values.resourcesPreset | quote }} spec: replicas: {{ .Values.replicas }} minReplicas: 1 diff --git a/packages/apps/postgres/values.schema.json b/packages/apps/postgres/values.schema.json index 98e29822..b2a4aeba 100644 --- a/packages/apps/postgres/values.schema.json +++ b/packages/apps/postgres/values.schema.json @@ -254,11 +254,6 @@ "description": "Timestamp (RFC3339) for point-in-time recovery; empty means latest.", "type": "string", "default": "" - }, - "serverName": { - "description": "Barman server name (S3 path prefix) used by the original cluster when writing backups. Set this only when the original cluster had an explicit barmanObjectStore.serverName that differed from its Kubernetes resource name.", - "type": "string", - "default": "" } } } diff --git a/packages/apps/postgres/values.yaml b/packages/apps/postgres/values.yaml index 2ceaa9cf..b8f07f63 100644 --- a/packages/apps/postgres/values.yaml +++ b/packages/apps/postgres/values.yaml @@ -154,7 +154,6 @@ backup: ## @field {bool} enabled - Whether to restore from a backup. ## @field {string} [recoveryTime] - Timestamp (RFC3339) for point-in-time recovery; empty means latest. ## @field {string} oldName - Previous cluster name before deletion. -## @field {string} [serverName] - Barman server name (S3 path prefix) used by the original cluster when writing backups. Set this only when the original cluster had an explicit barmanObjectStore.serverName that differed from its Kubernetes resource name. ## @param {Bootstrap} bootstrap - Bootstrap configuration. bootstrap: @@ -162,4 +161,3 @@ bootstrap: # example: 2020-11-26 15:22:00.00000+00 recoveryTime: "" oldName: "" - serverName: "" diff --git a/packages/apps/qdrant/templates/dashboard-resourcemap.yaml b/packages/apps/qdrant/templates/dashboard-resourcemap.yaml index 0986951f..b0a85bcc 100644 --- a/packages/apps/qdrant/templates/dashboard-resourcemap.yaml +++ b/packages/apps/qdrant/templates/dashboard-resourcemap.yaml @@ -41,8 +41,6 @@ apiVersion: cozystack.io/v1alpha1 kind: WorkloadMonitor metadata: name: {{ .Release.Name }} - labels: - workloads.cozystack.io/resource-preset: {{ .Values.resourcesPreset | quote }} spec: minReplicas: 1 replicas: {{ .Values.replicas }} diff --git a/packages/apps/rabbitmq/templates/workloadmonitor.yaml b/packages/apps/rabbitmq/templates/workloadmonitor.yaml index 66941153..0f7462c7 100644 --- a/packages/apps/rabbitmq/templates/workloadmonitor.yaml +++ b/packages/apps/rabbitmq/templates/workloadmonitor.yaml @@ -3,8 +3,6 @@ apiVersion: cozystack.io/v1alpha1 kind: WorkloadMonitor metadata: name: {{ $.Release.Name }} - labels: - workloads.cozystack.io/resource-preset: {{ .Values.resourcesPreset | quote }} spec: replicas: {{ .Values.replicas }} minReplicas: 1 diff --git a/packages/apps/redis/templates/redisfailover.yaml b/packages/apps/redis/templates/redisfailover.yaml index 160e030d..936217a3 100644 --- a/packages/apps/redis/templates/redisfailover.yaml +++ b/packages/apps/redis/templates/redisfailover.yaml @@ -75,8 +75,6 @@ kind: WorkloadMonitor metadata: name: {{ $.Release.Name }}-redis namespace: {{ $.Release.Namespace }} - labels: - workloads.cozystack.io/resource-preset: {{ .Values.resourcesPreset | quote }} spec: minReplicas: 1 replicas: {{ .Values.replicas }} @@ -92,8 +90,6 @@ kind: WorkloadMonitor metadata: name: {{ $.Release.Name }}-sentinel namespace: {{ $.Release.Namespace }} - labels: - workloads.cozystack.io/resource-preset: {{ .Values.resourcesPreset | quote }} spec: minReplicas: 2 replicas: 3 diff --git a/packages/apps/tcp-balancer/templates/workloadmonitor.yaml b/packages/apps/tcp-balancer/templates/workloadmonitor.yaml index 3478826b..41dce2ab 100644 --- a/packages/apps/tcp-balancer/templates/workloadmonitor.yaml +++ b/packages/apps/tcp-balancer/templates/workloadmonitor.yaml @@ -3,8 +3,6 @@ apiVersion: cozystack.io/v1alpha1 kind: WorkloadMonitor metadata: name: {{ $.Release.Name }} - labels: - workloads.cozystack.io/resource-preset: {{ .Values.resourcesPreset | quote }} spec: replicas: {{ .Values.replicas }} minReplicas: 1 diff --git a/packages/apps/tenant/Makefile b/packages/apps/tenant/Makefile index 2f51d836..3fe3810d 100644 --- a/packages/apps/tenant/Makefile +++ b/packages/apps/tenant/Makefile @@ -3,6 +3,3 @@ include ../../../hack/package.mk generate: cozyvalues-gen -m 'tenant' -v values.yaml -s values.schema.json -r README.md -g ../../../api/apps/v1alpha1/tenant/types.go ../../../hack/update-crd.sh - -test: - helm unittest . diff --git a/packages/apps/tenant/README.md b/packages/apps/tenant/README.md index 96cf5c6e..987e2c99 100644 --- a/packages/apps/tenant/README.md +++ b/packages/apps/tenant/README.md @@ -6,15 +6,15 @@ Tenants can be created recursively and are subject to the following rules: ### Tenant naming -Tenant names must be alphanumeric: - -- Lowercase letters (`a-z`) and digits (`0-9`) only -- Must start with a lowercase letter -- Dashes (`-`) are **not allowed**, unlike with other services +Tenant names must follow DNS-1035 naming rules: +- Must start with a lowercase letter (`a-z`) +- Can only contain lowercase letters, numbers, and hyphens (`a-z`, `0-9`, `-`) +- Must end with a letter or number (not a hyphen) - Maximum length depends on the cluster configuration (Helm release prefix and root domain) -This restriction exists to keep consistent naming in tenants, nested tenants, and services deployed in them. -A tenant cannot be named `foo-bar` because parsing internal resource names like `tenant-foo-bar` would be ambiguous. +**Note:** Using dashes (`-`) in tenant names is **allowed but discouraged**, unlike with other services. +This is to keep consistent naming in tenants, nested tenants, and services deployed in them. +Names with dashes (e.g., `foo-bar`) may lead to ambiguous parsing of internal resource names like `tenant-foo-bar`. For example: diff --git a/packages/apps/tenant/templates/cilium-lb-pool.yaml b/packages/apps/tenant/templates/cilium-lb-pool.yaml deleted file mode 100644 index 63dbcaca..00000000 --- a/packages/apps/tenant/templates/cilium-lb-pool.yaml +++ /dev/null @@ -1,28 +0,0 @@ -{{- $exposeMode := (index .Values._cluster "expose-mode") | default "externalIPs" }} -{{- $exposeIngress := (index .Values._cluster "expose-ingress") | default "tenant-root" }} -{{- $exposeIPs := (index .Values._cluster "expose-external-ips") | default "" | nospace }} -{{- $ipsList := list }} -{{- range splitList "," $exposeIPs }} - {{- $ip := . | trim }} - {{- if $ip }} - {{- $ipsList = append $ipsList $ip }} - {{- end }} -{{- end }} -{{- $isPublishingIngressLB := and - (eq $exposeMode "loadBalancer") - (eq $exposeIngress .Release.Namespace) - .Values.ingress }} -{{- if and $isPublishingIngressLB $ipsList }} -apiVersion: cilium.io/v2 -kind: CiliumLoadBalancerIPPool -metadata: - name: {{ trimPrefix "tenant-" .Release.Namespace }}-exposure -spec: - blocks: - {{- range $ipsList }} - - cidr: {{ . }}{{ if not (contains "/" .) }}/{{ if contains ":" . }}128{{ else }}32{{ end }}{{ end }} - {{- end }} - serviceSelector: - matchLabels: - "io.kubernetes.service.namespace": {{ .Release.Namespace | quote }} -{{- end }} diff --git a/packages/apps/tenant/templates/networkpolicy.yaml b/packages/apps/tenant/templates/networkpolicy.yaml index 4bd3ba02..ce3e33ab 100644 --- a/packages/apps/tenant/templates/networkpolicy.yaml +++ b/packages/apps/tenant/templates/networkpolicy.yaml @@ -186,23 +186,6 @@ spec: --- apiVersion: cilium.io/v2 kind: CiliumNetworkPolicy -metadata: - # Allow the tenant's ingress-nginx to reach both the linstor-gui - # gatekeeper pods (for /oauth2/* and app traffic) AND the transient - # ACME http-01 solver pods that cert-manager spins up in the same - # namespace during certificate issuance/renewal. Matching the whole - # namespace mirrors allow-to-dashboard / allow-to-keycloak. - name: allow-to-linstor-gui - namespace: {{ include "tenant.name" . }} -spec: - endpointSelector: {} - egress: - - toEndpoints: - - matchLabels: - "k8s:io.kubernetes.pod.namespace": cozy-linstor ---- -apiVersion: cilium.io/v2 -kind: CiliumNetworkPolicy metadata: name: allow-to-keycloak namespace: {{ include "tenant.name" . }} diff --git a/packages/apps/tenant/tests/exposure_test.yaml b/packages/apps/tenant/tests/exposure_test.yaml deleted file mode 100644 index e86fab5b..00000000 --- a/packages/apps/tenant/tests/exposure_test.yaml +++ /dev/null @@ -1,141 +0,0 @@ -suite: tenant CiliumLoadBalancerIPPool rendering for publishing.exposure=loadBalancer -templates: - - templates/cilium-lb-pool.yaml - -release: - name: tenant-root - namespace: tenant-root - -tests: - - it: default exposure (externalIPs) renders no pool - set: - ingress: true - _cluster: - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10,192.0.2.11" - asserts: - - hasDocuments: - count: 0 - - - it: loadBalancer mode in publishing tenant with ingress=true renders v2 pool with namespace-only selector - set: - ingress: true - _cluster: - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10,192.0.2.11" - expose-mode: loadBalancer - asserts: - - hasDocuments: - count: 1 - - equal: - path: apiVersion - value: cilium.io/v2 - - equal: - path: kind - value: CiliumLoadBalancerIPPool - - equal: - path: metadata.name - value: root-exposure - - equal: - path: spec.blocks - value: - - cidr: 192.0.2.10/32 - - cidr: 192.0.2.11/32 - - equal: - path: spec.serviceSelector.matchLabels["io.kubernetes.service.namespace"] - value: tenant-root - - notExists: - path: spec.serviceSelector.matchLabels["app.kubernetes.io/name"] - - - it: loadBalancer mode with IPv6 emits /128 CIDR - set: - ingress: true - _cluster: - expose-ingress: tenant-root - expose-external-ips: "2001:db8::1" - expose-mode: loadBalancer - asserts: - - equal: - path: spec.blocks - value: - - cidr: 2001:db8::1/128 - - - it: loadBalancer mode with mixed IPv4 and IPv6 emits correct CIDR per family - set: - ingress: true - _cluster: - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10,2001:db8::1" - expose-mode: loadBalancer - asserts: - - equal: - path: spec.blocks - value: - - cidr: 192.0.2.10/32 - - cidr: 2001:db8::1/128 - - - it: loadBalancer mode accepts pre-CIDR input without double-suffixing - set: - ingress: true - _cluster: - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10/32,2001:db8::1/128" - expose-mode: loadBalancer - asserts: - - equal: - path: spec.blocks - value: - - cidr: 192.0.2.10/32 - - cidr: 2001:db8::1/128 - - - it: loadBalancer mode filters out empty entries from externalIPs (trailing, leading, repeated commas) - set: - ingress: true - _cluster: - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10,,192.0.2.11," - expose-mode: loadBalancer - asserts: - - hasDocuments: - count: 1 - - equal: - path: spec.blocks - value: - - cidr: 192.0.2.10/32 - - cidr: 192.0.2.11/32 - - - it: loadBalancer mode with ingress=false in publishing tenant renders no pool - set: - ingress: false - _cluster: - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10" - expose-mode: loadBalancer - asserts: - - hasDocuments: - count: 0 - - - it: loadBalancer mode in a non-publishing tenant renders no pool - set: - ingress: true - _cluster: - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10" - expose-mode: loadBalancer - release: - name: tenant-u1 - namespace: tenant-u1 - asserts: - - hasDocuments: - count: 0 - - - it: loadBalancer mode in publishing tenant with empty externalIPs renders no pool - set: - ingress: true - _cluster: - expose-ingress: tenant-root - expose-external-ips: "" - expose-mode: loadBalancer - asserts: - - hasDocuments: - count: 0 diff --git a/packages/apps/vm-disk/README.md b/packages/apps/vm-disk/README.md index 3727d0f5..b8a4cdac 100644 --- a/packages/apps/vm-disk/README.md +++ b/packages/apps/vm-disk/README.md @@ -6,17 +6,15 @@ A Virtual Machine Disk ### Common parameters -| Name | Description | Type | Value | -| ------------------- | ------------------------------------------------------- | ---------- | ------------ | -| `source` | The source image location used to create a disk. | `object` | `{}` | -| `source.image` | Use image by name from default collection. | `*object` | `null` | -| `source.image.name` | Name of the image to use. | `string` | `""` | -| `source.upload` | Upload local image. | `*object` | `null` | -| `source.http` | Download image from an HTTP source. | `*object` | `null` | -| `source.http.url` | URL to download the image. | `string` | `""` | -| `source.disk` | Clone an existing vm-disk. | `*object` | `null` | -| `source.disk.name` | Name of the vm-disk to clone. | `string` | `""` | -| `optical` | Defines if disk should be considered optical. | `bool` | `false` | -| `storage` | The size of the disk allocated for the virtual machine. | `quantity` | `5Gi` | -| `storageClass` | StorageClass used to store the data. | `string` | `replicated` | +| Name | Description | Type | Value | +| ------------------- | ------------------------------------------------------------------------------------------------------------------------ | ---------- | ------------ | +| `source` | The source image location used to create a disk. | `object` | `{}` | +| `source.image` | Use image by name. | `*object` | `null` | +| `source.image.name` | Name of the image to use (uploaded as "golden image" or from the list: `ubuntu`, `fedora`, `cirros`, `alpine`, `talos`). | `string` | `""` | +| `source.upload` | Upload local image. | `*object` | `null` | +| `source.http` | Download image from an HTTP source. | `*object` | `null` | +| `source.http.url` | URL to download the image. | `string` | `""` | +| `optical` | Defines if disk should be considered optical. | `bool` | `false` | +| `storage` | The size of the disk allocated for the virtual machine. | `quantity` | `5Gi` | +| `storageClass` | StorageClass used to store the data. | `string` | `replicated` | diff --git a/packages/apps/vm-disk/templates/dv.yaml b/packages/apps/vm-disk/templates/dv.yaml index 521a82b8..3d68e639 100644 --- a/packages/apps/vm-disk/templates/dv.yaml +++ b/packages/apps/vm-disk/templates/dv.yaml @@ -21,18 +21,15 @@ spec: {{- end }} source: {{- if hasKey .Values.source "image" }} + {{- $dv := lookup "cdi.kubevirt.io/v1beta1" "DataVolume" "cozy-public" (printf "vm-image-%s" .Values.source.image.name) }} pvc: - name: vm-default-images-{{ required "A valid .Values.source.image.name entry required!" .Values.source.image.name }} + name: vm-image-{{ required "A valid .Values.source.image.name entry required!" .Values.source.image.name }} namespace: cozy-public {{- else if hasKey .Values.source "http" }} http: url: {{ required "A valid .Values.source.http.url entry required!" .Values.source.http.url }} {{- else if hasKey .Values.source "upload" }} upload: {} - {{- else if hasKey .Values.source "disk" }} - pvc: - name: vm-disk-{{ required "A valid .Values.source.disk.name entry required!" .Values.source.disk.name }} - namespace: {{ $.Release.Namespace }} {{- end }} {{- else }} source: diff --git a/packages/apps/vm-disk/values.schema.json b/packages/apps/vm-disk/values.schema.json index d61fc359..aec36a47 100644 --- a/packages/apps/vm-disk/values.schema.json +++ b/packages/apps/vm-disk/values.schema.json @@ -7,19 +7,6 @@ "type": "object", "default": {}, "properties": { - "disk": { - "description": "Clone an existing vm-disk.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "description": "Name of the vm-disk to clone.", - "type": "string" - } - } - }, "http": { "description": "Download image from an HTTP source.", "type": "object", @@ -34,14 +21,14 @@ } }, "image": { - "description": "Use image by name from default collection.", + "description": "Use image by name.", "type": "object", "required": [ "name" ], "properties": { "name": { - "description": "Name of the image to use.", + "description": "Name of the image to use (uploaded as \"golden image\" or from the list: `ubuntu`, `fedora`, `cirros`, `alpine`, `talos`).", "type": "string" } } diff --git a/packages/apps/vm-disk/values.yaml b/packages/apps/vm-disk/values.yaml index b9a20d68..33bd174f 100644 --- a/packages/apps/vm-disk/values.yaml +++ b/packages/apps/vm-disk/values.yaml @@ -3,21 +3,17 @@ ## ## @typedef {struct} SourceImage - Use image by name. -## @field {string} name - Name of the image to use. +## @field {string} name - Name of the image to use (uploaded as "golden image" or from the list: `ubuntu`, `fedora`, `cirros`, `alpine`, `talos`). ## @typedef {struct} SourceUpload - Upload local image. ## @typedef {struct} SourceHTTP - Download image from an HTTP source. ## @field {string} url - URL to download the image. -## @typedef {struct} SourceDisk - Clone an existing vm-disk. -## @field {string} name - Name of the vm-disk to clone. - ## @typedef {struct} Source - The source image location used to create a disk. -## @field {*SourceImage} [image] - Use image by name from default collection. +## @field {*SourceImage} [image] - Use image by name. ## @field {*SourceUpload} [upload] - Upload local image. ## @field {*SourceHTTP} [http] - Download image from an HTTP source. -## @field {*SourceDisk} [disk] - Clone an existing vm-disk. ## @param {Source} source - The source image location used to create a disk. source: {} diff --git a/packages/apps/vm-instance/README.md b/packages/apps/vm-instance/README.md index 9d6a52b2..a2b6603e 100644 --- a/packages/apps/vm-instance/README.md +++ b/packages/apps/vm-instance/README.md @@ -36,32 +36,31 @@ virtctl ssh @ ### Common parameters -| Name | Description | Type | Value | -| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | ----------- | -| `external` | Enable external access from outside the cluster. | `bool` | `false` | -| `externalMethod` | Method to pass through traffic to the VM. | `string` | `PortList` | -| `externalPorts` | Ports to forward from outside the cluster. | `[]int` | `[22]` | -| `externalAllowICMP` | Whether to accept ICMP traffic to the VM in PortList mode (preserves ping and PMTU discovery). No effect in WholeIP mode. Default true so ping behaves as users expect even when port filtering is in effect. | `bool` | `true` | -| `runStrategy` | Requested running state of the VirtualMachineInstance | `string` | `Always` | -| `instanceType` | Virtual Machine instance type. | `string` | `u1.medium` | -| `instanceProfile` | Virtual Machine preferences profile. | `string` | `ubuntu` | -| `disks` | List of disks to attach. | `[]object` | `[]` | -| `disks[i].name` | Disk name. | `string` | `""` | -| `disks[i].bus` | Disk bus type (e.g. "sata"). | `string` | `""` | -| `networks` | Networks to attach the VM to. | `[]object` | `[]` | -| `networks[i].name` | Network attachment name. | `string` | `""` | -| `subnets` | Deprecated: use networks instead. | `[]object` | `[]` | -| `subnets[i].name` | Network attachment name. | `string` | `""` | -| `gpus` | List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM). | `[]object` | `[]` | -| `gpus[i].name` | The name of the GPU resource to attach. | `string` | `""` | -| `cpuModel` | Model specifies the CPU model inside the VMI. List of available models https://github.com/libvirt/libvirt/tree/master/src/cpu_map | `string` | `""` | -| `resources` | Resource configuration for the virtual machine. | `object` | `{}` | -| `resources.cpu` | Number of CPU cores allocated. | `quantity` | `""` | -| `resources.memory` | Amount of memory allocated. | `quantity` | `""` | -| `resources.sockets` | Number of CPU sockets (vCPU topology). | `quantity` | `""` | -| `sshKeys` | List of SSH public keys for authentication. | `[]string` | `[]` | -| `cloudInit` | Cloud-init user data. | `string` | `""` | -| `cloudInitSeed` | Seed string to generate SMBIOS UUID for the VM. | `string` | `""` | +| Name | Description | Type | Value | +| ------------------- | --------------------------------------------------------------------------------------------------------------------------------- | ---------- | ----------- | +| `external` | Enable external access from outside the cluster. | `bool` | `false` | +| `externalMethod` | Method to pass through traffic to the VM. | `string` | `PortList` | +| `externalPorts` | Ports to forward from outside the cluster. | `[]int` | `[22]` | +| `runStrategy` | Requested running state of the VirtualMachineInstance | `string` | `Always` | +| `instanceType` | Virtual Machine instance type. | `string` | `u1.medium` | +| `instanceProfile` | Virtual Machine preferences profile. | `string` | `ubuntu` | +| `disks` | List of disks to attach. | `[]object` | `[]` | +| `disks[i].name` | Disk name. | `string` | `""` | +| `disks[i].bus` | Disk bus type (e.g. "sata"). | `string` | `""` | +| `networks` | Networks to attach the VM to. | `[]object` | `[]` | +| `networks[i].name` | Network attachment name. | `string` | `""` | +| `subnets` | Deprecated: use networks instead. | `[]object` | `[]` | +| `subnets[i].name` | Network attachment name. | `string` | `""` | +| `gpus` | List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM). | `[]object` | `[]` | +| `gpus[i].name` | The name of the GPU resource to attach. | `string` | `""` | +| `cpuModel` | Model specifies the CPU model inside the VMI. List of available models https://github.com/libvirt/libvirt/tree/master/src/cpu_map | `string` | `""` | +| `resources` | Resource configuration for the virtual machine. | `object` | `{}` | +| `resources.cpu` | Number of CPU cores allocated. | `quantity` | `""` | +| `resources.memory` | Amount of memory allocated. | `quantity` | `""` | +| `resources.sockets` | Number of CPU sockets (vCPU topology). | `quantity` | `""` | +| `sshKeys` | List of SSH public keys for authentication. | `[]string` | `[]` | +| `cloudInit` | Cloud-init user data. | `string` | `""` | +| `cloudInitSeed` | Seed string to generate SMBIOS UUID for the VM. | `string` | `""` | ## U Series diff --git a/packages/apps/vm-instance/templates/service.yaml b/packages/apps/vm-instance/templates/service.yaml index cfeffa81..b12f7612 100644 --- a/packages/apps/vm-instance/templates/service.yaml +++ b/packages/apps/vm-instance/templates/service.yaml @@ -7,12 +7,8 @@ metadata: apps.cozystack.io/user-service: "true" {{- include "virtual-machine.labels" . | nindent 4 }} {{- if .Values.external }} - service.kubernetes.io/service-proxy-name: "cozy-proxy" annotations: - networking.cozystack.io/wholeIP: {{ ternary "true" "false" (eq .Values.externalMethod "WholeIP") | quote }} - {{- if eq .Values.externalMethod "PortList" }} - networking.cozystack.io/allowICMP: {{ ternary "true" "false" (ne .Values.externalAllowICMP false) | quote }} - {{- end }} + networking.cozystack.io/wholeIP: "true" {{- end }} spec: type: {{ ternary "LoadBalancer" "ClusterIP" .Values.external }} diff --git a/packages/apps/vm-instance/values.schema.json b/packages/apps/vm-instance/values.schema.json index 01bd30a9..34f7f634 100644 --- a/packages/apps/vm-instance/values.schema.json +++ b/packages/apps/vm-instance/values.schema.json @@ -26,11 +26,6 @@ "type": "integer" } }, - "externalAllowICMP": { - "description": "Whether to accept ICMP traffic to the VM in PortList mode (preserves ping and PMTU discovery). No effect in WholeIP mode. Default true so ping behaves as users expect even when port filtering is in effect.", - "type": "boolean", - "default": true - }, "runStrategy": { "description": "Requested running state of the VirtualMachineInstance", "type": "string", diff --git a/packages/apps/vm-instance/values.yaml b/packages/apps/vm-instance/values.yaml index 92e399c2..f07b75d3 100644 --- a/packages/apps/vm-instance/values.yaml +++ b/packages/apps/vm-instance/values.yaml @@ -31,9 +31,6 @@ externalMethod: PortList externalPorts: - 22 -## @param {bool} externalAllowICMP - Whether to accept ICMP traffic to the VM in PortList mode (preserves ping and PMTU discovery). No effect in WholeIP mode. Default true so ping behaves as users expect even when port filtering is in effect. -externalAllowICMP: true - ## @enum {string} RunStrategy - Requested running state of the VirtualMachineInstance ## @value Always - VMI should always be running ## @value Halted - VMI should never be running diff --git a/packages/apps/vpn/templates/workloadmonitor.yaml b/packages/apps/vpn/templates/workloadmonitor.yaml index 7fc1ec7a..a75f7940 100644 --- a/packages/apps/vpn/templates/workloadmonitor.yaml +++ b/packages/apps/vpn/templates/workloadmonitor.yaml @@ -2,8 +2,6 @@ apiVersion: cozystack.io/v1alpha1 kind: WorkloadMonitor metadata: name: {{ $.Release.Name }} - labels: - workloads.cozystack.io/resource-preset: {{ .Values.resourcesPreset | quote }} spec: replicas: {{ .Values.replicas }} minReplicas: 1 diff --git a/packages/core/installer/values.yaml b/packages/core/installer/values.yaml index eef691ea..98443211 100644 --- a/packages/core/installer/values.yaml +++ b/packages/core/installer/values.yaml @@ -1,9 +1,9 @@ cozystackOperator: # Deployment variant: talos, generic, hosted variant: talos - image: ghcr.io/cozystack/cozystack/cozystack-operator:v1.3.0@sha256:62574f12486bb40c901cf5ed484cca264405ce5810196d86555cbb27cce1ba48 + image: ghcr.io/cozystack/cozystack/cozystack-operator:v1.2.1@sha256:c89352808022944c4791d63cf82cc95d78124ce799d355b60427f317087e8909 platformSourceUrl: 'oci://ghcr.io/cozystack/cozystack/cozystack-packages' - platformSourceRef: 'digest=sha256:a0b9ef938446b3132d3d22ad2262beb1027c48c9037b6c2346fdc2f19acd3036' + platformSourceRef: 'digest=sha256:d02f211f79d4912f04c21856de078b5b41bb3976d84504d37cf8d9108f7cbec6' # Generic variant configuration (only used when cozystackOperator.variant=generic) cozystack: # Kubernetes API server host (IP only, no protocol/port) diff --git a/packages/core/platform/images/migrations/migrations/37 b/packages/core/platform/images/migrations/migrations/37 index 856da262..b8ff3147 100755 --- a/packages/core/platform/images/migrations/migrations/37 +++ b/packages/core/platform/images/migrations/migrations/37 @@ -1,77 +1,42 @@ -#!/bin/bash +#!/bin/sh # Migration 37 --> 38 -# Pin PostgreSQL image to 17.7-standard-trixie for system databases. +# Backfill spec.version on postgreses.apps.cozystack.io resources. # -# This migration updates the imageName for all CNPG clusters belonging to -# system components (keycloak-db, grafana-db, alerta-db, seaweedfs-db, and -# harbor's dynamically-named DB): -# - If imageName is not set: pins to 17.7-standard-trixie -# - If imageName has any PG 17 tag: forces 17.7-standard-trixie -# - If imageName has a bare version tag (e.g. :18.1): appends -standard-trixie -# -# NOTE: This migration ONLY updates system CNPG Cluster resources (clusters.postgresql.cnpg.io), -# NOT user-created Postgres applications. +# Before this migration PostgreSQL had a default version field set to v18. +# This migration sets spec.version to "v17" for any postgres app resource that +# does not already have it set, to ensure compatibility with monitoring +# configurations that are hardcoded to PostgreSQL 17. set -euo pipefail -TARGET_IMAGE="ghcr.io/cloudnative-pg/postgresql:17.7-standard-trixie" +DEFAULT_VERSION="v17" -# Static system database names -STATIC_DB_NAMES="keycloak-db grafana-db alerta-db seaweedfs-db" +# Skip if the CRD does not exist (postgres was never installed) +if ! kubectl api-resources --api-group=apps.cozystack.io -o name 2>/dev/null | grep -q '^postgreses\.'; then + echo "CRD postgreses.apps.cozystack.io not found, skipping migration" + kubectl create configmap -n cozy-system cozystack-version \ + --from-literal=version=38 --dry-run=client -o yaml | kubectl apply -f- + exit 0 +fi -echo "=== Updating PostgreSQL images for system databases ===" -echo "Target image: $TARGET_IMAGE" +POSTGRESES=$(kubectl get postgreses.apps.cozystack.io -A -o jsonpath='{range .items[*]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}') +for resource in $POSTGRESES; do + NS="${resource%%/*}" + APP_NAME="${resource##*/}" -# Fetch all CNPG clusters with their name, namespace, imageName, and Helm release annotation in one call -ALL_CLUSTERS=$(kubectl get clusters.postgresql.cnpg.io -A \ - -o jsonpath='{range .items[*]}{.metadata.namespace}{"\t"}{.metadata.name}{"\t"}{.spec.imageName}{"\t"}{.metadata.annotations.meta\.helm\.sh/release-name}{"\n"}{end}' 2>/dev/null || true) - -while IFS=$'\t' read -r NAMESPACE CLUSTER_NAME CURRENT_IMAGE HELM_RELEASE; do - [ -z "$NAMESPACE" ] && continue - - # Check if cluster name matches one of the static system databases - MATCH=false - for db_name in $STATIC_DB_NAMES; do - if [ "$CLUSTER_NAME" = "$db_name" ]; then - MATCH=true - break - fi - done - - # For harbor: match clusters ending with -db whose Helm release ends with -system - if [ "$MATCH" = false ] && [[ "$CLUSTER_NAME" == *-db ]] && [[ "$HELM_RELEASE" == *-system ]]; then - MATCH=true - fi - - if [ "$MATCH" = false ]; then + # Skip if spec.version is already set + CURRENT_VER=$(kubectl get postgreses.apps.cozystack.io -n "$NS" "$APP_NAME" \ + -o jsonpath='{.spec.version}') + if [ -n "$CURRENT_VER" ]; then + echo "SKIP $NS/$APP_NAME: spec.version already set to '$CURRENT_VER'" continue fi - PATCH_IMAGE="" + echo "Patching postgres/$APP_NAME in $NS: setting version=$DEFAULT_VERSION" - if [ -z "$CURRENT_IMAGE" ]; then - # imageName not set — pin to target - PATCH_IMAGE="$TARGET_IMAGE" - echo "PATCH $NAMESPACE/$CLUSTER_NAME: imageName not set, setting to $PATCH_IMAGE" - elif [[ "$CURRENT_IMAGE" =~ :17\. ]]; then - # Any PG 17 image — force to the pinned 17.7-standard-trixie - PATCH_IMAGE="$TARGET_IMAGE" - echo "PATCH $NAMESPACE/$CLUSTER_NAME: PG 17 detected, $CURRENT_IMAGE -> $PATCH_IMAGE" - elif [[ "$CURRENT_IMAGE" =~ :[0-9]+\.[0-9]+$ ]]; then - # Bare version tag for other majors (e.g. :18.1) — append -standard-trixie suffix - PATCH_IMAGE="${CURRENT_IMAGE}-standard-trixie" - echo "PATCH $NAMESPACE/$CLUSTER_NAME: bare tag detected, $CURRENT_IMAGE -> $PATCH_IMAGE" - else - echo "SKIP $NAMESPACE/$CLUSTER_NAME: imageName already set to $CURRENT_IMAGE" - continue - fi - - kubectl patch clusters.postgresql.cnpg.io -n "$NAMESPACE" "$CLUSTER_NAME" \ - --type=merge \ - --patch "{\"spec\":{\"imageName\":\"${PATCH_IMAGE}\"}}" -done <<< "$ALL_CLUSTERS" - -echo "=== PostgreSQL image update completed ===" + kubectl patch postgreses.apps.cozystack.io -n "$NS" "$APP_NAME" --type=merge \ + --patch "{\"spec\":{\"version\":\"${DEFAULT_VERSION}\"}}" +done # Stamp version kubectl create configmap -n cozy-system cozystack-version \ diff --git a/packages/core/platform/images/migrations/migrations/38 b/packages/core/platform/images/migrations/migrations/38 deleted file mode 100755 index 59536cb1..00000000 --- a/packages/core/platform/images/migrations/migrations/38 +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/sh -# Migration 38 --> 39 -# Rename DataVolumes/PVCs in cozy-public from vm-image- to vm-default-images-. -# -# The vm-disk package previously looked up golden images with the prefix "vm-image-". -# The new vm-default-images package uses the prefix "vm-default-images-". -# This migration renames any existing DataVolumes so that live vm-disk resources -# continue to resolve their source PVC after upgrade. - -set -euo pipefail - -NAMESPACE="cozy-public" - -# Skip if CDI DataVolume CRD is not installed -if ! kubectl api-resources --api-group=cdi.kubevirt.io -o name 2>/dev/null | grep -q '^datavolumes\.'; then - echo "CDI DataVolume CRD not found, skipping rename" - kubectl create configmap -n cozy-system cozystack-version \ - --from-literal=version=39 --dry-run=client -o yaml | kubectl apply -f- - exit 0 -fi - -# Find DataVolumes whose name starts with "vm-image-" -DVNAMES=$(kubectl get datavolumes.cdi.kubevirt.io -n "$NAMESPACE" \ - -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null \ - | grep '^vm-image-' || true) - -if [ -z "$DVNAMES" ]; then - echo "No vm-image-* DataVolumes found in $NAMESPACE, nothing to rename" -else - for DVNAME in $DVNAMES; do - NEWNAME="vm-default-images-${DVNAME#vm-image-}" - echo "Renaming DataVolume $DVNAME -> $NEWNAME in namespace $NAMESPACE" - - # Export existing DV spec, swap name, and apply as new object - kubectl get datavolumes.cdi.kubevirt.io -n "$NAMESPACE" "$DVNAME" -o json \ - | jq --arg new "$NEWNAME" ' - del(.metadata.resourceVersion, .metadata.uid, .metadata.creationTimestamp, - .metadata.generation, .metadata.selfLink, .metadata.managedFields, - .status) - | .metadata.name = $new - ' \ - | kubectl apply -f - - - # Delete the old DataVolume - kubectl delete datavolumes.cdi.kubevirt.io -n "$NAMESPACE" "$DVNAME" --ignore-not-found=true - echo "Renamed $DVNAME -> $NEWNAME" - done -fi - -# Stamp version -kubectl create configmap -n cozy-system cozystack-version \ - --from-literal=version=39 --dry-run=client -o yaml | kubectl apply -f- diff --git a/packages/core/platform/sources/cozystack-scheduler.yaml b/packages/core/platform/sources/cozystack-scheduler.yaml index 52e85a4f..af4c8338 100644 --- a/packages/core/platform/sources/cozystack-scheduler.yaml +++ b/packages/core/platform/sources/cozystack-scheduler.yaml @@ -17,14 +17,3 @@ spec: install: namespace: kube-system releaseName: cozystack-scheduler - - name: linstor - dependsOn: - - cozystack.linstor-scheduler - components: - - name: cozystack-scheduler - path: system/cozystack-scheduler - valuesFiles: - - values-linstor.yaml - install: - namespace: kube-system - releaseName: cozystack-scheduler diff --git a/packages/core/platform/sources/hami.yaml b/packages/core/platform/sources/hami.yaml deleted file mode 100644 index 0184e405..00000000 --- a/packages/core/platform/sources/hami.yaml +++ /dev/null @@ -1,24 +0,0 @@ ---- -apiVersion: cozystack.io/v1alpha1 -kind: PackageSource -metadata: - name: cozystack.hami -spec: - sourceRef: - kind: OCIRepository - name: cozystack-packages - namespace: cozy-system - path: / - variants: - - name: default - dependsOn: - - cozystack.gpu-operator - components: - - name: hami - path: system/hami - valuesFiles: - - values.yaml - install: - privileged: true - namespace: cozy-hami - releaseName: hami diff --git a/packages/core/platform/sources/kubernetes-application.yaml b/packages/core/platform/sources/kubernetes-application.yaml index 088383ad..9787cb19 100644 --- a/packages/core/platform/sources/kubernetes-application.yaml +++ b/packages/core/platform/sources/kubernetes-application.yaml @@ -52,8 +52,6 @@ spec: path: system/cilium - name: kubernetes-gpu-operator path: system/gpu-operator - - name: kubernetes-hami - path: system/hami - name: kubernetes-vertical-pod-autoscaler path: system/vertical-pod-autoscaler - name: kubernetes-prometheus-operator-crds diff --git a/packages/core/platform/sources/linstor-gui.yaml b/packages/core/platform/sources/linstor-gui.yaml deleted file mode 100644 index 005e1fb7..00000000 --- a/packages/core/platform/sources/linstor-gui.yaml +++ /dev/null @@ -1,21 +0,0 @@ ---- -apiVersion: cozystack.io/v1alpha1 -kind: PackageSource -metadata: - name: cozystack.linstor-gui -spec: - sourceRef: - kind: OCIRepository - name: cozystack-packages - namespace: cozy-system - path: / - variants: - - name: default - dependsOn: - - cozystack.linstor - components: - - name: linstor-gui - path: system/linstor-gui - install: - namespace: cozy-linstor - releaseName: linstor-gui diff --git a/packages/core/platform/sources/networking.yaml b/packages/core/platform/sources/networking.yaml index bece76f8..8e8485b3 100644 --- a/packages/core/platform/sources/networking.yaml +++ b/packages/core/platform/sources/networking.yaml @@ -66,7 +66,6 @@ spec: path: system/cilium valuesFiles: - values.yaml - - values-apparmor.yaml install: privileged: true namespace: cozy-cilium @@ -118,7 +117,6 @@ spec: path: system/cilium valuesFiles: - values.yaml - - values-apparmor.yaml - values-kubeovn.yaml install: privileged: true diff --git a/packages/core/platform/sources/vm-default-images.yaml b/packages/core/platform/sources/vm-default-images.yaml deleted file mode 100644 index 16cd6a94..00000000 --- a/packages/core/platform/sources/vm-default-images.yaml +++ /dev/null @@ -1,22 +0,0 @@ ---- -apiVersion: cozystack.io/v1alpha1 -kind: PackageSource -metadata: - name: cozystack.vm-default-images -spec: - sourceRef: - kind: OCIRepository - name: cozystack-packages - namespace: cozy-system - path: / - variants: - - name: default - dependsOn: - - cozystack.networking - - cozystack.kubevirt-cdi - components: - - name: vm-default-images - path: system/vm-default-images - install: - namespace: cozy-system - releaseName: vm-default-images diff --git a/packages/core/platform/templates/apps.yaml b/packages/core/platform/templates/apps.yaml index 6e1ae25a..7ee0de52 100644 --- a/packages/core/platform/templates/apps.yaml +++ b/packages/core/platform/templates/apps.yaml @@ -30,7 +30,6 @@ stringData: expose-services: {{ .Values.publishing.exposedServices | join "," | quote }} expose-ingress: {{ .Values.publishing.ingressName | quote }} expose-external-ips: {{ .Values.publishing.externalIPs | join "," | quote }} - expose-mode: {{ .Values.publishing.exposure | default "externalIPs" | quote }} cluster-domain: {{ .Values.networking.clusterDomain | quote }} api-server-endpoint: {{ .Values.publishing.apiServerEndpoint | quote }} {{- with .Values.branding }} diff --git a/packages/core/platform/templates/bundles/iaas.yaml b/packages/core/platform/templates/bundles/iaas.yaml index 66ec98b2..bd002c16 100644 --- a/packages/core/platform/templates/bundles/iaas.yaml +++ b/packages/core/platform/templates/bundles/iaas.yaml @@ -8,9 +8,7 @@ {{- end -}} {{include "cozystack.platform.package" (list "cozystack.kubevirt" "default" $ $kubevirtComponents) }} {{include "cozystack.platform.package.default" (list "cozystack.kubevirt-cdi" $) }} -{{include "cozystack.platform.package.optional.default" (list "cozystack.vm-default-images" $) }} {{include "cozystack.platform.package.optional.default" (list "cozystack.gpu-operator" $) }} -{{include "cozystack.platform.package.optional.default" (list "cozystack.hami" $) }} {{include "cozystack.platform.package.default" (list "cozystack.kamaji" $) }} {{include "cozystack.platform.package.default" (list "cozystack.capi-operator" $) }} {{include "cozystack.platform.package.default" (list "cozystack.capi-provider-bootstrap-kubeadm" $) }} diff --git a/packages/core/platform/templates/bundles/system.yaml b/packages/core/platform/templates/bundles/system.yaml index 6c587c82..795b523f 100644 --- a/packages/core/platform/templates/bundles/system.yaml +++ b/packages/core/platform/templates/bundles/system.yaml @@ -25,12 +25,10 @@ {{include "cozystack.platform.package" (list "cozystack.networking" "kubeovn-cilium" $ $networkingComponents) }} {{include "cozystack.platform.system.common-packages" $ }} {{include "cozystack.platform.package.default" (list "cozystack.linstor" $) }} -{{include "cozystack.platform.package" (list "cozystack.cozystack-scheduler" "linstor" $) }} {{- end }} {{- if eq .Values.bundles.system.variant "isp-hosted" }} {{include "cozystack.platform.package" (list "cozystack.networking" "noop" $) }} -{{include "cozystack.platform.package.default" (list "cozystack.cozystack-scheduler" $) }} {{- end }} {{- if eq .Values.bundles.system.variant "isp-full-generic" }} @@ -94,7 +92,6 @@ {{- /* Pass talos.enabled: false to linstor for generic Linux */ -}} {{- $linstorComponents := dict "linstor" (dict "values" (dict "talos" (dict "enabled" false))) -}} {{include "cozystack.platform.package" (list "cozystack.linstor" "default" $ $linstorComponents) }} -{{include "cozystack.platform.package" (list "cozystack.cozystack-scheduler" "linstor" $) }} {{- end }} # Cozystack Engine @@ -105,6 +102,9 @@ {{- /* cozystack-api DaemonSet */ -}} {{- $apiValues := dict "cozystackAPI" (dict "nodeSelector" $genericNodeSelector) -}} {{- $_ := set $cozystackEngineComponents "cozystack-api" (dict "values" $apiValues) -}} +{{- /* lineage-controller-webhook DaemonSet */ -}} +{{- $lineageValues := dict "lineageControllerWebhook" (dict "nodeSelector" $genericNodeSelector) -}} +{{- $_ := set $cozystackEngineComponents "lineage-controller-webhook" (dict "values" $lineageValues) -}} {{- end -}} {{- if .Values.authentication.oidc.enabled }} {{include "cozystack.platform.package" (list "cozystack.cozystack-engine" "oidc" $ $cozystackEngineComponents) }} @@ -130,6 +130,7 @@ {{include "cozystack.platform.package.default" (list "cozystack.cozystack-basics" $) }} {{include "cozystack.platform.package.default" (list "cozystack.backupstrategy-controller" $) }} {{include "cozystack.platform.package.default" (list "cozystack.backup-controller" $) }} +{{include "cozystack.platform.package.default" (list "cozystack.cozystack-scheduler" $) }} {{include "cozystack.platform.package.default" (list "cozystack.vertical-pod-autoscaler" $) }} {{include "cozystack.platform.package.default" (list "cozystack.metrics-server" $) }} {{- $monitoringAgentsComponents := dict "monitoring-agents" (dict "values" (dict "global" (dict "target" "tenant-root"))) -}} @@ -148,7 +149,6 @@ {{include "cozystack.platform.package.optional.default" (list "cozystack.external-dns-application" $) }} {{include "cozystack.platform.package.optional.default" (list "cozystack.external-secrets-operator" $) }} {{include "cozystack.platform.package.optional.default" (list "cozystack.velero" $) }} -{{include "cozystack.platform.package.optional.default" (list "cozystack.linstor-gui" $) }} {{- if has "cozystack.bootbox" (default (list) .Values.bundles.enabledPackages) }} {{include "cozystack.platform.package.default" (list "cozystack.bootbox-application" $) }} {{include "cozystack.platform.package.default" (list "cozystack.bootbox" $) }} diff --git a/packages/core/platform/values.yaml b/packages/core/platform/values.yaml index 605e23bc..a2c97d8c 100644 --- a/packages/core/platform/values.yaml +++ b/packages/core/platform/values.yaml @@ -5,8 +5,8 @@ sourceRef: path: / migrations: enabled: false - image: ghcr.io/cozystack/cozystack/platform-migrations:v1.3.0@sha256:555e4b76421361805a84bc9088b01b23a9c4a9430bd8ebd2db82ef9677d7008c - targetVersion: 39 + image: ghcr.io/cozystack/cozystack/platform-migrations:v1.2.1@sha256:e8fcf006a4451fc0e961455e9b27a61b7103ee49b1a81fe5e4662ffed093fad6 + targetVersion: 37 # Bundle deployment configuration bundles: system: @@ -45,41 +45,6 @@ publishing: - cdi-uploadproxy apiServerEndpoint: "" # example: "https://api.example.org" externalIPs: [] - # Exposure mode for the ingress-nginx Service. When "externalIPs" (current - # default) is selected, the Service is created as ClusterIP with - # Service.spec.externalIPs set from publishing.externalIPs. When - # "loadBalancer" is selected, the Service is type: LoadBalancer and a - # CiliumLoadBalancerIPPool makes those same addresses allocatable via LB IPAM. - # - # Service.spec.externalIPs is deprecated upstream in Kubernetes v1.36 - # (KEP-5707). The AllowServiceExternalIPs feature gate is expected to default - # to false around v1.40 and the implementation removed around v1.43 — switch - # to "loadBalancer" before upgrading past v1.40. - # - # Caveats for the "loadBalancer" mode: - # - publishing.externalIPs must contain at least one non-empty address, - # otherwise the chart render fails with an explicit error (a LoadBalancer - # Service without a pool would sit in forever). - # - The ingress-nginx Service is created with externalTrafficPolicy: Local - # to preserve the client source IP. Traffic arriving on a node that does - # not host an ingress-nginx pod is dropped, so the external IP must be - # routed to a node that runs the ingress pod (floating IP / keepalived / - # upstream router / podAntiAffinity). - # - Cilium does NOT announce the IP on its own unless L2 announcements or - # BGP are enabled in the Cilium values (disabled by default in Cozystack). - # This mode assumes the operator already routes the externalIPs to a - # cluster node; enabling announcements is out of scope for this setting. - # - Switching this value on a running cluster causes the ingress-nginx - # Service to be recreated (the HelmRelease has upgrade.force: true and - # the Service kind changes between ClusterIP and LoadBalancer). Expect a - # brief interruption of ingress traffic during the flip. - # - # Scope: this setting only controls the ingress-nginx Service. Other - # cozystack components that currently write Service.spec.externalIPs directly - # (e.g. the vpn app at packages/apps/vpn/templates/service.yaml) are NOT - # migrated by flipping this value and must be addressed separately before - # the AllowServiceExternalIPs feature gate flips to off in ~v1.40. - exposure: externalIPs # "externalIPs" or "loadBalancer" certificates: solver: http01 # "http01" or "dns01" issuerName: letsencrypt-prod diff --git a/packages/core/testing/values.yaml b/packages/core/testing/values.yaml index 9d4bdd10..b17da614 100644 --- a/packages/core/testing/values.yaml +++ b/packages/core/testing/values.yaml @@ -1,2 +1,2 @@ e2e: - image: ghcr.io/cozystack/cozystack/e2e-sandbox:v1.3.0@sha256:0367a03b981df2a3ea13f411d4cb7869c2bf2c89c07d3d5c8971b9a28921ccef + image: ghcr.io/cozystack/cozystack/e2e-sandbox:v1.2.1@sha256:7964a3e4b11053887be201d62f94138c3a7836c861036c26fb833aa1fcb934e1 diff --git a/packages/extra/bootbox/images/matchbox.tag b/packages/extra/bootbox/images/matchbox.tag index f51c7675..0ba57d47 100644 --- a/packages/extra/bootbox/images/matchbox.tag +++ b/packages/extra/bootbox/images/matchbox.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/matchbox:v1.3.0@sha256:85b8e04bf6f0690612dd63e80475df269f4a436d16680f8a40f2860cf16e2f74 +ghcr.io/cozystack/cozystack/matchbox:v1.2.1@sha256:64685523ba693964f22ce7d7dc545bf7bf96dea34e5ae525a2315614adef4b3d diff --git a/packages/extra/bootbox/templates/matchbox/ingress.yaml b/packages/extra/bootbox/templates/matchbox/ingress.yaml index 5eef3489..4262cd10 100644 --- a/packages/extra/bootbox/templates/matchbox/ingress.yaml +++ b/packages/extra/bootbox/templates/matchbox/ingress.yaml @@ -10,7 +10,7 @@ metadata: app: bootbox annotations: {{- if eq $solver "http01" }} - acme.cert-manager.io/http01-ingress-ingressclassname: {{ $ingress }} + acme.cert-manager.io/http01-ingress-class: {{ $ingress }} {{- end }} cert-manager.io/cluster-issuer: {{ $clusterIssuer }} {{- if .Values.whitelistHTTP }} diff --git a/packages/extra/etcd/Makefile b/packages/extra/etcd/Makefile index 2b3ed61e..f37d6e1e 100644 --- a/packages/extra/etcd/Makefile +++ b/packages/extra/etcd/Makefile @@ -5,6 +5,3 @@ include ../../../hack/package.mk generate: cozyvalues-gen -v values.yaml -s values.schema.json -r README.md ../../../hack/update-crd.sh - -test: - helm unittest . diff --git a/packages/extra/etcd/templates/hook/job.yaml b/packages/extra/etcd/templates/hook/job.yaml new file mode 100644 index 00000000..3bf2f84b --- /dev/null +++ b/packages/extra/etcd/templates/hook/job.yaml @@ -0,0 +1,39 @@ +{{- $shouldUpdateCerts := true }} +{{- $configMap := lookup "v1" "ConfigMap" .Release.Namespace "etcd-deployed-version" }} +{{- if $configMap }} + {{- $deployedVersion := index $configMap "data" "version" }} + {{- if $deployedVersion | semverCompare ">= 2.6.1" }} + {{- $shouldUpdateCerts = false }} + {{- end }} +{{- end }} + +{{- if $shouldUpdateCerts }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: etcd-hook + annotations: + helm.sh/hook: post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +spec: + template: + metadata: + labels: + policy.cozystack.io/allow-to-apiserver: "true" + spec: + serviceAccountName: etcd-hook + containers: + - name: kubectl + image: docker.io/alpine/k8s:1.33.4 + command: + - sh + args: + - -exc + - |- + kubectl --namespace={{ .Release.Namespace }} delete secrets etcd-ca-tls etcd-peer-ca-tls + sleep 10 + kubectl --namespace={{ .Release.Namespace }} delete secrets etcd-client-tls etcd-peer-tls etcd-server-tls + kubectl --namespace={{ .Release.Namespace }} delete pods --selector=app.kubernetes.io/instance=etcd,app.kubernetes.io/managed-by=etcd-operator,app.kubernetes.io/name=etcd,cozystack.io/service=etcd + restartPolicy: Never +{{- end }} diff --git a/packages/extra/etcd/templates/hook/role.yaml b/packages/extra/etcd/templates/hook/role.yaml new file mode 100644 index 00000000..327eeadb --- /dev/null +++ b/packages/extra/etcd/templates/hook/role.yaml @@ -0,0 +1,26 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + annotations: + helm.sh/hook: post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + name: etcd-hook +rules: +- apiGroups: + - "" + resources: + - secrets + - pods + verbs: + - get + - list + - watch + - delete +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch diff --git a/packages/extra/etcd/templates/hook/rolebinding.yaml b/packages/extra/etcd/templates/hook/rolebinding.yaml new file mode 100644 index 00000000..0ee0ffd1 --- /dev/null +++ b/packages/extra/etcd/templates/hook/rolebinding.yaml @@ -0,0 +1,15 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: etcd-hook + annotations: + helm.sh/hook: post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: etcd-hook +subjects: + - kind: ServiceAccount + name: etcd-hook + namespace: {{ .Release.Namespace | quote }} diff --git a/packages/extra/etcd/templates/hook/serviceaccount.yaml b/packages/extra/etcd/templates/hook/serviceaccount.yaml new file mode 100644 index 00000000..552fb5fc --- /dev/null +++ b/packages/extra/etcd/templates/hook/serviceaccount.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: etcd-hook + annotations: + helm.sh/hook: post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded diff --git a/packages/extra/etcd/templates/version.yaml b/packages/extra/etcd/templates/version.yaml new file mode 100644 index 00000000..cc9375bb --- /dev/null +++ b/packages/extra/etcd/templates/version.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: etcd-deployed-version +data: + version: {{ .Chart.Version }} diff --git a/packages/extra/etcd/tests/no-post-upgrade-hook_test.yaml b/packages/extra/etcd/tests/no-post-upgrade-hook_test.yaml deleted file mode 100644 index 0e4f5aa3..00000000 --- a/packages/extra/etcd/tests/no-post-upgrade-hook_test.yaml +++ /dev/null @@ -1,34 +0,0 @@ -suite: etcd chart does not ship a destructive post-upgrade cert-regeneration hook - -release: - name: etcd - namespace: tenant-root - -templates: - - templates/check-release-name.yaml - - templates/dashboard-resourcemap.yaml - - templates/datastore.yaml - - templates/etcd-defrag.yaml - - templates/hook/job.yaml - - templates/podscrape.yaml - - templates/prometheus-rules.yaml - - templates/version.yaml - -tests: - - it: renders no Job named etcd-hook - documentSelector: - path: metadata.name - value: etcd-hook - skipEmptyTemplates: true - asserts: - - hasDocuments: - count: 0 - - - it: renders no ConfigMap named etcd-deployed-version - documentSelector: - path: metadata.name - value: etcd-deployed-version - skipEmptyTemplates: true - asserts: - - hasDocuments: - count: 0 diff --git a/packages/extra/ingress/Makefile b/packages/extra/ingress/Makefile index 65b53c03..958ce484 100644 --- a/packages/extra/ingress/Makefile +++ b/packages/extra/ingress/Makefile @@ -10,6 +10,3 @@ get-cloudflare-ips: generate: cozyvalues-gen -v values.yaml -s values.schema.json -r README.md ../../../hack/update-crd.sh - -test: - helm unittest . diff --git a/packages/extra/ingress/README.md b/packages/extra/ingress/README.md index c541d8e9..0e786dfb 100644 --- a/packages/extra/ingress/README.md +++ b/packages/extra/ingress/README.md @@ -14,17 +14,3 @@ | `resources.memory` | Memory (RAM) available to each replica. | `quantity` | `""` | | `resourcesPreset` | Default sizing preset used when `resources` is omitted. | `string` | `micro` | - -## Exposure mode - -The ingress Service type is driven by the cluster-wide `publishing.exposure` value in the platform chart, not by any key in this package. Two modes exist: - -- `externalIPs` (default) has three rendered shapes: - - Release namespace matches `publishing.ingressName` AND `publishing.externalIPs` is non-empty → Service is `ClusterIP` with `Service.spec.externalIPs` set from that list and `externalTrafficPolicy: Cluster`. - - Release namespace matches `publishing.ingressName` but `publishing.externalIPs` is empty → Service falls back to `type: LoadBalancer` with `externalTrafficPolicy: Local`. - - Release namespace does not match `publishing.ingressName` (non-root tenants) → Service is `type: LoadBalancer` with `externalTrafficPolicy: Local`. - `Service.spec.externalIPs` is deprecated upstream in Kubernetes v1.36 (KEP-5707); plan migration before v1.40. -- `loadBalancer` — Service is `type: LoadBalancer` with `externalTrafficPolicy: Local`, and a `CiliumLoadBalancerIPPool` makes the addresses in `publishing.externalIPs` allocatable via Cilium LB IPAM. Requires `publishing.externalIPs` to contain at least one non-empty address (render fails otherwise) and assumes the addresses are already routed to a cluster node (floating IP / upstream router). See the inline comment on `publishing.exposure` in the platform chart for full caveats, including the note that switching the value on a running cluster causes the ingress Service to be recreated. - -This setting only migrates ingress-nginx away from `Service.spec.externalIPs`. Other cozystack components that use the same deprecated field (e.g. the `vpn` app) must be migrated separately before Kubernetes v1.40 flips the `AllowServiceExternalIPs` feature gate off. - diff --git a/packages/extra/ingress/templates/nginx-ingress.yaml b/packages/extra/ingress/templates/nginx-ingress.yaml index 8e1f00fb..ca50d276 100644 --- a/packages/extra/ingress/templates/nginx-ingress.yaml +++ b/packages/extra/ingress/templates/nginx-ingress.yaml @@ -1,19 +1,5 @@ {{- $exposeIngress := (index .Values._cluster "expose-ingress") | default "tenant-root" }} {{- $exposeExternalIPs := (index .Values._cluster "expose-external-ips") | default "" | nospace }} -{{- $exposeMode := (index .Values._cluster "expose-mode") | default "externalIPs" }} -{{- $exposeIPsList := list }} -{{- range splitList "," $exposeExternalIPs }} - {{- $ip := . | trim }} - {{- if $ip }} - {{- $exposeIPsList = append $exposeIPsList $ip }} - {{- end }} -{{- end }} -{{- if not (has $exposeMode (list "externalIPs" "loadBalancer")) }} -{{- fail (printf "unknown publishing.exposure mode %q: must be \"externalIPs\" or \"loadBalancer\"" $exposeMode) }} -{{- end }} -{{- if and (eq $exposeMode "loadBalancer") (eq $exposeIngress .Release.Namespace) (not $exposeIPsList) }} -{{- fail "publishing.exposure=loadBalancer requires publishing.externalIPs to contain at least one non-empty address: the CiliumLoadBalancerIPPool has nothing to advertise and the Service would stay in ." }} -{{- end }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: @@ -55,12 +41,9 @@ spec: enabled: false {{- end }} service: - {{- if and (eq $exposeIngress .Release.Namespace) (eq $exposeMode "loadBalancer") }} - type: LoadBalancer - externalTrafficPolicy: Local - {{- else if and (eq $exposeIngress .Release.Namespace) $exposeIPsList }} + {{- if and (eq $exposeIngress .Release.Namespace) $exposeExternalIPs }} externalIPs: - {{- toYaml $exposeIPsList | nindent 12 }} + {{- toYaml (splitList "," $exposeExternalIPs) | nindent 12 }} type: ClusterIP externalTrafficPolicy: Cluster {{- else }} diff --git a/packages/extra/ingress/tests/exposure_test.yaml b/packages/extra/ingress/tests/exposure_test.yaml deleted file mode 100644 index 1c76b3da..00000000 --- a/packages/extra/ingress/tests/exposure_test.yaml +++ /dev/null @@ -1,165 +0,0 @@ -suite: ingress exposure modes -templates: - - templates/nginx-ingress.yaml - -release: - name: ingress - namespace: tenant-root - -tests: - - it: default exposure (externalIPs) renders ClusterIP Service with spec.externalIPs - set: - _cluster: - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10,192.0.2.11" - asserts: - - template: templates/nginx-ingress.yaml - equal: - path: spec.values.ingress-nginx.controller.service.type - value: ClusterIP - - template: templates/nginx-ingress.yaml - equal: - path: spec.values.ingress-nginx.controller.service.externalTrafficPolicy - value: Cluster - - template: templates/nginx-ingress.yaml - equal: - path: spec.values.ingress-nginx.controller.service.externalIPs - value: - - 192.0.2.10 - - 192.0.2.11 - - template: templates/nginx-ingress.yaml - notExists: - path: spec.values.ingress-nginx.controller.service.labels - - - it: legacy config without expose-mode falls back to externalIPs behavior - set: - _cluster: - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10" - asserts: - - template: templates/nginx-ingress.yaml - equal: - path: spec.values.ingress-nginx.controller.service.type - value: ClusterIP - - - it: externalIPs mode in a namespace other than publishing.ingressName renders LoadBalancer fallback without externalIPs - set: - _cluster: - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10" - release: - namespace: tenant-other - asserts: - - template: templates/nginx-ingress.yaml - equal: - path: spec.values.ingress-nginx.controller.service.type - value: LoadBalancer - - template: templates/nginx-ingress.yaml - equal: - path: spec.values.ingress-nginx.controller.service.externalTrafficPolicy - value: Local - - template: templates/nginx-ingress.yaml - notExists: - path: spec.values.ingress-nginx.controller.service.externalIPs - - - it: loadBalancer mode renders LoadBalancer Service without externalIPs on the Service - set: - _cluster: - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10,192.0.2.11" - expose-mode: loadBalancer - asserts: - - template: templates/nginx-ingress.yaml - equal: - path: spec.values.ingress-nginx.controller.service.type - value: LoadBalancer - - template: templates/nginx-ingress.yaml - equal: - path: spec.values.ingress-nginx.controller.service.externalTrafficPolicy - value: Local - - template: templates/nginx-ingress.yaml - notExists: - path: spec.values.ingress-nginx.controller.service.labels - - template: templates/nginx-ingress.yaml - notExists: - path: spec.values.ingress-nginx.controller.service.externalIPs - - - it: loadBalancer mode without externalIPs fails chart render with explicit message - set: - _cluster: - expose-ingress: tenant-root - expose-external-ips: "" - expose-mode: loadBalancer - asserts: - - template: templates/nginx-ingress.yaml - failedTemplate: - errorMessage: "publishing.exposure=loadBalancer requires publishing.externalIPs to contain at least one non-empty address: the CiliumLoadBalancerIPPool has nothing to advertise and the Service would stay in ." - - - it: unknown exposure mode is rejected with a clear error (case-sensitive enum) - set: - _cluster: - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10" - expose-mode: LoadBalancer - asserts: - - template: templates/nginx-ingress.yaml - failedTemplate: - errorMessage: "unknown publishing.exposure mode \"LoadBalancer\": must be \"externalIPs\" or \"loadBalancer\"" - - - it: another typo in exposure mode also fails - set: - _cluster: - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10" - expose-mode: loadbalancer - asserts: - - template: templates/nginx-ingress.yaml - failedTemplate: - errorMessage: "unknown publishing.exposure mode \"loadbalancer\": must be \"externalIPs\" or \"loadBalancer\"" - - - it: loadBalancer mode in a namespace other than publishing.ingressName falls back to LoadBalancer Service without pool - set: - _cluster: - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10" - expose-mode: loadBalancer - release: - namespace: tenant-other - asserts: - - template: templates/nginx-ingress.yaml - equal: - path: spec.values.ingress-nginx.controller.service.type - value: LoadBalancer - - template: templates/nginx-ingress.yaml - equal: - path: spec.values.ingress-nginx.controller.service.externalTrafficPolicy - value: Local - - template: templates/nginx-ingress.yaml - notExists: - path: spec.values.ingress-nginx.controller.service.labels - - template: templates/nginx-ingress.yaml - notExists: - path: spec.values.ingress-nginx.controller.service.externalIPs - - - it: loadBalancer mode with only-empty externalIPs fails chart render (comma-only input) - set: - _cluster: - expose-ingress: tenant-root - expose-external-ips: ",," - expose-mode: loadBalancer - asserts: - - template: templates/nginx-ingress.yaml - failedTemplate: - errorMessage: "publishing.exposure=loadBalancer requires publishing.externalIPs to contain at least one non-empty address: the CiliumLoadBalancerIPPool has nothing to advertise and the Service would stay in ." - - - it: externalIPs mode also filters out empty entries (trailing comma) - set: - _cluster: - expose-ingress: tenant-root - expose-external-ips: "192.0.2.10," - asserts: - - template: templates/nginx-ingress.yaml - equal: - path: spec.values.ingress-nginx.controller.service.externalIPs - value: - - 192.0.2.10 diff --git a/packages/extra/seaweedfs/images/objectstorage-sidecar.tag b/packages/extra/seaweedfs/images/objectstorage-sidecar.tag index 9b413da5..bb0702e8 100644 --- a/packages/extra/seaweedfs/images/objectstorage-sidecar.tag +++ b/packages/extra/seaweedfs/images/objectstorage-sidecar.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.3.0@sha256:5bcccbdb13979a16cee535eb5fbcdf0d87973689010a10b7b25e55c5be3edaa6 +ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.2.1@sha256:1ad07fb9e96477e5c322f240f6023ceedd8762261d8379ec784dde92797ee206 diff --git a/packages/extra/seaweedfs/templates/seaweedfs.yaml b/packages/extra/seaweedfs/templates/seaweedfs.yaml index 2f5720ca..42441b26 100644 --- a/packages/extra/seaweedfs/templates/seaweedfs.yaml +++ b/packages/extra/seaweedfs/templates/seaweedfs.yaml @@ -243,7 +243,7 @@ spec: nginx.ingress.kubernetes.io/proxy-body-size: "0" nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" {{- if eq $solver "http01" }} - acme.cert-manager.io/http01-ingress-ingressclassname: {{ $ingress }} + acme.cert-manager.io/http01-ingress-class: {{ $ingress }} {{- end }} cert-manager.io/cluster-issuer: {{ $clusterIssuer }} tls: diff --git a/packages/system/backup-controller/definitions/backups.cozystack.io_backups.yaml b/packages/system/backup-controller/definitions/backups.cozystack.io_backups.yaml index d48df533..6e031a84 100644 --- a/packages/system/backup-controller/definitions/backups.cozystack.io_backups.yaml +++ b/packages/system/backup-controller/definitions/backups.cozystack.io_backups.yaml @@ -142,6 +142,14 @@ spec: required: - uri type: object + underlyingResources: + description: |- + Holds application-specific resource metadata discovered during backup. + The payload is a self-typed JSON object carrying an inlined TypeMeta + (kind/apiVersion) so the consuming controller can dispatch on the + application kind. + type: object + x-kubernetes-preserve-unknown-fields: true conditions: description: Conditions represents the latest available observations of a Backup's state. @@ -205,14 +213,6 @@ spec: Phase is a simple, high-level summary of the backup's state. Typical values are: Pending, Ready, Failed. type: string - underlyingResources: - description: |- - UnderlyingResources holds application-specific resource metadata discovered - during backup (e.g., VM disks, network configuration). The payload is a - self-typed JSON object carrying an inlined TypeMeta (kind/apiVersion) so - the consuming controller can dispatch on the application kind. - type: object - x-kubernetes-preserve-unknown-fields: true type: object type: object selectableFields: diff --git a/packages/system/backup-controller/definitions/backups.cozystack.io_restorejobs.yaml b/packages/system/backup-controller/definitions/backups.cozystack.io_restorejobs.yaml index b9465994..400737fc 100644 --- a/packages/system/backup-controller/definitions/backups.cozystack.io_restorejobs.yaml +++ b/packages/system/backup-controller/definitions/backups.cozystack.io_restorejobs.yaml @@ -59,12 +59,6 @@ spec: type: string type: object x-kubernetes-map-type: atomic - options: - description: |- - Options is a driver-specific blob of restore options, typed based on - targetApplicationRef and the current controller implementation. - type: object - x-kubernetes-preserve-unknown-fields: true targetApplicationRef: description: |- TargetApplicationRef refers to the application into which the backup diff --git a/packages/system/backup-controller/values.yaml b/packages/system/backup-controller/values.yaml index 34e033ce..ff5fe774 100644 --- a/packages/system/backup-controller/values.yaml +++ b/packages/system/backup-controller/values.yaml @@ -1,5 +1,5 @@ backupController: - image: "ghcr.io/cozystack/cozystack/backup-controller:v1.3.0@sha256:e1a083dc92f26dfef004f47c1cd20a6357174aad835004f58e751c494b76649a" + image: "ghcr.io/cozystack/cozystack/backup-controller:v1.2.1@sha256:6e515cdfe302bb079dafa0f7f302c6ff7b7e9c02a75574880a45e35d5acb1f59" replicas: 2 debug: false metrics: diff --git a/packages/system/backupstrategy-controller/templates/rbac-bind.yaml b/packages/system/backupstrategy-controller/templates/rbac-bind.yaml index 03578cc2..4ecacce0 100644 --- a/packages/system/backupstrategy-controller/templates/rbac-bind.yaml +++ b/packages/system/backupstrategy-controller/templates/rbac-bind.yaml @@ -10,3 +10,17 @@ subjects: - kind: ServiceAccount name: backupstrategy-controller namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: backups.cozystack.io:strategy-controller:velero-configmaps + namespace: cozy-velero +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: backups.cozystack.io:strategy-controller:velero-configmaps +subjects: +- kind: ServiceAccount + name: backupstrategy-controller + namespace: {{ .Release.Namespace }} diff --git a/packages/system/backupstrategy-controller/templates/rbac.yaml b/packages/system/backupstrategy-controller/templates/rbac.yaml index 634ea88b..2216577e 100644 --- a/packages/system/backupstrategy-controller/templates/rbac.yaml +++ b/packages/system/backupstrategy-controller/templates/rbac.yaml @@ -27,9 +27,7 @@ rules: resources: ["pods"] verbs: ["get", "list", "watch"] # ConfigMaps: controller-runtime cache requires cluster-scoped list/watch; -# create/update/delete is scoped to cozy-velero via a Role shipped by the -# velero chart (packages/system/velero/templates/backupstrategy-controller-rbac.yaml) -# so it is only created when velero is installed. +# create/update/delete is scoped to cozy-velero via the Role below. - apiGroups: [""] resources: ["configmaps"] verbs: ["get", "list", "watch"] @@ -48,10 +46,10 @@ rules: - apiGroups: ["cdi.kubevirt.io"] resources: ["datavolumes"] verbs: ["delete"] -# HelmReleases: pre-restore suspends HRs; post-restore rename creates new HR and deletes old +# HelmReleases: pre-restore suspends HRs to prevent Flux interference - apiGroups: ["helm.toolkit.fluxcd.io"] resources: ["helmreleases"] - verbs: ["get", "create", "update", "delete"] + verbs: ["get", "update"] # PVCs and PVs: pre-restore renames PVCs (delete old, create new) and patches PV reclaim policy - apiGroups: [""] resources: ["persistentvolumeclaims"] @@ -67,11 +65,6 @@ rules: - apiGroups: ["velero.io"] resources: ["deletebackuprequests"] verbs: ["create", "get", "list", "watch"] -# Namespaces: validate target namespace exists for cross-namespace restore -# controller-runtime cache requires list/watch for any resource accessed via r.Get() -- apiGroups: [""] - resources: ["namespaces"] - verbs: ["get", "list", "watch"] # Events from Recorder.Event() calls - apiGroups: [""] resources: ["events"] @@ -80,3 +73,14 @@ rules: - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "list", "watch", "create", "update", "patch"] +--- +# To create ResourceModifiers in ConfigMaps for Restore in Velero install namespace. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: backups.cozystack.io:strategy-controller:velero-configmaps + namespace: cozy-velero +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["create", "get", "list", "delete", "deletecollection", "patch", "update"] diff --git a/packages/system/backupstrategy-controller/values.yaml b/packages/system/backupstrategy-controller/values.yaml index d6453124..bd3cad8c 100644 --- a/packages/system/backupstrategy-controller/values.yaml +++ b/packages/system/backupstrategy-controller/values.yaml @@ -1,5 +1,5 @@ backupStrategyController: - image: "ghcr.io/cozystack/cozystack/backupstrategy-controller:v1.3.0@sha256:be0a9ec1f4307064b16388a24628aee46e06252738338add80b99ea1e04e62bf" + image: "ghcr.io/cozystack/cozystack/backupstrategy-controller:v1.2.1@sha256:3544a08bfac2df6dcb1842f73747768b24625c6d6b97230be9a00f80637f0d32" replicas: 2 debug: false metrics: diff --git a/packages/system/bucket/images/s3manager.tag b/packages/system/bucket/images/s3manager.tag index 889db0af..6c3d1449 100644 --- a/packages/system/bucket/images/s3manager.tag +++ b/packages/system/bucket/images/s3manager.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/s3manager:v0.5.0@sha256:fb65e734bbdbdaef2238769cc2ecfb54dbddaf1e0952ad438a9b3e26b9dbb4b5 +ghcr.io/cozystack/cozystack/s3manager:v0.5.0@sha256:73382d7911274e6528cc2272ca9bfdb095c59e78845a3cdffd39d6c8ac29d920 diff --git a/packages/system/bucket/templates/ingress.yaml b/packages/system/bucket/templates/ingress.yaml index b7ffaf8b..6e5028fc 100644 --- a/packages/system/bucket/templates/ingress.yaml +++ b/packages/system/bucket/templates/ingress.yaml @@ -12,7 +12,7 @@ metadata: nginx.ingress.kubernetes.io/proxy-read-timeout: "99999" nginx.ingress.kubernetes.io/proxy-send-timeout: "99999" {{- if eq $solver "http01" }} - acme.cert-manager.io/http01-ingress-ingressclassname: {{ $ingress }} + acme.cert-manager.io/http01-ingress-class: {{ $ingress }} {{- end }} cert-manager.io/cluster-issuer: {{ $clusterIssuer }} spec: diff --git a/packages/system/cert-manager-issuers/templates/cluster-issuers.yaml b/packages/system/cert-manager-issuers/templates/cluster-issuers.yaml index 3b582082..442f1fb3 100644 --- a/packages/system/cert-manager-issuers/templates/cluster-issuers.yaml +++ b/packages/system/cert-manager-issuers/templates/cluster-issuers.yaml @@ -1,7 +1,6 @@ {{- $solver := (index .Values._cluster "solver") | default "http01" }} -{{- $exposeIngress := (index .Values._cluster "expose-ingress") | default "tenant-root" }} -apiVersion: cert-manager.io/v1 +apiVersion: cert-manager.io/v1 kind: ClusterIssuer metadata: name: letsencrypt-prod @@ -18,9 +17,9 @@ spec: name: cloudflare-api-token-secret key: api-token {{- else }} - http01: - ingress: - ingressClassName: {{ $exposeIngress }} + http01: + ingress: + class: nginx {{- end }} --- @@ -42,9 +41,9 @@ spec: name: cloudflare-api-token-secret key: api-token {{- else }} - http01: - ingress: - ingressClassName: {{ $exposeIngress }} + http01: + ingress: + class: nginx {{- end }} --- diff --git a/packages/system/cilium/Makefile b/packages/system/cilium/Makefile index 71b47a77..07d4ca97 100644 --- a/packages/system/cilium/Makefile +++ b/packages/system/cilium/Makefile @@ -12,15 +12,6 @@ update: helm repo update cilium helm pull cilium/cilium --untar --untardir charts --version 1.19 $(SED_INPLACE) -e '/Used in iptables/d' -e '/SYS_MODULE/d' charts/cilium/values.yaml - # Drop the whole k8s<1.30 AppArmor annotations block from the cilium-agent - # DaemonSet. Cozystack manages these annotations through cilium.podAnnotations - # in values.yaml on every k8s version, so keeping the upstream block would - # produce duplicate mapping keys on k8s<1.30. - perl -i -0pe 's|\n \{\{- if not \.Values\.securityContext\.privileged \}\}\n \{\{- if semverCompare "<1\.30\.0".*?\n \{\{- end \}\}\n \{\{- end \}\}\n \{\{- end \}\}||s' \ - charts/cilium/templates/cilium-agent/daemonset.yaml - @! grep -q 'container\.apparmor\.security\.beta\.kubernetes\.io' \ - charts/cilium/templates/cilium-agent/daemonset.yaml || \ - { echo 'ERROR: perl patch did not remove the upstream AppArmor block from cilium-agent/daemonset.yaml (upstream template format may have changed)' >&2; exit 1; } version=$$(awk '$$1 == "version:" {print $$2}' charts/cilium/Chart.yaml) && \ $(SED_INPLACE) "s/ARG VERSION=.*/ARG VERSION=v$${version}/" images/cilium/Dockerfile diff --git a/packages/system/cilium/charts/cilium/Chart.yaml b/packages/system/cilium/charts/cilium/Chart.yaml index 5fe2baed..0bb34451 100644 --- a/packages/system/cilium/charts/cilium/Chart.yaml +++ b/packages/system/cilium/charts/cilium/Chart.yaml @@ -76,7 +76,7 @@ annotations: Cilium Gateway Class Config\n description: |\n CiliumGatewayClassConfig defines a configuration for Gateway API GatewayClass.\n" apiVersion: v2 -appVersion: 1.19.3 +appVersion: 1.19.1 description: eBPF-based Networking, Security, and Observability home: https://cilium.io/ icon: https://cdn.jsdelivr.net/gh/cilium/cilium@main/Documentation/images/logo-solo.svg @@ -92,4 +92,4 @@ kubeVersion: '>= 1.21.0-0' name: cilium sources: - https://github.com/cilium/cilium -version: 1.19.3 +version: 1.19.1 diff --git a/packages/system/cilium/charts/cilium/README.md b/packages/system/cilium/charts/cilium/README.md index d7697c56..fe8aea3f 100644 --- a/packages/system/cilium/charts/cilium/README.md +++ b/packages/system/cilium/charts/cilium/README.md @@ -1,6 +1,6 @@ # cilium -![Version: 1.19.3](https://img.shields.io/badge/Version-1.19.3-informational?style=flat-square) ![AppVersion: 1.19.3](https://img.shields.io/badge/AppVersion-1.19.3-informational?style=flat-square) +![Version: 1.19.1](https://img.shields.io/badge/Version-1.19.1-informational?style=flat-square) ![AppVersion: 1.19.1](https://img.shields.io/badge/AppVersion-1.19.1-informational?style=flat-square) Cilium is open source software for providing and transparently securing network connectivity and loadbalancing between application workloads such as @@ -89,7 +89,7 @@ contributors across the globe, there is almost always someone available to help. | authentication.mutual.spire.install.agent.tolerations | list | `[{"effect":"NoSchedule","key":"node.kubernetes.io/not-ready"},{"effect":"NoSchedule","key":"node-role.kubernetes.io/master"},{"effect":"NoSchedule","key":"node-role.kubernetes.io/control-plane"},{"effect":"NoSchedule","key":"node.cloudprovider.kubernetes.io/uninitialized","value":"true"},{"key":"CriticalAddonsOnly","operator":"Exists"}]` | SPIRE agent tolerations configuration By default it follows the same tolerations as the agent itself to allow the Cilium agent on this node to connect to SPIRE. ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ | | authentication.mutual.spire.install.enabled | bool | `true` | Enable SPIRE installation. This will only take effect only if authentication.mutual.spire.enabled is true | | authentication.mutual.spire.install.existingNamespace | bool | `false` | SPIRE namespace already exists. Set to true if Helm should not create, manage, and import the SPIRE namespace. | -| authentication.mutual.spire.install.initImage | object | `{"digest":"sha256:1487d0af5f52b4ba31c7e465126ee2123fe3f2305d638e7827681e7cf6c83d5e","override":null,"pullPolicy":"IfNotPresent","repository":"docker.io/library/busybox","tag":"1.37.0","useDigest":true}` | init container image of SPIRE agent and server | +| authentication.mutual.spire.install.initImage | object | `{"digest":"sha256:b3255e7dfbcd10cb367af0d409747d511aeb66dfac98cf30e97e87e4207dd76f","override":null,"pullPolicy":"IfNotPresent","repository":"docker.io/library/busybox","tag":"1.37.0","useDigest":true}` | init container image of SPIRE agent and server | | authentication.mutual.spire.install.namespace | string | `"cilium-spire"` | SPIRE namespace to install into | | authentication.mutual.spire.install.server.affinity | object | `{}` | SPIRE server affinity configuration | | authentication.mutual.spire.install.server.annotations | object | `{}` | SPIRE server annotations | @@ -175,7 +175,7 @@ contributors across the globe, there is almost always someone available to help. | bpf.tproxy | bool | `false` | Configure the eBPF-based TPROXY (beta) to reduce reliance on iptables rules for implementing Layer 7 policy. Note this is incompatible with netkit (`bpf.datapathMode=netkit`, `bpf.datapathMode=netkit-l2`). | | bpf.vlanBypass | list | `[]` | Configure explicitly allowed VLAN id's for bpf logic bypass. [0] will allow all VLAN id's without any filtering. | | bpfClockProbe | bool | `false` | Enable BPF clock source probing for more efficient tick retrieval. | -| certgen | object | `{"affinity":{},"annotations":{"cronJob":{},"job":{}},"cronJob":{"failedJobsHistoryLimit":1,"successfulJobsHistoryLimit":3},"extraVolumeMounts":[],"extraVolumes":[],"generateCA":true,"image":{"digest":"sha256:f0c656830e856d26b24b0e144df1f8b327d3b46748d76a630514111fc365b697","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/certgen","tag":"v0.4.1","useDigest":true},"nodeSelector":{},"podLabels":{},"priorityClassName":"","resources":{},"tolerations":[],"ttlSecondsAfterFinished":null}` | Configure certificate generation for Hubble integration. If hubble.tls.auto.method=cronJob, these values are used for the Kubernetes CronJob which will be scheduled regularly to (re)generate any certificates not provided manually. | +| certgen | object | `{"affinity":{},"annotations":{"cronJob":{},"job":{}},"cronJob":{"failedJobsHistoryLimit":1,"successfulJobsHistoryLimit":3},"extraVolumeMounts":[],"extraVolumes":[],"generateCA":true,"image":{"digest":"sha256:19921f48ee7e2295ea4dca955878a6cd8d70e6d4219d08f688e866ece9d95d4d","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/certgen","tag":"v0.3.2","useDigest":true},"nodeSelector":{},"podLabels":{},"priorityClassName":"","resources":{},"tolerations":[],"ttlSecondsAfterFinished":null}` | Configure certificate generation for Hubble integration. If hubble.tls.auto.method=cronJob, these values are used for the Kubernetes CronJob which will be scheduled regularly to (re)generate any certificates not provided manually. | | certgen.affinity | object | `{}` | Affinity for certgen | | certgen.annotations | object | `{"cronJob":{},"job":{}}` | Annotations to be added to the hubble-certgen initial Job and CronJob | | certgen.cronJob.failedJobsHistoryLimit | int | `1` | The number of failed finished jobs to keep | @@ -214,7 +214,7 @@ contributors across the globe, there is almost always someone available to help. | clustermesh.apiserver.extraVolumeMounts | list | `[]` | Additional clustermesh-apiserver volumeMounts. | | clustermesh.apiserver.extraVolumes | list | `[]` | Additional clustermesh-apiserver volumes. | | clustermesh.apiserver.healthPort | int | `9880` | TCP port for the clustermesh-apiserver health API. | -| clustermesh.apiserver.image | object | `{"digest":"sha256:a8136a7615d6c6041d3aa6f2674d17beaec238170d669507ccc05328a778e2b7","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/clustermesh-apiserver","tag":"v1.19.3","useDigest":true}` | Clustermesh API server image. | +| clustermesh.apiserver.image | object | `{"digest":"sha256:56d6c3dc13b50126b80ecb571707a0ea97f6db694182b9d61efd386d04e5bb28","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/clustermesh-apiserver","tag":"v1.19.1","useDigest":true}` | Clustermesh API server image. | | clustermesh.apiserver.kvstoremesh.enabled | bool | `true` | Enable KVStoreMesh. KVStoreMesh caches the information retrieved from the remote clusters in the local etcd instance (deprecated - KVStoreMesh will always be enabled once the option is removed). | | clustermesh.apiserver.kvstoremesh.extraArgs | list | `[]` | Additional KVStoreMesh arguments. | | clustermesh.apiserver.kvstoremesh.extraEnv | list | `[]` | Additional KVStoreMesh environment variables. | @@ -340,10 +340,6 @@ contributors across the globe, there is almost always someone available to help. | cni.resources | object | `{"limits":{"cpu":1,"memory":"1Gi"},"requests":{"cpu":"100m","memory":"10Mi"}}` | Specifies the resources for the cni initContainer | | cni.uninstall | bool | `false` | Remove the CNI configuration and binary files on agent shutdown. Enable this if you're removing Cilium from the cluster. Disable this to prevent the CNI configuration file from being removed during agent upgrade, which can cause nodes to go unmanageable. | | commonLabels | object | `{}` | commonLabels allows users to add common labels for all Cilium resources. | -| configDriftDetection | object | `{"driftChecker":true,"enabled":true,"ignoredKeys":[]}` | Configuration for the ConfigMap drift detection feature. When enabled, the agent continuously watches the cilium-config ConfigMap and exposes a cilium_drift_checker_config_delta Prometheus metric reporting the number of keys that differ between the ConfigMap and the agent's active settings. A non-zero value indicates that the agent has not yet applied all current ConfigMap changes and needs to be restarted. | -| configDriftDetection.driftChecker | bool | `true` | Enable the drift checker which compares the DynamicConfig table against the agent's active settings and publishes the cilium_drift_checker_config_delta metric. | -| configDriftDetection.enabled | bool | `true` | Enable watching of the cilium-config ConfigMap and reflecting its contents into the agent's internal DynamicConfig table. | -| configDriftDetection.ignoredKeys | list | `[]` | List of config-map keys to ignore when computing the drift delta. | | connectivityProbeFrequencyRatio | float64 | `0.5` | Ratio of the connectivity probe frequency vs resource usage, a float in [0, 1]. 0 will give more frequent probing, 1 will give less frequent probing. Probing frequency is dynamically adjusted based on the cluster size. | | conntrackGCInterval | string | `"0s"` | Configure how frequently garbage collection should occur for the datapath connection tracking table. | | conntrackGCMaxInterval | string | `""` | Configure the maximum frequency for the garbage collection of the connection tracking table. Only affects the automatic computation for the frequency and has no effect when 'conntrackGCInterval' is set. This can be set to more frequently clean up unused identities created from ToFQDN policies. | @@ -384,6 +380,7 @@ contributors across the globe, there is almost always someone available to help. | enableMasqueradeRouteSource | bool | `false` | Enables masquerading to the source of the route for traffic leaving the node from endpoints. | | enableNoServiceEndpointsRoutable | bool | `true` | Enable routing to a service that has zero endpoints | | enableNonDefaultDenyPolicies | bool | `true` | Enable Non-Default-Deny policies | +| enableTunnelBIGTCP | bool | `false` | Enable BIG TCP in tunneling mode and increase maximum GRO/GSO limits for VXLAN/GENEVE tunnels | | enableXTSocketFallback | bool | `true` | Enables the fallback compatibility solution for when the xt_socket kernel module is missing and it is needed for the datapath L7 redirection to work properly. See documentation for details on when this can be disabled: https://docs.cilium.io/en/stable/operations/system_requirements/#linux-kernel. | | encryption.enabled | bool | `false` | Enable transparent network encryption. | | encryption.ipsec.encryptedOverlay | bool | `false` | Enable IPsec encrypted overlay | @@ -404,29 +401,8 @@ contributors across the globe, there is almost always someone available to help. | encryption.strictMode.ingress.enabled | bool | `false` | Enable strict ingress encryption. When enabled, all unencrypted overlay ingress traffic will be dropped. This option is only applicable when WireGuard and tunneling are enabled. | | encryption.type | string | `"ipsec"` | Encryption method. Can be one of ipsec, wireguard or ztunnel. | | encryption.wireguard.persistentKeepalive | string | `"0s"` | Controls WireGuard PersistentKeepalive option. Set 0s to disable. | -| encryption.ztunnel | object | `{"affinity":{},"annotations":{},"caAddress":"https://localhost:15012","extraEnv":[],"extraVolumeMounts":[],"extraVolumes":[],"healthPort":15021,"image":{"digest":null,"override":null,"pullPolicy":"IfNotPresent","repository":"docker.io/istio/ztunnel","tag":"1.28.0-distroless","useDigest":false},"nodeSelector":{"kubernetes.io/os":"linux"},"podAnnotations":{},"podLabels":{},"priorityClassName":null,"readinessProbe":{"failureThreshold":3,"initialDelaySeconds":0,"periodSeconds":10},"resources":{"requests":{"cpu":"200m","memory":"512Mi"}},"secrets":{"bootstrapRootCert":null},"terminationGracePeriodSeconds":30,"tolerations":[{"effect":"NoSchedule","operator":"Exists"},{"key":"CriticalAddonsOnly","operator":"Exists"},{"effect":"NoExecute","operator":"Exists"}],"updateStrategy":{"rollingUpdate":{"maxSurge":1,"maxUnavailable":0},"type":"RollingUpdate"}}` | ztunnel encryption configuration. ztunnel is Istio's purpose-built, per-node proxy for handling L4 traffic in ambient mesh mode. These settings only apply when encryption.type is set to "ztunnel". | -| encryption.ztunnel.affinity | object | `{}` | Affinity for ztunnel pods. | -| encryption.ztunnel.annotations | object | `{}` | Annotations to be added to all ztunnel resources. | -| encryption.ztunnel.caAddress | string | `"https://localhost:15012"` | CA server address for certificate requests. | -| encryption.ztunnel.extraEnv | list | `[]` | Additional ztunnel container environment variables. | -| encryption.ztunnel.extraVolumeMounts | list | `[]` | Additional ztunnel volumeMounts. | -| encryption.ztunnel.extraVolumes | list | `[]` | Additional ztunnel volumes. | -| encryption.ztunnel.healthPort | int | `15021` | TCP port for the health API. | -| encryption.ztunnel.image | object | `{"digest":null,"override":null,"pullPolicy":"IfNotPresent","repository":"docker.io/istio/ztunnel","tag":"1.28.0-distroless","useDigest":false}` | ztunnel container image. | -| encryption.ztunnel.nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node selector for ztunnel pods. | -| encryption.ztunnel.podAnnotations | object | `{}` | Annotations to be added to ztunnel pods. | -| encryption.ztunnel.podLabels | object | `{}` | Labels to be added to ztunnel pods. | -| encryption.ztunnel.priorityClassName | string | `nil` | The priority class to use for ztunnel pods. | -| encryption.ztunnel.readinessProbe | object | `{"failureThreshold":3,"initialDelaySeconds":0,"periodSeconds":10}` | Readiness probe configuration. | -| encryption.ztunnel.resources | object | `{"requests":{"cpu":"200m","memory":"512Mi"}}` | ztunnel resource limits & requests. | -| encryption.ztunnel.secrets | object | `{"bootstrapRootCert":null}` | ztunnel secrets configuration. | -| encryption.ztunnel.secrets.bootstrapRootCert | string | `nil` | Base64-encoded bootstrap root certificate content. If not provided, the secret must be created manually before deploying. @schema type: [null, string] @schema | -| encryption.ztunnel.terminationGracePeriodSeconds | int | `30` | Configure termination grace period for ztunnel DaemonSet. | -| encryption.ztunnel.tolerations | list | `[{"effect":"NoSchedule","operator":"Exists"},{"key":"CriticalAddonsOnly","operator":"Exists"},{"effect":"NoExecute","operator":"Exists"}]` | Node tolerations for ztunnel scheduling. | -| encryption.ztunnel.updateStrategy | object | `{"rollingUpdate":{"maxSurge":1,"maxUnavailable":0},"type":"RollingUpdate"}` | ztunnel update strategy. | | endpointHealthChecking.enabled | bool | `true` | Enable connectivity health checking between virtual endpoints. | | endpointLockdownOnMapOverflow | bool | `false` | Enable endpoint lockdown on policy map overflow. | -| endpointPolicyUpdateTimeoutDuration | string | `nil` | Max duration to wait for envoy to respond to configuration changes. Default "10s". | | endpointRoutes.enabled | bool | `false` | Enable use of per endpoint routes instead of routing via the cilium_host interface. | | eni.awsEnablePrefixDelegation | bool | `false` | Enable ENI prefix delegation | | eni.awsReleaseExcessIPs | bool | `false` | Release IPs not used from the ENI | @@ -470,7 +446,7 @@ contributors across the globe, there is almost always someone available to help. | envoy.httpRetryCount | int | `3` | Maximum number of retries for each HTTP request | | envoy.httpUpstreamLingerTimeout | string | `nil` | Time in seconds to block Envoy worker thread while an upstream HTTP connection is closing. If set to 0, the connection is closed immediately (with TCP RST). If set to -1, the connection is closed asynchronously in the background. | | envoy.idleTimeoutDurationSeconds | int | `60` | Set Envoy upstream HTTP idle connection timeout seconds. Does not apply to connections with pending requests. Default 60s | -| envoy.image | object | `{"digest":"sha256:ba0ab8adac082d50d525fd2c5ba096c8facea3a471561b7c61c7a5b9c2e0de0d","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/cilium-envoy","tag":"v1.36.6-1776000132-2437d2edeaf4d9b56ef279bd0d71127440c067aa","useDigest":true}` | Envoy container image. | +| envoy.image | object | `{"digest":"sha256:8188114a2768b5f49d6ce58e168b20d765e0fbc64eee0d83241aa2b150ccd788","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/cilium-envoy","tag":"v1.35.9-1770979049-232ed4a26881e4ab4f766f251f258ed424fff663","useDigest":true}` | Envoy container image. | | envoy.initContainers | list | `[]` | Init containers added to the cilium Envoy DaemonSet. | | envoy.initialFetchTimeoutSeconds | int | `30` | Time in seconds after which the initial fetch on an xDS stream is considered timed out | | envoy.livenessProbe.enabled | bool | `true` | Enable liveness probe for cilium-envoy | @@ -615,7 +591,7 @@ contributors across the globe, there is almost always someone available to help. | hubble.relay.extraVolumes | list | `[]` | Additional hubble-relay volumes. | | hubble.relay.gops.enabled | bool | `true` | Enable gops for hubble-relay | | hubble.relay.gops.port | int | `9893` | Configure gops listen port for hubble-relay | -| hubble.relay.image | object | `{"digest":"sha256:5ee21d57b6ef2aa6db67e603a735fdceb162454b352b7335b651456e308f681b","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/hubble-relay","tag":"v1.19.3","useDigest":true}` | Hubble-relay container image. | +| hubble.relay.image | object | `{"digest":"sha256:d8c4e13bc36a56179292bb52bc6255379cb94cb873700d316ea3139b1bdb8165","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/hubble-relay","tag":"v1.19.1","useDigest":true}` | Hubble-relay container image. | | hubble.relay.listenHost | string | `""` | Host to listen to. Specify an empty string to bind to all the interfaces. | | hubble.relay.listenPort | string | `"4245"` | Port to listen to. | | hubble.relay.logOptions | object | `{"format":null,"level":null}` | Logging configuration for hubble-relay. | @@ -733,7 +709,7 @@ contributors across the globe, there is almost always someone available to help. | identityAllocationMode | string | `"crd"` | Method to use for identity allocation (`crd`, `kvstore` or `doublewrite-readkvstore` / `doublewrite-readcrd` for migrating between identity backends). | | identityChangeGracePeriod | string | `"5s"` | Time to wait before using new identity on endpoint identity change. | | identityManagementMode | string | `"agent"` | Control whether CiliumIdentities are created by the agent ("agent"), the operator ("operator") or both ("both"). "Both" should be used only to migrate between "agent" and "operator". Operator-managed identities is a beta feature. | -| image | object | `{"digest":"sha256:2e61680593cddca8b6c055f6d4c849d87a26a1c91c7e3b8b56c7fb76ab7b7b10","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/cilium","tag":"v1.19.3","useDigest":true}` | Agent container image. | +| image | object | `{"digest":"sha256:41f1f74a0000de8656f1de4088ea00c8f2d49d6edea579034c73c5fd5fe01792","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/cilium","tag":"v1.19.1","useDigest":true}` | Agent container image. | | imagePullSecrets | list | `[]` | Configure image pull secrets for pulling container images | | ingressController.default | bool | `false` | Set cilium ingress controller to be the default ingress controller This will let cilium ingress controller route entries without ingress class set | | ingressController.defaultSecretName | string | `nil` | Default secret name for ingresses without .spec.tls[].secretName set. | @@ -821,13 +797,12 @@ contributors across the globe, there is almost always someone available to help. | livenessProbe.failureThreshold | int | `10` | failure threshold of liveness probe | | livenessProbe.periodSeconds | int | `30` | interval between checks of the liveness probe | | livenessProbe.requireK8sConnectivity | bool | `false` | whether to require k8s connectivity as part of the check. | -| loadBalancer | object | `{"acceleration":"disabled","l7":{"algorithm":"round_robin","backend":"disabled","ports":[]},"serviceTopology":false}` | Configure service load balancing | +| loadBalancer | object | `{"acceleration":"disabled","l7":{"algorithm":"round_robin","backend":"disabled","ports":[]}}` | Configure service load balancing | | loadBalancer.acceleration | string | `"disabled"` | acceleration is the option to accelerate service handling via XDP Applicable values can be: disabled (do not use XDP), native (XDP BPF program is run directly out of the networking driver's early receive path), or best-effort (use native mode XDP acceleration on devices that support it). | | loadBalancer.l7 | object | `{"algorithm":"round_robin","backend":"disabled","ports":[]}` | L7 LoadBalancer | | loadBalancer.l7.algorithm | string | `"round_robin"` | Default LB algorithm The default LB algorithm to be used for services, which can be overridden by the service annotation (e.g. service.cilium.io/lb-l7-algorithm) Applicable values: round_robin, least_request, random | | loadBalancer.l7.backend | string | `"disabled"` | Enable L7 service load balancing via envoy proxy. The request to a k8s service, which has specific annotation e.g. service.cilium.io/lb-l7, will be forwarded to the local backend proxy to be load balanced to the service endpoints. Please refer to docs for supported annotations for more configuration. Applicable values: - envoy: Enable L7 load balancing via envoy proxy. This will automatically set enable-envoy-config as well. - disabled: Disable L7 load balancing by way of service annotation. | | loadBalancer.l7.ports | list | `[]` | List of ports from service to be automatically redirected to above backend. Any service exposing one of these ports will be automatically redirected. Fine-grained control can be achieved by using the service annotation. | -| loadBalancer.serviceTopology | bool | `false` | serviceTopology enables K8s Topology Aware Hints -based service endpoints filtering | | localRedirectPolicies.addressMatcherCIDRs | string | `nil` | Limit the allowed addresses in Address Matcher rule of Local Redirect Policies to the given CIDRs. @schema@ type: [null, array] @schema@ | | localRedirectPolicies.enabled | bool | `false` | Enable local redirect policies. | | localRedirectPolicy | bool | `false` | Enable Local Redirect Policy (deprecated, please use 'localRedirectPolicies.enabled' instead) | @@ -885,7 +860,7 @@ contributors across the globe, there is almost always someone available to help. | operator.hostNetwork | bool | `true` | HostNetwork setting | | operator.identityGCInterval | string | `"15m0s"` | Interval for identity garbage collection. | | operator.identityHeartbeatTimeout | string | `"30m0s"` | Timeout for identity heartbeats. | -| operator.image | object | `{"alibabacloudDigest":"sha256:176321a65123373ff8c7823b25183102cbad98375e8d6c80b96d68b6e8491103","awsDigest":"sha256:a53dcbfb77282bf2ddd3abbe60f6d49762e7c1389a36cb35b71d504644a56640","azureDigest":"sha256:699c1571a3df1a98882ee13610d47cffb7b34ee7e8d276096db798a5f6c7e4cb","genericDigest":"sha256:205b09b0ed6accbf9fe688d312a9f0fcfc6a316fc081c23fbffb472af5dd62cd","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/operator","suffix":"","tag":"v1.19.3","useDigest":true}` | cilium-operator image. | +| operator.image | object | `{"alibabacloudDigest":"sha256:837b12f4239e88ea5b4b5708ab982c319a94ee05edaecaafe5fd0e5b1962f554","awsDigest":"sha256:18913d05a6c4d205f0b7126c4723bb9ccbd4dc24403da46ed0f9f4bf2a142804","azureDigest":"sha256:82bce78603056e709d4c4e9f9ebb25c222c36d8a07f8c05381c2372d9078eca8","genericDigest":"sha256:e7278d763e448bf6c184b0682cf98cdca078d58a27e1b2f3c906792670aa211a","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/operator","suffix":"","tag":"v1.19.1","useDigest":true}` | cilium-operator image. | | operator.nodeGCInterval | string | `"5m0s"` | Interval for cilium node garbage collection. | | operator.nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node labels for cilium-operator pod assignment ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector | | operator.podAnnotations | object | `{}` | Annotations to be added to cilium-operator pods | @@ -943,11 +918,11 @@ contributors across the globe, there is almost always someone available to help. | preflight.affinity | object | `{"podAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":[{"labelSelector":{"matchLabels":{"k8s-app":"cilium"}},"topologyKey":"kubernetes.io/hostname"}]}}` | Affinity for cilium-preflight | | preflight.annotations | object | `{}` | Annotations to be added to all top-level preflight objects (resources under templates/cilium-preflight) | | preflight.enabled | bool | `false` | Enable Cilium pre-flight resources (required for upgrade) | -| preflight.envoy.image | object | `{"digest":"sha256:ba0ab8adac082d50d525fd2c5ba096c8facea3a471561b7c61c7a5b9c2e0de0d","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/cilium-envoy","tag":"v1.36.6-1776000132-2437d2edeaf4d9b56ef279bd0d71127440c067aa","useDigest":true}` | Envoy pre-flight image. | +| preflight.envoy.image | object | `{"digest":"sha256:8188114a2768b5f49d6ce58e168b20d765e0fbc64eee0d83241aa2b150ccd788","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/cilium-envoy","tag":"v1.35.9-1770979049-232ed4a26881e4ab4f766f251f258ed424fff663","useDigest":true}` | Envoy pre-flight image. | | preflight.extraEnv | list | `[]` | Additional preflight environment variables. | | preflight.extraVolumeMounts | list | `[]` | Additional preflight volumeMounts. | | preflight.extraVolumes | list | `[]` | Additional preflight volumes. | -| preflight.image | object | `{"digest":"sha256:2e61680593cddca8b6c055f6d4c849d87a26a1c91c7e3b8b56c7fb76ab7b7b10","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/cilium","tag":"v1.19.3","useDigest":true}` | Cilium pre-flight image. | +| preflight.image | object | `{"digest":"sha256:41f1f74a0000de8656f1de4088ea00c8f2d49d6edea579034c73c5fd5fe01792","override":null,"pullPolicy":"IfNotPresent","repository":"quay.io/cilium/cilium","tag":"v1.19.1","useDigest":true}` | Cilium pre-flight image. | | preflight.nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node labels for preflight pod assignment ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector | | preflight.podAnnotations | object | `{}` | Annotations to be added to preflight pods | | preflight.podDisruptionBudget.enabled | bool | `false` | enable PodDisruptionBudget ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/ | @@ -1004,7 +979,6 @@ contributors across the globe, there is almost always someone available to help. | serviceAccounts.corednsMCSAPI | object | `{"annotations":{},"automount":true,"create":true,"name":"cilium-coredns-mcsapi-autoconfig"}` | CorednsMCSAPI is used if clustermesh.mcsapi.corednsAutoConfigure.enabled=true | | serviceAccounts.hubblecertgen | object | `{"annotations":{},"automount":true,"create":true,"name":"hubble-generate-certs"}` | Hubblecertgen is used if hubble.tls.auto.method=cronJob | | serviceAccounts.nodeinit.enabled | bool | `false` | Enabled is temporary until https://github.com/cilium/cilium-cli/issues/1396 is implemented. Cilium CLI doesn't create the SAs for node-init, thus the workaround. Helm is not affected by this issue. Name and automount can be configured, if enabled is set to true. Otherwise, they are ignored. Enabled can be removed once the issue is fixed. Cilium-nodeinit DS must also be fixed. | -| serviceAccounts.ztunnel | object | `{"annotations":{},"automount":false,"create":true,"name":"ztunnel-cilium"}` | Ztunnel is used if encryption.type=ztunnel | | serviceNoBackendResponse | string | `"reject"` | Configure what the response should be to traffic for a service without backends. Possible values: - reject (default) - drop | | sleepAfterInit | bool | `false` | Do not run Cilium agent when running with clean mode. Useful to completely uninstall Cilium as it will stop Cilium from starting and create artifacts in the node. | | socketLB | object | `{"enabled":false}` | Configure socket LB | diff --git a/packages/system/cilium/charts/cilium/files/cilium-envoy/configmap/bootstrap-config.yaml b/packages/system/cilium/charts/cilium/files/cilium-envoy/configmap/bootstrap-config.yaml index 90ebf4ac..ea1d3bda 100644 --- a/packages/system/cilium/charts/cilium/files/cilium-envoy/configmap/bootstrap-config.yaml +++ b/packages/system/cilium/charts/cilium/files/cilium-envoy/configmap/bootstrap-config.yaml @@ -167,8 +167,6 @@ staticResources: circuitBreakers: thresholds: - maxRetries: {{ .Values.envoy.maxConcurrentRetries }} - maxConnections: {{ .Values.envoy.clusterMaxConnections }} - maxRequests: {{ .Values.envoy.clusterMaxRequests }} lbPolicy: "CLUSTER_PROVIDED" typedExtensionProtocolOptions: envoy.extensions.upstreams.http.v3.HttpProtocolOptions: @@ -185,8 +183,6 @@ staticResources: circuitBreakers: thresholds: - maxRetries: {{ .Values.envoy.maxConcurrentRetries }} - maxConnections: {{ .Values.envoy.clusterMaxConnections }} - maxRequests: {{ .Values.envoy.clusterMaxRequests }} lbPolicy: "CLUSTER_PROVIDED" typedExtensionProtocolOptions: envoy.extensions.upstreams.http.v3.HttpProtocolOptions: @@ -208,8 +204,6 @@ staticResources: circuitBreakers: thresholds: - maxRetries: {{ .Values.envoy.maxConcurrentRetries }} - maxConnections: {{ .Values.envoy.clusterMaxConnections }} - maxRequests: {{ .Values.envoy.clusterMaxRequests }} lbPolicy: "CLUSTER_PROVIDED" typedExtensionProtocolOptions: envoy.extensions.upstreams.http.v3.HttpProtocolOptions: @@ -226,8 +220,6 @@ staticResources: circuitBreakers: thresholds: - maxRetries: {{ .Values.envoy.maxConcurrentRetries }} - maxConnections: {{ .Values.envoy.clusterMaxConnections }} - maxRequests: {{ .Values.envoy.clusterMaxRequests }} lbPolicy: "CLUSTER_PROVIDED" typedExtensionProtocolOptions: envoy.extensions.upstreams.http.v3.HttpProtocolOptions: @@ -312,4 +304,3 @@ admin: address: pipe: path: "/var/run/cilium/envoy/sockets/admin.sock" - mode: 0660 diff --git a/packages/system/cilium/charts/cilium/templates/cilium-agent/daemonset.yaml b/packages/system/cilium/charts/cilium/templates/cilium-agent/daemonset.yaml index fff1b384..f9a4cec7 100644 --- a/packages/system/cilium/charts/cilium/templates/cilium-agent/daemonset.yaml +++ b/packages/system/cilium/charts/cilium/templates/cilium-agent/daemonset.yaml @@ -57,6 +57,19 @@ spec: # ensure pods roll when configmap updates cilium.io/cilium-configmap-checksum: {{ include (print $.Template.BasePath "/cilium-configmap.yaml") . | sha256sum | quote }} {{- end }} + {{- if not .Values.securityContext.privileged }} + {{- if semverCompare "<1.30.0" (printf "%d.%d.0" (semver .Capabilities.KubeVersion.Version).Major (semver .Capabilities.KubeVersion.Version).Minor) }} + # Set app AppArmor's profile to "unconfined". The value of this annotation + # can be modified as long users know which profiles they have available + # in AppArmor. + container.apparmor.security.beta.kubernetes.io/cilium-agent: "unconfined" + container.apparmor.security.beta.kubernetes.io/clean-cilium-state: "unconfined" + {{- if .Values.cgroup.autoMount.enabled }} + container.apparmor.security.beta.kubernetes.io/mount-cgroup: "unconfined" + container.apparmor.security.beta.kubernetes.io/apply-sysctl-overwrites: "unconfined" + {{- end }} + {{- end }} + {{- end }} {{- with .Values.podAnnotations }} {{- toYaml . | nindent 8 }} {{- end }} @@ -546,7 +559,7 @@ spec: {{- toYaml .Values.initResources | trim | nindent 10 }} {{- end }} command: - - bash + - sh - -ec # The statically linked Go program binary is invoked to avoid any # dependency on utilities like sh and mount that can be missing on certain @@ -592,7 +605,7 @@ spec: - name: BIN_PATH value: {{ .Values.cni.binPath }} command: - - bash + - sh - -ec # The statically linked Go program binary is invoked to avoid any # dependency on utilities like sh that can be missing on certain @@ -660,7 +673,7 @@ spec: {{- toYaml . | trim | nindent 10 }} {{- end }} command: - - bash + - sh - -c - | until test -s {{ (print "/tmp/cilium-bootstrap.d/" (.Values.nodeinit.bootstrapFile | base)) | quote }}; do diff --git a/packages/system/cilium/charts/cilium/templates/cilium-configmap.yaml b/packages/system/cilium/charts/cilium/templates/cilium-configmap.yaml index 5d76944d..a3a38c09 100644 --- a/packages/system/cilium/charts/cilium/templates/cilium-configmap.yaml +++ b/packages/system/cilium/charts/cilium/templates/cilium-configmap.yaml @@ -464,9 +464,6 @@ data: {{- if has (kindOf .Values.bpf.policyMapPressureMetricsThreshold) (list "int64" "float64") }} bpf-policy-map-pressure-metrics-threshold: {{ .Values.bpf.policyMapPressureMetricsThreshold | quote }} {{- end }} -{{- if .Values.endpointPolicyUpdateTimeoutDuration }} - endpoint-policy-update-timeout: {{ .Values.endpointPolicyUpdateTimeoutDuration | quote }} -{{- end }} {{- if hasKey .Values.bpf "policyStatsMapMax" }} # bpf-policy-stats-map-max specifies the maximum number of entries in global # policy stats map @@ -709,6 +706,7 @@ data: enable-ipv4-big-tcp: {{ .Values.enableIPv4BIGTCP | quote }} enable-ipv6-big-tcp: {{ .Values.enableIPv6BIGTCP | quote }} enable-ipv6-masquerade: {{ .Values.enableIPv6Masquerade | quote }} + enable-tunnel-big-tcp: {{ .Values.enableTunnelBIGTCP | quote }} {{- if hasKey .Values.bpf "enableTCX" }} enable-tcx: {{ .Values.bpf.enableTCX | quote }} @@ -908,9 +906,9 @@ data: {{- end }} {{- if hasKey .Values.loadBalancer "serviceTopology" }} enable-service-topology: {{ .Values.loadBalancer.serviceTopology | quote }} -{{- end }} -{{- end }} +# {{- end }} +{{- end }} {{- if hasKey .Values.maglev "tableSize" }} bpf-lb-maglev-table-size: {{ .Values.maglev.tableSize | quote}} {{- end }} @@ -1382,7 +1380,11 @@ data: {{- if .Values.operator.unmanagedPodWatcher.restart }} {{- $interval := .Values.operator.unmanagedPodWatcher.intervalSeconds }} + {{- if kindIs "float64" $interval }} unmanaged-pod-watcher-interval: {{ printf "%ds" (int $interval) | quote }} + {{- else }} + unmanaged-pod-watcher-interval: {{ $interval | quote }} + {{- end }} {{- else }} unmanaged-pod-watcher-interval: "0" {{- end }} @@ -1515,14 +1517,6 @@ data: connectivity-probe-frequency-ratio: {{ .Values.connectivityProbeFrequencyRatio | quote }} {{- end }} -{{- if hasKey .Values "configDriftDetection" }} - enable-dynamic-config: {{ .Values.configDriftDetection.enabled | quote }} - enable-drift-checker: {{ .Values.configDriftDetection.driftChecker | quote }} - {{- if .Values.configDriftDetection.ignoredKeys }} - ignore-flags-drift-checker: {{ join "," .Values.configDriftDetection.ignoredKeys | quote }} - {{- end }} -{{- end }} - # Extra config allows adding arbitrary properties to the cilium config. # By putting it at the end of the ConfigMap, it's also possible to override existing properties. {{- if .Values.extraConfig }} diff --git a/packages/system/cilium/charts/cilium/templates/cilium-operator/clusterrole.yaml b/packages/system/cilium/charts/cilium/templates/cilium-operator/clusterrole.yaml index c147e2b5..96f72c7f 100644 --- a/packages/system/cilium/charts/cilium/templates/cilium-operator/clusterrole.yaml +++ b/packages/system/cilium/charts/cilium/templates/cilium-operator/clusterrole.yaml @@ -407,15 +407,6 @@ rules: verbs: - update - patch -- apiGroups: - - multicluster.x-k8s.io - resources: - # The controller needs to be able to set serviceimport finalizers to be able to create a derived Service - # resource that is owned by the ServiceImport and sets blockOwnerDeletion=true in its ownerRef. - # This is required when the admission plugin OwnerReferencesPermissionEnforcement is activated. - - serviceimports/finalizers - verbs: - - update - apiGroups: - multicluster.x-k8s.io resources: diff --git a/packages/system/cilium/charts/cilium/templates/ztunnel/daemonset.yaml b/packages/system/cilium/charts/cilium/templates/ztunnel/daemonset.yaml deleted file mode 100644 index f9363068..00000000 --- a/packages/system/cilium/charts/cilium/templates/ztunnel/daemonset.yaml +++ /dev/null @@ -1,165 +0,0 @@ -{{- if and .Values.encryption.enabled (eq .Values.encryption.type "ztunnel") }} ---- -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: ztunnel-cilium - namespace: {{ include "cilium.namespace" . }} - {{- with .Values.encryption.ztunnel.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} - labels: - app: ztunnel-cilium - app.kubernetes.io/part-of: cilium - app.kubernetes.io/name: ztunnel-cilium - {{- with .Values.commonLabels }} - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - revisionHistoryLimit: 10 - selector: - matchLabels: - app: ztunnel-cilium - {{- with .Values.encryption.ztunnel.updateStrategy }} - updateStrategy: - {{- toYaml . | trim | nindent 4 }} - {{- end }} - template: - metadata: - annotations: - sidecar.istio.io/inject: "false" - {{- with .Values.encryption.ztunnel.podAnnotations }} - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - app: ztunnel-cilium - app.kubernetes.io/part-of: cilium - app.kubernetes.io/name: ztunnel-cilium - {{- with .Values.commonLabels }} - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.encryption.ztunnel.podLabels }} - {{- toYaml . | nindent 8 }} - {{- end }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - hostNetwork: true - dnsPolicy: ClusterFirst - containers: - - name: istio-proxy - image: {{ include "cilium.image" .Values.encryption.ztunnel.image | quote }} - imagePullPolicy: {{ .Values.encryption.ztunnel.image.pullPolicy }} - args: - - proxy - - ztunnel - env: - - name: XDS_ADDRESS - value: "https://localhost:15012" - - name: XDS_ROOT_CA - value: "/etc/ztunnel/bootstrap-root.crt" - - name: CA_ROOT_CA - value: "/etc/ztunnel/bootstrap-root.crt" - - name: CA_ADDRESS - value: {{ .Values.encryption.ztunnel.caAddress | quote }} - - name: ISTIO_META_DNS_CAPTURE - value: "false" - - name: INPOD_UDS - value: "/var/run/cilium/ztunnel.sock" - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.hostIP - - name: INSTANCE_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - - name: ISTIO_META_ENABLE_HBONE - value: "true" - {{- with .Values.encryption.ztunnel.extraEnv }} - {{- toYaml . | trim | nindent 12 }} - {{- end }} - readinessProbe: - httpGet: - path: /healthz/ready - port: {{ .Values.encryption.ztunnel.healthPort }} - host: "127.0.0.1" - scheme: HTTP - initialDelaySeconds: {{ .Values.encryption.ztunnel.readinessProbe.initialDelaySeconds }} - periodSeconds: {{ .Values.encryption.ztunnel.readinessProbe.periodSeconds }} - failureThreshold: {{ .Values.encryption.ztunnel.readinessProbe.failureThreshold }} - successThreshold: 1 - timeoutSeconds: 1 - {{- with .Values.encryption.ztunnel.resources }} - resources: - {{- toYaml . | trim | nindent 12 }} - {{- end }} - securityContext: - allowPrivilegeEscalation: true - capabilities: - add: - - NET_ADMIN - - SYS_ADMIN - - NET_RAW - drop: - - ALL - privileged: false - readOnlyRootFilesystem: true - runAsGroup: 1337 - runAsNonRoot: false - runAsUser: 0 - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - volumeMounts: - - mountPath: /var/run/cilium - name: cilium-dir - readOnly: false - - mountPath: /etc/ztunnel - name: cilium-ztunnel-secrets - readOnly: true - {{- with .Values.encryption.ztunnel.extraVolumeMounts }} - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.encryption.ztunnel.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.encryption.ztunnel.nodeSelector }} - nodeSelector: - {{- toYaml . | trim | nindent 8 }} - {{- end }} - {{- with .Values.encryption.ztunnel.tolerations }} - tolerations: - {{- toYaml . | trim | nindent 8 }} - {{- end }} - priorityClassName: {{ include "cilium.priorityClass" (list $ .Values.encryption.ztunnel.priorityClassName "system-node-critical") }} - restartPolicy: Always - terminationGracePeriodSeconds: {{ .Values.encryption.ztunnel.terminationGracePeriodSeconds }} - {{- if .Values.serviceAccounts.ztunnel.create }} - serviceAccountName: {{ .Values.serviceAccounts.ztunnel.name | quote }} - automountServiceAccountToken: {{ .Values.serviceAccounts.ztunnel.automount }} - {{- else }} - automountServiceAccountToken: false - {{- end }} - volumes: - - name: cilium-dir - hostPath: - path: /var/run/cilium - type: DirectoryOrCreate - - name: cilium-ztunnel-secrets - secret: - secretName: cilium-ztunnel-secrets - defaultMode: 420 - items: - - key: bootstrap-root.crt - path: bootstrap-root.crt - mode: 420 - {{- with .Values.encryption.ztunnel.extraVolumes }} - {{- toYaml . | nindent 8 }} - {{- end }} -{{- end }} diff --git a/packages/system/cilium/charts/cilium/templates/ztunnel/secret.yaml b/packages/system/cilium/charts/cilium/templates/ztunnel/secret.yaml deleted file mode 100644 index 520857dc..00000000 --- a/packages/system/cilium/charts/cilium/templates/ztunnel/secret.yaml +++ /dev/null @@ -1,23 +0,0 @@ -{{- if and .Values.encryption.enabled (eq .Values.encryption.type "ztunnel") }} -{{- if .Values.encryption.ztunnel.secrets.bootstrapRootCert }} ---- -apiVersion: v1 -kind: Secret -metadata: - name: cilium-ztunnel-secrets - namespace: {{ include "cilium.namespace" . }} - {{- with .Values.encryption.ztunnel.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} - labels: - app.kubernetes.io/part-of: cilium - app.kubernetes.io/name: ztunnel-cilium - {{- with .Values.commonLabels }} - {{- toYaml . | nindent 4 }} - {{- end }} -type: Opaque -data: - bootstrap-root.crt: {{ .Values.encryption.ztunnel.secrets.bootstrapRootCert | b64enc }} -{{- end }} -{{- end }} diff --git a/packages/system/cilium/charts/cilium/templates/ztunnel/serviceaccount.yaml b/packages/system/cilium/charts/cilium/templates/ztunnel/serviceaccount.yaml deleted file mode 100644 index 4ef9e2bc..00000000 --- a/packages/system/cilium/charts/cilium/templates/ztunnel/serviceaccount.yaml +++ /dev/null @@ -1,21 +0,0 @@ -{{- if and .Values.encryption.enabled (eq .Values.encryption.type "ztunnel") .Values.serviceAccounts.ztunnel.create }} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ .Values.serviceAccounts.ztunnel.name | quote }} - namespace: {{ include "cilium.namespace" . }} - {{- with .Values.commonLabels }} - labels: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- if or .Values.serviceAccounts.ztunnel.annotations .Values.encryption.ztunnel.annotations }} - annotations: - {{- with .Values.encryption.ztunnel.annotations }} - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with .Values.serviceAccounts.ztunnel.annotations }} - {{- toYaml . | nindent 4 }} - {{- end }} - {{- end }} -{{- end }} diff --git a/packages/system/cilium/charts/cilium/values.schema.json b/packages/system/cilium/charts/cilium/values.schema.json index 65c84af5..d18566d7 100644 --- a/packages/system/cilium/charts/cilium/values.schema.json +++ b/packages/system/cilium/charts/cilium/values.schema.json @@ -1714,21 +1714,6 @@ "object" ] }, - "configDriftDetection": { - "properties": { - "driftChecker": { - "type": "boolean" - }, - "enabled": { - "type": "boolean" - }, - "ignoredKeys": { - "items": {}, - "type": "array" - } - }, - "type": "object" - }, "connectivityProbeFrequencyRatio": { "type": [ "null", @@ -1913,6 +1898,9 @@ "enableNonDefaultDenyPolicies": { "type": "boolean" }, + "enableTunnelBIGTCP": { + "type": "boolean" + }, "enableXTSocketFallback": { "type": "boolean" }, @@ -1996,184 +1984,6 @@ } }, "type": "object" - }, - "ztunnel": { - "properties": { - "affinity": { - "type": "object" - }, - "annotations": { - "type": "object" - }, - "caAddress": { - "type": "string" - }, - "extraEnv": { - "items": {}, - "type": "array" - }, - "extraVolumeMounts": { - "items": {}, - "type": "array" - }, - "extraVolumes": { - "items": {}, - "type": "array" - }, - "healthPort": { - "type": "integer" - }, - "image": { - "properties": { - "digest": { - "type": [ - "null", - "string" - ] - }, - "override": { - "type": [ - "null", - "string" - ] - }, - "pullPolicy": { - "type": "string" - }, - "repository": { - "type": "string" - }, - "tag": { - "type": "string" - }, - "useDigest": { - "type": "boolean" - } - }, - "type": "object" - }, - "nodeSelector": { - "properties": { - "kubernetes.io/os": { - "type": "string" - } - }, - "type": "object" - }, - "podAnnotations": { - "type": "object" - }, - "podLabels": { - "type": "object" - }, - "priorityClassName": { - "type": [ - "null", - "string" - ] - }, - "readinessProbe": { - "properties": { - "failureThreshold": { - "type": "integer" - }, - "initialDelaySeconds": { - "type": "integer" - }, - "periodSeconds": { - "type": "integer" - } - }, - "type": "object" - }, - "resources": { - "properties": { - "requests": { - "properties": { - "cpu": { - "type": "string" - }, - "memory": { - "type": "string" - } - }, - "type": "object" - } - }, - "type": "object" - }, - "secrets": { - "properties": { - "bootstrapRootCert": { - "type": [ - "null", - "string" - ] - } - }, - "type": "object" - }, - "terminationGracePeriodSeconds": { - "type": "integer" - }, - "tolerations": { - "items": { - "anyOf": [ - { - "properties": { - "effect": { - "type": "string" - }, - "operator": { - "type": "string" - } - } - }, - { - "properties": { - "key": { - "type": "string" - }, - "operator": { - "type": "string" - } - } - }, - { - "properties": { - "effect": { - "type": "string" - }, - "operator": { - "type": "string" - } - } - } - ] - }, - "type": "array" - }, - "updateStrategy": { - "properties": { - "rollingUpdate": { - "properties": { - "maxSurge": { - "type": "integer" - }, - "maxUnavailable": { - "type": "integer" - } - }, - "type": "object" - }, - "type": { - "type": "string" - } - }, - "type": "object" - } - }, - "type": "object" } }, "type": "object" @@ -2189,9 +1999,6 @@ "endpointLockdownOnMapOverflow": { "type": "boolean" }, - "endpointPolicyUpdateTimeoutDuration": { - "type": "null" - }, "endpointRoutes": { "properties": { "enabled": { @@ -4677,9 +4484,6 @@ } }, "type": "object" - }, - "serviceTopology": { - "type": "boolean" } }, "type": "object" @@ -6225,23 +6029,6 @@ } }, "type": "object" - }, - "ztunnel": { - "properties": { - "annotations": { - "type": "object" - }, - "automount": { - "type": "boolean" - }, - "create": { - "type": "boolean" - }, - "name": { - "type": "string" - } - }, - "type": "object" } }, "type": "object" diff --git a/packages/system/cilium/charts/cilium/values.yaml b/packages/system/cilium/charts/cilium/values.yaml index 66f76982..b9b30830 100644 --- a/packages/system/cilium/charts/cilium/values.yaml +++ b/packages/system/cilium/charts/cilium/values.yaml @@ -210,12 +210,6 @@ serviceAccounts: name: cilium-coredns-mcsapi-autoconfig automount: true annotations: {} - # -- Ztunnel is used if encryption.type=ztunnel - ztunnel: - create: true - name: ztunnel-cilium - automount: false - annotations: {} # -- Configure termination grace period for cilium-agent DaemonSet. terminationGracePeriodSeconds: 1 # -- Install the cilium agent resources. @@ -224,22 +218,6 @@ agent: true name: cilium # -- Roll out cilium agent pods automatically when configmap is updated. rollOutCiliumPods: false -# -- Configuration for the ConfigMap drift detection feature. -# When enabled, the agent continuously watches the cilium-config ConfigMap -# and exposes a cilium_drift_checker_config_delta Prometheus metric reporting -# the number of keys that differ between the ConfigMap and the agent's active -# settings. A non-zero value indicates that the agent has not yet applied all -# current ConfigMap changes and needs to be restarted. -configDriftDetection: - # -- Enable watching of the cilium-config ConfigMap and reflecting its - # contents into the agent's internal DynamicConfig table. - enabled: true - # -- Enable the drift checker which compares the DynamicConfig table against - # the agent's active settings and publishes the - # cilium_drift_checker_config_delta metric. - driftChecker: true - # -- List of config-map keys to ignore when computing the drift delta. - ignoredKeys: [] # -- Agent container image. image: # @schema @@ -247,10 +225,10 @@ image: # @schema override: ~ repository: "quay.io/cilium/cilium" - tag: "v1.19.3" + tag: "v1.19.1" pullPolicy: "IfNotPresent" # cilium-digest - digest: sha256:2e61680593cddca8b6c055f6d4c849d87a26a1c91c7e3b8b56c7fb76ab7b7b10 + digest: sha256:41f1f74a0000de8656f1de4088ea00c8f2d49d6edea579034c73c5fd5fe01792 useDigest: true # -- Scheduling configurations for cilium pods scheduling: @@ -1155,89 +1133,9 @@ encryption: wireguard: # -- Controls WireGuard PersistentKeepalive option. Set 0s to disable. persistentKeepalive: 0s - # -- ztunnel encryption configuration. - # ztunnel is Istio's purpose-built, per-node proxy for handling L4 traffic in ambient mesh mode. - # These settings only apply when encryption.type is set to "ztunnel". - ztunnel: - # -- ztunnel container image. - image: - # @schema - # type: [null, string] - # @schema - override: ~ - repository: "docker.io/istio/ztunnel" - tag: "1.28.0-distroless" - pullPolicy: "IfNotPresent" - # @schema - # type: [null, string] - # @schema - digest: ~ - useDigest: false - # -- CA server address for certificate requests. - caAddress: "https://localhost:15012" - # -- TCP port for the health API. - healthPort: 15021 - # -- ztunnel resource limits & requests. - resources: - requests: - cpu: 200m - memory: 512Mi - # -- ztunnel update strategy. - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - # -- Configure termination grace period for ztunnel DaemonSet. - terminationGracePeriodSeconds: 30 - # -- Readiness probe configuration. - readinessProbe: - initialDelaySeconds: 0 - periodSeconds: 10 - failureThreshold: 3 - # -- Node selector for ztunnel pods. - nodeSelector: - kubernetes.io/os: linux - # -- Node tolerations for ztunnel scheduling. - tolerations: - - effect: NoSchedule - operator: Exists - - key: CriticalAddonsOnly - operator: Exists - - effect: NoExecute - operator: Exists - # -- Affinity for ztunnel pods. - affinity: {} - # @schema - # type: [null, string] - # @schema - # -- The priority class to use for ztunnel pods. - priorityClassName: ~ - # -- Annotations to be added to all ztunnel resources. - annotations: {} - # -- Annotations to be added to ztunnel pods. - podAnnotations: {} - # -- Labels to be added to ztunnel pods. - podLabels: {} - # -- Additional ztunnel container environment variables. - extraEnv: [] - # -- Additional ztunnel volumes. - extraVolumes: [] - # -- Additional ztunnel volumeMounts. - extraVolumeMounts: [] - # -- ztunnel secrets configuration. - secrets: - # -- Base64-encoded bootstrap root certificate content. - # If not provided, the secret must be created manually before deploying. - # @schema - # type: [null, string] - # @schema - bootstrapRootCert: ~ endpointHealthChecking: # -- Enable connectivity health checking between virtual endpoints. enabled: true -# -- Max duration to wait for envoy to respond to configuration changes. Default "10s". -endpointPolicyUpdateTimeoutDuration: null endpointRoutes: # @schema # type: [boolean, string] @@ -1352,8 +1250,8 @@ certgen: # @schema override: ~ repository: "quay.io/cilium/certgen" - tag: "v0.4.1" - digest: "sha256:f0c656830e856d26b24b0e144df1f8b327d3b46748d76a630514111fc365b697" + tag: "v0.3.2" + digest: "sha256:19921f48ee7e2295ea4dca955878a6cd8d70e6d4219d08f688e866ece9d95d4d" useDigest: true pullPolicy: "IfNotPresent" # @schema @@ -1701,9 +1599,9 @@ hubble: # @schema override: ~ repository: "quay.io/cilium/hubble-relay" - tag: "v1.19.3" + tag: "v1.19.1" # hubble-relay-digest - digest: sha256:5ee21d57b6ef2aa6db67e603a735fdceb162454b352b7335b651456e308f681b + digest: sha256:d8c4e13bc36a56179292bb52bc6255379cb94cb873700d316ea3139b1bdb8165 useDigest: true pullPolicy: "IfNotPresent" # -- Specifies the resources for the hubble-relay pods @@ -2415,6 +2313,8 @@ enableMasqueradeRouteSource: false enableIPv4BIGTCP: false # -- Enables IPv6 BIG TCP support which increases maximum IPv6 GSO/GRO limits for nodes and pods enableIPv6BIGTCP: false +# -- Enable BIG TCP in tunneling mode and increase maximum GRO/GSO limits for VXLAN/GENEVE tunnels +enableTunnelBIGTCP: false nat: # -- Number of the top-k SNAT map connections to track in Cilium statedb. mapStatsEntries: 32 @@ -2492,7 +2392,8 @@ loadBalancer: # -- serviceTopology enables K8s Topology Aware Hints -based service # endpoints filtering - serviceTopology: false + # serviceTopology: false + # -- L7 LoadBalancer l7: # -- Enable L7 service load balancing via envoy proxy. @@ -2725,9 +2626,9 @@ envoy: # @schema override: ~ repository: "quay.io/cilium/cilium-envoy" - tag: "v1.36.6-1776000132-2437d2edeaf4d9b56ef279bd0d71127440c067aa" + tag: "v1.35.9-1770979049-232ed4a26881e4ab4f766f251f258ed424fff663" pullPolicy: "IfNotPresent" - digest: "sha256:ba0ab8adac082d50d525fd2c5ba096c8facea3a471561b7c61c7a5b9c2e0de0d" + digest: "sha256:8188114a2768b5f49d6ce58e168b20d765e0fbc64eee0d83241aa2b150ccd788" useDigest: true # -- Init containers added to the cilium Envoy DaemonSet. initContainers: [] @@ -3110,15 +3011,15 @@ operator: # @schema override: ~ repository: "quay.io/cilium/operator" - tag: "v1.19.3" + tag: "v1.19.1" # operator-generic-digest - genericDigest: sha256:205b09b0ed6accbf9fe688d312a9f0fcfc6a316fc081c23fbffb472af5dd62cd + genericDigest: sha256:e7278d763e448bf6c184b0682cf98cdca078d58a27e1b2f3c906792670aa211a # operator-azure-digest - azureDigest: sha256:699c1571a3df1a98882ee13610d47cffb7b34ee7e8d276096db798a5f6c7e4cb + azureDigest: sha256:82bce78603056e709d4c4e9f9ebb25c222c36d8a07f8c05381c2372d9078eca8 # operator-aws-digest - awsDigest: sha256:a53dcbfb77282bf2ddd3abbe60f6d49762e7c1389a36cb35b71d504644a56640 + awsDigest: sha256:18913d05a6c4d205f0b7126c4723bb9ccbd4dc24403da46ed0f9f4bf2a142804 # operator-alibabacloud-digest - alibabacloudDigest: sha256:176321a65123373ff8c7823b25183102cbad98375e8d6c80b96d68b6e8491103 + alibabacloudDigest: sha256:837b12f4239e88ea5b4b5708ab982c319a94ee05edaecaafe5fd0e5b1962f554 useDigest: true pullPolicy: "IfNotPresent" suffix: "" @@ -3443,9 +3344,9 @@ preflight: # @schema override: ~ repository: "quay.io/cilium/cilium" - tag: "v1.19.3" + tag: "v1.19.1" # cilium-digest - digest: sha256:2e61680593cddca8b6c055f6d4c849d87a26a1c91c7e3b8b56c7fb76ab7b7b10 + digest: sha256:41f1f74a0000de8656f1de4088ea00c8f2d49d6edea579034c73c5fd5fe01792 useDigest: true pullPolicy: "IfNotPresent" envoy: @@ -3456,9 +3357,9 @@ preflight: # @schema override: ~ repository: "quay.io/cilium/cilium-envoy" - tag: "v1.36.6-1776000132-2437d2edeaf4d9b56ef279bd0d71127440c067aa" + tag: "v1.35.9-1770979049-232ed4a26881e4ab4f766f251f258ed424fff663" pullPolicy: "IfNotPresent" - digest: "sha256:ba0ab8adac082d50d525fd2c5ba096c8facea3a471561b7c61c7a5b9c2e0de0d" + digest: "sha256:8188114a2768b5f49d6ce58e168b20d765e0fbc64eee0d83241aa2b150ccd788" useDigest: true # -- The priority class to use for the preflight pod. priorityClassName: "" @@ -3702,9 +3603,9 @@ clustermesh: # @schema override: ~ repository: "quay.io/cilium/clustermesh-apiserver" - tag: "v1.19.3" + tag: "v1.19.1" # clustermesh-apiserver-digest - digest: sha256:a8136a7615d6c6041d3aa6f2674d17beaec238170d669507ccc05328a778e2b7 + digest: sha256:56d6c3dc13b50126b80ecb571707a0ea97f6db694182b9d61efd386d04e5bb28 useDigest: true pullPolicy: "IfNotPresent" # -- TCP port for the clustermesh-apiserver health API. @@ -4239,7 +4140,7 @@ authentication: override: ~ repository: "docker.io/library/busybox" tag: "1.37.0" - digest: "sha256:1487d0af5f52b4ba31c7e465126ee2123fe3f2305d638e7827681e7cf6c83d5e" + digest: "sha256:b3255e7dfbcd10cb367af0d409747d511aeb66dfac98cf30e97e87e4207dd76f" useDigest: true pullPolicy: "IfNotPresent" # SPIRE agent configuration diff --git a/packages/system/cilium/charts/cilium/values.yaml.tmpl b/packages/system/cilium/charts/cilium/values.yaml.tmpl index 8039b40c..c21f1c26 100644 --- a/packages/system/cilium/charts/cilium/values.yaml.tmpl +++ b/packages/system/cilium/charts/cilium/values.yaml.tmpl @@ -213,12 +213,6 @@ serviceAccounts: name: cilium-coredns-mcsapi-autoconfig automount: true annotations: {} - # -- Ztunnel is used if encryption.type=ztunnel - ztunnel: - create: true - name: ztunnel-cilium - automount: false - annotations: {} # -- Configure termination grace period for cilium-agent DaemonSet. terminationGracePeriodSeconds: 1 # -- Install the cilium agent resources. @@ -227,22 +221,6 @@ agent: true name: cilium # -- Roll out cilium agent pods automatically when configmap is updated. rollOutCiliumPods: false -# -- Configuration for the ConfigMap drift detection feature. -# When enabled, the agent continuously watches the cilium-config ConfigMap -# and exposes a cilium_drift_checker_config_delta Prometheus metric reporting -# the number of keys that differ between the ConfigMap and the agent's active -# settings. A non-zero value indicates that the agent has not yet applied all -# current ConfigMap changes and needs to be restarted. -configDriftDetection: - # -- Enable watching of the cilium-config ConfigMap and reflecting its - # contents into the agent's internal DynamicConfig table. - enabled: true - # -- Enable the drift checker which compares the DynamicConfig table against - # the agent's active settings and publishes the - # cilium_drift_checker_config_delta metric. - driftChecker: true - # -- List of config-map keys to ignore when computing the drift delta. - ignoredKeys: [] # -- Agent container image. image: # @schema @@ -1170,89 +1148,9 @@ encryption: wireguard: # -- Controls WireGuard PersistentKeepalive option. Set 0s to disable. persistentKeepalive: 0s - # -- ztunnel encryption configuration. - # ztunnel is Istio's purpose-built, per-node proxy for handling L4 traffic in ambient mesh mode. - # These settings only apply when encryption.type is set to "ztunnel". - ztunnel: - # -- ztunnel container image. - image: - # @schema - # type: [null, string] - # @schema - override: ~ - repository: "docker.io/istio/ztunnel" - tag: "1.28.0-distroless" - pullPolicy: "IfNotPresent" - # @schema - # type: [null, string] - # @schema - digest: ~ - useDigest: false - # -- CA server address for certificate requests. - caAddress: "https://localhost:15012" - # -- TCP port for the health API. - healthPort: 15021 - # -- ztunnel resource limits & requests. - resources: - requests: - cpu: 200m - memory: 512Mi - # -- ztunnel update strategy. - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - # -- Configure termination grace period for ztunnel DaemonSet. - terminationGracePeriodSeconds: 30 - # -- Readiness probe configuration. - readinessProbe: - initialDelaySeconds: 0 - periodSeconds: 10 - failureThreshold: 3 - # -- Node selector for ztunnel pods. - nodeSelector: - kubernetes.io/os: linux - # -- Node tolerations for ztunnel scheduling. - tolerations: - - effect: NoSchedule - operator: Exists - - key: CriticalAddonsOnly - operator: Exists - - effect: NoExecute - operator: Exists - # -- Affinity for ztunnel pods. - affinity: {} - # @schema - # type: [null, string] - # @schema - # -- The priority class to use for ztunnel pods. - priorityClassName: ~ - # -- Annotations to be added to all ztunnel resources. - annotations: {} - # -- Annotations to be added to ztunnel pods. - podAnnotations: {} - # -- Labels to be added to ztunnel pods. - podLabels: {} - # -- Additional ztunnel container environment variables. - extraEnv: [] - # -- Additional ztunnel volumes. - extraVolumes: [] - # -- Additional ztunnel volumeMounts. - extraVolumeMounts: [] - # -- ztunnel secrets configuration. - secrets: - # -- Base64-encoded bootstrap root certificate content. - # If not provided, the secret must be created manually before deploying. - # @schema - # type: [null, string] - # @schema - bootstrapRootCert: ~ endpointHealthChecking: # -- Enable connectivity health checking between virtual endpoints. enabled: true -# -- Max duration to wait for envoy to respond to configuration changes. Default "10s". -endpointPolicyUpdateTimeoutDuration: null endpointRoutes: # @schema # type: [boolean, string] @@ -2441,6 +2339,8 @@ enableMasqueradeRouteSource: false enableIPv4BIGTCP: false # -- Enables IPv6 BIG TCP support which increases maximum IPv6 GSO/GRO limits for nodes and pods enableIPv6BIGTCP: false +# -- Enable BIG TCP in tunneling mode and increase maximum GRO/GSO limits for VXLAN/GENEVE tunnels +enableTunnelBIGTCP: false nat: # -- Number of the top-k SNAT map connections to track in Cilium statedb. @@ -2520,7 +2420,7 @@ loadBalancer: # -- serviceTopology enables K8s Topology Aware Hints -based service # endpoints filtering - serviceTopology: false + # serviceTopology: false # -- L7 LoadBalancer l7: diff --git a/packages/system/cilium/images/cilium/Dockerfile b/packages/system/cilium/images/cilium/Dockerfile index 32fc6fb8..b62f9b46 100644 --- a/packages/system/cilium/images/cilium/Dockerfile +++ b/packages/system/cilium/images/cilium/Dockerfile @@ -1,2 +1,2 @@ -ARG VERSION=v1.19.3 +ARG VERSION=v1.19.1 FROM quay.io/cilium/cilium:${VERSION} diff --git a/packages/system/cilium/values-apparmor.yaml b/packages/system/cilium/values-apparmor.yaml deleted file mode 100644 index 0712836c..00000000 --- a/packages/system/cilium/values-apparmor.yaml +++ /dev/null @@ -1,20 +0,0 @@ -cilium: - # Opt out of the cri-containerd.apparmor.d profile for containers that invoke - # nsenter to join the host cgroup/mount namespace. The kernel reports the - # denial as a blocked "ptrace" operation. Required on Ubuntu 22.04+ and - # other distros that load this AppArmor profile by default, where the - # denial otherwise puts cilium-agent into Init:CrashLoopBackOff. - # The deprecated annotations are used because k8s >= 1.30 pod-level - # appArmorProfile: Unconfined is not honoured by containerd CRI today - # (kubernetes/kubernetes#125069). - # Only applied on non-Talos cilium variants where cgroup.autoMount.enabled is - # true — on Talos mount-cgroup is not rendered and the kube-apiserver would - # reject an annotation referencing a non-existent container. - # mount-bpf-fs is intentionally excluded: it renders with its own - # securityContext.privileged=true, and the kernel does not apply AppArmor - # profiles to privileged containers. - podAnnotations: - container.apparmor.security.beta.kubernetes.io/cilium-agent: unconfined - container.apparmor.security.beta.kubernetes.io/clean-cilium-state: unconfined - container.apparmor.security.beta.kubernetes.io/mount-cgroup: unconfined - container.apparmor.security.beta.kubernetes.io/apply-sysctl-overwrites: unconfined diff --git a/packages/system/cilium/values.yaml b/packages/system/cilium/values.yaml index 9aba7cb9..cfb2cd6b 100644 --- a/packages/system/cilium/values.yaml +++ b/packages/system/cilium/values.yaml @@ -15,8 +15,8 @@ cilium: mode: "kubernetes" image: repository: ghcr.io/cozystack/cozystack/cilium - tag: 1.19.3 - digest: "sha256:700f06f4803a838a8e830be5ace4650e3ad82bdefabfb2f4d110368d307a5efb" + tag: 1.19.1 + digest: "sha256:ab3acf270821df4614a8456348a4e0d3098aed72a4b2016a0edfa30d91428c3d" envoy: enabled: false rollOutCiliumPods: true diff --git a/packages/system/cozy-proxy/charts/cozy-proxy/Chart.yaml b/packages/system/cozy-proxy/charts/cozy-proxy/Chart.yaml index 58f166f6..13352680 100644 --- a/packages/system/cozy-proxy/charts/cozy-proxy/Chart.yaml +++ b/packages/system/cozy-proxy/charts/cozy-proxy/Chart.yaml @@ -2,5 +2,5 @@ apiVersion: v2 name: cozy-proxy description: A simple kube-proxy addon for 1:1 NAT services in Kubernetes using an NFT backend type: application -version: 0.3.0 -appVersion: 0.3.0 +version: 0.2.0 +appVersion: 0.2.0 diff --git a/packages/system/cozy-proxy/charts/cozy-proxy/values.yaml b/packages/system/cozy-proxy/charts/cozy-proxy/values.yaml index e143e926..8cde5bed 100644 --- a/packages/system/cozy-proxy/charts/cozy-proxy/values.yaml +++ b/packages/system/cozy-proxy/charts/cozy-proxy/values.yaml @@ -1,6 +1,6 @@ image: repository: ghcr.io/cozystack/cozystack/cozy-proxy - tag: v0.3.0 + tag: v0.2.0 pullPolicy: IfNotPresent daemonset: diff --git a/packages/system/cozystack-api/values.yaml b/packages/system/cozystack-api/values.yaml index dddabf54..ca4cf5e2 100644 --- a/packages/system/cozystack-api/values.yaml +++ b/packages/system/cozystack-api/values.yaml @@ -1,3 +1,3 @@ cozystackAPI: - image: ghcr.io/cozystack/cozystack/cozystack-api:v1.3.0@sha256:5fa8648821cf1e9e08cf7c2899c4b1c4226bb74c6773327141456c7d3f2b9b7e + image: ghcr.io/cozystack/cozystack/cozystack-api:v1.2.1@sha256:afcc72ec5b8ff6eea7fbf927d518f277a7bfead44948f45ed17eda90d47253d5 replicas: 2 diff --git a/packages/system/cozystack-controller/values.yaml b/packages/system/cozystack-controller/values.yaml index 4d0d0fde..030a4564 100644 --- a/packages/system/cozystack-controller/values.yaml +++ b/packages/system/cozystack-controller/values.yaml @@ -1,4 +1,4 @@ cozystackController: - image: ghcr.io/cozystack/cozystack/cozystack-controller:v1.3.0@sha256:d03d19b78c4c98f970ac549a68b01ef6bd1ad755f5e0dcb9e08503511cdf2fdc + image: ghcr.io/cozystack/cozystack/cozystack-controller:v1.2.1@sha256:0b1d49b29e8860f13f7afc8d7b6265f36c9d24d6ad7c40f5164535dd8e933c73 debug: false disableTelemetry: false diff --git a/packages/system/cozystack-scheduler/Chart.yaml b/packages/system/cozystack-scheduler/Chart.yaml index f2dd13ee..c869abb4 100644 --- a/packages/system/cozystack-scheduler/Chart.yaml +++ b/packages/system/cozystack-scheduler/Chart.yaml @@ -1,3 +1,3 @@ apiVersion: v2 name: cozy-cozystack-scheduler -version: 0.3.0 +version: 0.2.0 diff --git a/packages/system/cozystack-scheduler/Makefile b/packages/system/cozystack-scheduler/Makefile index d075ec63..fd4bdf30 100644 --- a/packages/system/cozystack-scheduler/Makefile +++ b/packages/system/cozystack-scheduler/Makefile @@ -3,9 +3,6 @@ export NAMESPACE=kube-system include ../../../hack/package.mk -test: - helm unittest . - update: rm -rf charts tag=$$(git ls-remote --tags --sort="v:refname" https://github.com/cozystack/cozystack-scheduler | awk -F'[/^]' 'END{print $$3}') && \ diff --git a/packages/system/cozystack-scheduler/charts/cozystack-scheduler/Chart.yaml b/packages/system/cozystack-scheduler/charts/cozystack-scheduler/Chart.yaml index f2dd13ee..c869abb4 100644 --- a/packages/system/cozystack-scheduler/charts/cozystack-scheduler/Chart.yaml +++ b/packages/system/cozystack-scheduler/charts/cozystack-scheduler/Chart.yaml @@ -1,3 +1,3 @@ apiVersion: v2 name: cozy-cozystack-scheduler -version: 0.3.0 +version: 0.2.0 diff --git a/packages/system/cozystack-scheduler/charts/cozystack-scheduler/templates/configmap.yaml b/packages/system/cozystack-scheduler/charts/cozystack-scheduler/templates/configmap.yaml index 24e0bc6f..1d78b225 100644 --- a/packages/system/cozystack-scheduler/charts/cozystack-scheduler/templates/configmap.yaml +++ b/packages/system/cozystack-scheduler/charts/cozystack-scheduler/templates/configmap.yaml @@ -52,7 +52,3 @@ data: - name: CozystackInterPodAffinity - name: CozystackNodeAffinity - name: CozystackPodTopologySpread - {{- with .Values.extenders }} - extenders: - {{- toYaml . | nindent 6 }} - {{- end }} diff --git a/packages/system/cozystack-scheduler/charts/cozystack-scheduler/values.yaml b/packages/system/cozystack-scheduler/charts/cozystack-scheduler/values.yaml index 55b6faff..5ab78dbe 100644 --- a/packages/system/cozystack-scheduler/charts/cozystack-scheduler/values.yaml +++ b/packages/system/cozystack-scheduler/charts/cozystack-scheduler/values.yaml @@ -1,4 +1,4 @@ -image: ghcr.io/cozystack/cozystack/cozystack-scheduler:v0.3.0@sha256:89c285c5c5fe3ed8d7d597acf32fc9f045394e0f8efafd1878d6080cf112f6c2 +image: ghcr.io/cozystack/cozystack/cozystack-scheduler:v0.2.0@sha256:89c285c5c5fe3ed8d7d597acf32fc9f045394e0f8efafd1878d6080cf112f6c2 replicas: 1 # defaultLabelSelectorKeys overrides the pod label keys used to auto-populate # nil LabelSelectors in SchedulingClass affinity and topology spread terms. @@ -7,17 +7,3 @@ replicas: 1 # - apps.cozystack.io/application.kind # - apps.cozystack.io/application.name # defaultLabelSelectorKeys: [] - -# extenders is a list of scheduler extenders to call during the scheduling cycle. -# Each entry is passed directly into KubeSchedulerConfiguration.extenders[]. -# Example: -# extenders: -# - urlPrefix: http://linstor-scheduler-extender.cozy-linstor.svc:8099 -# filterVerb: filter -# prioritizeVerb: prioritize -# weight: 5 -# enableHTTPS: false -# httpTimeout: 10s -# nodeCacheCapable: false -# ignorable: true -extenders: [] diff --git a/packages/system/cozystack-scheduler/tests/configmap_test.yaml b/packages/system/cozystack-scheduler/tests/configmap_test.yaml deleted file mode 100644 index ce8f372d..00000000 --- a/packages/system/cozystack-scheduler/tests/configmap_test.yaml +++ /dev/null @@ -1,26 +0,0 @@ -suite: scheduler configmap tests - -templates: - - charts/cozy-cozystack-scheduler/templates/configmap.yaml - -tests: - - it: renders extenders when configured - values: - - ../values-linstor.yaml - asserts: - - isKind: - of: ConfigMap - - matchRegex: - path: data["scheduler-config.yaml"] - pattern: "urlPrefix: http://linstor-scheduler-extender" - - matchRegex: - path: data["scheduler-config.yaml"] - pattern: "ignorable: true" - - - it: omits extenders by default - asserts: - - isKind: - of: ConfigMap - - notMatchRegex: - path: data["scheduler-config.yaml"] - pattern: "extenders:" diff --git a/packages/system/cozystack-scheduler/values-linstor.yaml b/packages/system/cozystack-scheduler/values-linstor.yaml deleted file mode 100644 index ab16ca57..00000000 --- a/packages/system/cozystack-scheduler/values-linstor.yaml +++ /dev/null @@ -1,10 +0,0 @@ -cozy-cozystack-scheduler: - extenders: - - urlPrefix: http://linstor-scheduler-extender.cozy-linstor.svc:8099 - filterVerb: filter - prioritizeVerb: prioritize - weight: 5 - enableHTTPS: false - httpTimeout: 10s - nodeCacheCapable: false - ignorable: true diff --git a/packages/system/dashboard/templates/configmap.yaml b/packages/system/dashboard/templates/configmap.yaml index a2b7684c..d260c5d9 100644 --- a/packages/system/dashboard/templates/configmap.yaml +++ b/packages/system/dashboard/templates/configmap.yaml @@ -1,6 +1,6 @@ {{- $brandingConfig := .Values._cluster.branding | default dict }} -{{- $tenantText := "v1.3.0" }} +{{- $tenantText := "v1.2.1" }} {{- $footerText := "Cozystack" }} {{- $titleText := "Cozystack Dashboard" }} {{- $logoText := "" }} diff --git a/packages/system/dashboard/templates/ingress.yaml b/packages/system/dashboard/templates/ingress.yaml index 6cf01490..a10797fb 100644 --- a/packages/system/dashboard/templates/ingress.yaml +++ b/packages/system/dashboard/templates/ingress.yaml @@ -11,7 +11,7 @@ metadata: annotations: cert-manager.io/cluster-issuer: {{ $clusterIssuer }} {{- if eq $solver "http01" }} - acme.cert-manager.io/http01-ingress-ingressclassname: {{ $exposeIngress }} + acme.cert-manager.io/http01-ingress-class: {{ $exposeIngress }} {{- end }} nginx.ingress.kubernetes.io/rewrite-target: / nginx.ingress.kubernetes.io/client-max-body-size: 100m diff --git a/packages/system/dashboard/values.yaml b/packages/system/dashboard/values.yaml index a5188825..1aa69b09 100644 --- a/packages/system/dashboard/values.yaml +++ b/packages/system/dashboard/values.yaml @@ -1,6 +1,6 @@ openapiUI: - image: ghcr.io/cozystack/cozystack/openapi-ui:v1.3.0@sha256:0fa79c373a62840a617ff1ca1b0e31931c13a6cf7b0bb0ff0dc191f047a465a3 + image: ghcr.io/cozystack/cozystack/openapi-ui:v1.2.1@sha256:a77c502c4b67fa5f0698f66a82eb2ad30e138f9bd2700aa40a63f73cb86a559d openapiUIK8sBff: - image: ghcr.io/cozystack/cozystack/openapi-ui-k8s-bff:v1.3.0@sha256:1b3ea6d4c7dbbe6a8def3b2807fffdfab2ac4afc39d7a846e57dd491fa168f92 + image: ghcr.io/cozystack/cozystack/openapi-ui-k8s-bff:v1.2.1@sha256:1b3ea6d4c7dbbe6a8def3b2807fffdfab2ac4afc39d7a846e57dd491fa168f92 tokenProxy: - image: ghcr.io/cozystack/cozystack/token-proxy:v1.3.0@sha256:2e280991e07853ea48f97b0a42946afffa10d03d6a83d41099ed83e6ffc94fdc + image: ghcr.io/cozystack/cozystack/token-proxy:v1.2.1@sha256:2e280991e07853ea48f97b0a42946afffa10d03d6a83d41099ed83e6ffc94fdc diff --git a/packages/system/gpu-operator/examples/README.md b/packages/system/gpu-operator/examples/README.md deleted file mode 100644 index b6f170c8..00000000 --- a/packages/system/gpu-operator/examples/README.md +++ /dev/null @@ -1,121 +0,0 @@ -# GPU operator — native pod workload on Talos (reference) - -The files in this directory are **not** templates. They are reference -artifacts that document one working configuration for running GPU -workloads directly in pods on a Talos-based Cozystack cluster, together -with the DCGM metrics needed by the `gpu/gpu-performance` Grafana -dashboard. - -The out-of-the-box `values-talos.yaml` for this package targets the -sandbox (VFIO passthrough to KubeVirt VMs) scenario. The files here -illustrate an alternative — running CUDA workloads in regular pods with -the NVIDIA device plugin — and the workarounds it currently requires on -Talos. - -## Files - -- [`values-native-talos.yaml`](./values-native-talos.yaml) — Cozystack - `Package` values that disable sandbox workloads, enable the device - plugin, point `hostPaths.driverInstallDir` at the staging location - used by the compat DaemonSet, and wire DCGM to the custom metrics - ConfigMap. -- [`dcgm-custom-metrics.yaml`](./dcgm-custom-metrics.yaml) — `ConfigMap` - with a DCGM metrics CSV that adds profiling, ECC, throttling and - energy counters on top of the upstream defaults. The CSV is a - superset needed for full coverage of the `gpu/gpu-performance` - dashboard. Which parts are actually required depends on which - dashboards you ship — see the table below. -- [`nvidia-driver-compat.yaml`](./nvidia-driver-compat.yaml) — DaemonSet - that stages `libnvidia-ml.so.1` and `nvidia-smi` from the Talos glibc - tree into a path where the NVIDIA GPU Operator validator expects - them. See the "Why the compat DaemonSet exists" section below. - -## Why these are reference, not templates - -Shipping these as first-class templates would silently impose -assumptions that do not hold for every user: - -- Whether the NVIDIA Talos system extension is installed on the nodes. -- Whether GPUs are exposed directly to pods or passed through to VMs. -- The exact path the installed driver ends up at (depends on the - extension version and Talos release). - -The sandbox-oriented `values-talos.yaml` remains the default. Operators -who want native pod GPU workloads can start from this directory and -adapt as needed. - -## Why the compat DaemonSet exists - -The NVIDIA GPU Operator validator checks for `libnvidia-ml.so.1` and -`bin/nvidia-smi` in the path given by `hostPaths.driverInstallDir`. -Talos installs them under `/usr/local/glibc/usr/lib/` and -`/usr/local/bin/`, which the validator does not look at. Until upstream -addresses [NVIDIA/gpu-operator#1687][1], the DaemonSet copies those -files into a directory the validator does inspect and creates the -`.driver-ctr-ready` flag file so the validator proceeds. - -[1]: https://github.com/NVIDIA/gpu-operator/issues/1687 - -The compat DaemonSet runs privileged and bind-mounts host paths, so -the target namespace must allow privileged pods. On clusters that -enforce the Kubernetes Pod Security Standards at `baseline` or -`restricted`, label the namespace with -`pod-security.kubernetes.io/enforce: privileged` (and the matching -`audit`/`warn` labels if the admission webhook is configured to -surface violations) before applying the manifest. - -## Dashboards and what DCGM metrics they need - -Five GPU dashboards live under `gpu/*` in -`packages/system/monitoring/dashboards-infra.list`. All of them share -`packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml` as -their source of aggregated series. The recording rules are safe to -ship on any cluster — they evaluate to empty series when DCGM is not -scraped, or when optional counters are missing. - -What each dashboard needs on top of the upstream DCGM Exporter -[`default-counters.csv`][default-csv]: - -| Dashboard | Scope | Needs beyond defaults | -| ----------------- | ---------------------------------- | ----------------------------------------------------------------------- | -| `gpu-performance` | Per-node, per-GPU deep dive | `DCGM_FI_DEV_POWER_VIOLATION`, `DCGM_FI_DEV_THERMAL_VIOLATION` | -| `gpu-efficiency` | Per-workload util vs tensor active | `DCGM_FI_DEV_POWER_VIOLATION`, `DCGM_FI_DEV_THERMAL_VIOLATION` (via `gpu:*_throttle_fraction:rate5m` recording rules) | -| `gpu-fleet` | Cluster-wide admin inventory | `DCGM_FI_DEV_POWER_MGMT_LIMIT` (for the TDP vs draw panel) | -| `gpu-quotas` | Kube-quota vs live usage | `kube_pod_container_resource_requests`, `kube_pod_status_phase`, `kube_node_status_allocatable` (via `namespace:gpu_count:allocated` / `cluster:gpu_count:*` recording rules) | -| `gpu-tenants` | Per-namespace tenant view | nothing (works on default counters) | - -`DCGM_FI_PROF_PIPE_TENSOR_ACTIVE` and `DCGM_FI_PROF_GR_ENGINE_ACTIVE` -are already in the upstream default set for the pinned DCGM Exporter -version, so the tensor-saturation and engine-active panels work without -any CSV override. The three counters listed in the table — throttling -violations and the power management limit — are the only extras the -tracked dashboards need. The recording rules in -`gpu-recording.rules.yaml` consume utilization, FB used, power, -temperature and the tensor-active profiling counter from the default -set, plus `DCGM_FI_DEV_POWER_VIOLATION` and -`DCGM_FI_DEV_THERMAL_VIOLATION` — used by the -`gpu.recording.efficiency.1m` group to derive the -`gpu:power_throttle_fraction:rate5m` and -`gpu:thermal_throttle_fraction:rate5m` series consumed by the -throttling panels on the efficiency and fleet dashboards. - -The `gpu.recording.throttle.validation.5m` group additionally ships the -`GPUThrottleFractionOverOne` alert (severity `warning`) as a regression -detector: it fires when either throttle-fraction series exceeds 1.0, -which would indicate that DCGM changed the scale/divisor of the -underlying violation counters and the recording rules need to be -re-derived. - -## Verification status - -The minimum-CSV claims above are verified by -`hack/check-gpu-recording-rules.bats`, which cross-checks every -`DCGM_FI_*` reference in the tracked GPU dashboards and recording rules -against the union of the upstream default set (snapshotted at -`hack/dcgm-default-counters.csv` for the pinned DCGM Exporter version) -and the custom CSV in `dcgm-custom-metrics.yaml`. When the DCGM -Exporter image in `packages/system/gpu-operator/charts/gpu-operator/values.yaml` -is bumped, refresh the snapshot from the matching tag of the -[`NVIDIA/dcgm-exporter`][default-csv] repository and rerun the test. - -[default-csv]: https://github.com/NVIDIA/dcgm-exporter/blob/main/etc/default-counters.csv diff --git a/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml b/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml deleted file mode 100644 index ef2b0b88..00000000 --- a/packages/system/gpu-operator/examples/dcgm-custom-metrics.yaml +++ /dev/null @@ -1,88 +0,0 @@ -# Custom DCGM Exporter metrics CSV. Referenced from -# examples/values-native-talos.yaml via dcgmExporter.config.name. -# -# Extends the upstream default set with profiling counters, ECC, page -# retirement, row remap, energy and throttling violations — everything -# the gpu/gpu-performance dashboard and the GPU recording rules in -# monitoring-agents expect. -apiVersion: v1 -kind: ConfigMap -metadata: - name: dcgm-custom-metrics - namespace: cozy-gpu-operator -data: - dcgm-metrics.csv: | - # Format - # If line starts with a '#' it is considered a comment - # DCGM FIELD, Prometheus metric type, help message - - # Identity - DCGM_FI_DRIVER_VERSION, label, Driver version. - - # Clocks - DCGM_FI_DEV_SM_CLOCK, gauge, SM clock frequency (in MHz). - DCGM_FI_DEV_MEM_CLOCK, gauge, Memory clock frequency (in MHz). - - # Temperature - DCGM_FI_DEV_MEMORY_TEMP, gauge, Memory temperature (in C). - DCGM_FI_DEV_GPU_TEMP, gauge, GPU temperature (in C). - - # Power - DCGM_FI_DEV_POWER_USAGE, gauge, Power draw (in W). - DCGM_FI_DEV_POWER_MGMT_LIMIT, gauge, Current power management limit (in W). - DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION, counter, Total energy consumption since boot (in mJ). - - # PCIE - DCGM_FI_DEV_PCIE_REPLAY_COUNTER, counter, Total number of PCIe retries. - - # Utilization (the sample period varies depending on the product) - DCGM_FI_DEV_GPU_UTIL, gauge, GPU utilization (in %). - DCGM_FI_DEV_MEM_COPY_UTIL, gauge, Memory utilization (in %). - DCGM_FI_DEV_ENC_UTIL, gauge, Encoder utilization (in %). - DCGM_FI_DEV_DEC_UTIL, gauge, Decoder utilization (in %). - - # Errors and violations - DCGM_FI_DEV_XID_ERRORS, gauge, Value of the last XID error encountered. - - # Memory usage - DCGM_FI_DEV_FB_FREE, gauge, Framebuffer memory free (in MiB). - DCGM_FI_DEV_FB_USED, gauge, Framebuffer memory used (in MiB). - DCGM_FI_DEV_FB_RESERVED, gauge, Framebuffer memory reserved (in MiB). - - # ECC (supported on datacenter-class GPUs such as A10) - DCGM_FI_DEV_ECC_SBE_VOL_TOTAL, counter, Total number of single-bit volatile ECC errors. - DCGM_FI_DEV_ECC_DBE_VOL_TOTAL, counter, Total number of double-bit volatile ECC errors. - DCGM_FI_DEV_ECC_SBE_AGG_TOTAL, counter, Total number of single-bit persistent ECC errors. - DCGM_FI_DEV_ECC_DBE_AGG_TOTAL, counter, Total number of double-bit persistent ECC errors. - - # Retired pages - DCGM_FI_DEV_RETIRED_SBE, counter, Total number of retired pages due to single-bit errors. - DCGM_FI_DEV_RETIRED_DBE, counter, Total number of retired pages due to double-bit errors. - DCGM_FI_DEV_RETIRED_PENDING, counter, Total number of pages pending retirement. - - # Row remapping (not applicable to GDDR6 but left for datacenter GPUs) - DCGM_FI_DEV_UNCORRECTABLE_REMAPPED_ROWS, counter, Number of remapped rows for uncorrectable errors. - DCGM_FI_DEV_CORRECTABLE_REMAPPED_ROWS, counter, Number of remapped rows for correctable errors. - DCGM_FI_DEV_ROW_REMAP_FAILURE, gauge, Whether remapping of rows has failed. - - # Throttle / violation counters (crucial for SLA and tenant monitoring) - DCGM_FI_DEV_POWER_VIOLATION, counter, Throttling duration due to power constraints (us per docs; ns on DCGM 3.x). - DCGM_FI_DEV_THERMAL_VIOLATION, counter, Throttling duration due to thermal constraints (us per docs; ns on DCGM 3.x). - DCGM_FI_DEV_SYNC_BOOST_VIOLATION, counter, Throttling duration due to sync-boost constraints (us per docs; ns on DCGM 3.x). - DCGM_FI_DEV_BOARD_LIMIT_VIOLATION, counter, Throttling duration due to board limit constraints (us per docs; ns on DCGM 3.x). - DCGM_FI_DEV_LOW_UTIL_VIOLATION, counter, Throttling duration due to low utilization (us per docs; ns on DCGM 3.x). - DCGM_FI_DEV_RELIABILITY_VIOLATION, counter, Throttling duration due to reliability constraints (us per docs; ns on DCGM 3.x). - - # NVLink — DCGM silently drops this metric on GPUs without NVLink, so - # enabling it unconditionally is safe and keeps this file reusable - # across GPU families. - DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL, counter, Total number of NVLink bandwidth counters for all lanes. - - # DCP (profiling) metrics — supported from Ampere onwards. - DCGM_FI_PROF_GR_ENGINE_ACTIVE, gauge, Ratio of time the graphics engine is active. - DCGM_FI_PROF_SM_ACTIVE, gauge, The ratio of cycles an SM has at least 1 warp assigned. - DCGM_FI_PROF_SM_OCCUPANCY, gauge, The ratio of number of warps resident on an SM. - DCGM_FI_PROF_PIPE_TENSOR_ACTIVE, gauge, Ratio of cycles the tensor (HMMA) pipe is active. - DCGM_FI_PROF_DRAM_ACTIVE, gauge, Ratio of cycles the device memory interface is active sending or receiving data. - DCGM_FI_PROF_PCIE_TX_BYTES, counter, The number of bytes of active PCIe tx (transmit) data including both header and payload. - DCGM_FI_PROF_PCIE_RX_BYTES, counter, The number of bytes of active PCIe rx (read) data including both header and payload. diff --git a/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml b/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml deleted file mode 100644 index fc9d5981..00000000 --- a/packages/system/gpu-operator/examples/nvidia-driver-compat.yaml +++ /dev/null @@ -1,109 +0,0 @@ -# Workaround for https://github.com/NVIDIA/gpu-operator/issues/1687 -# -# On Talos, the NVIDIA system extension installs driver files under -# /usr/local/glibc/usr/lib/ and /usr/local/bin/. The GPU Operator -# validator only looks for libnvidia-ml.so.1 and bin/nvidia-smi under -# the path configured as hostPaths.driverInstallDir. This DaemonSet -# stages the required files into /var/nvidia-driver on each node and -# creates the .driver-ctr-ready flag so the validator proceeds. -# -# Paired with examples/values-native-talos.yaml, which sets -# hostPaths.driverInstallDir to /var/nvidia-driver. -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: nvidia-driver-compat - namespace: cozy-gpu-operator - labels: - app: nvidia-driver-compat -spec: - selector: - matchLabels: - app: nvidia-driver-compat - template: - metadata: - labels: - app: nvidia-driver-compat - # DaemonSet rather than a one-shot Job: staging is idempotent per-node, - # must re-run after every Talos reboot (hostPath contents are wiped on - # reboot when the system extension re-installs), and the pause container - # keeps the pod visible for operator observability (kubectl get pods). - spec: - priorityClassName: system-node-critical - # Restrict to GPU nodes only. Without this the DaemonSet schedules onto - # every node (including control-plane and CPU-only workers) and burns - # resources on hosts where the compat tree is meaningless. - # The label is published by Node Feature Discovery / GPU Operator's NFD - # subchart; if NFD is disabled, replace this with whatever label your - # cluster uses to mark GPU hosts. - nodeSelector: - nvidia.com/gpu.present: "true" - # Tolerate-all is intentionally broad: the nodeSelector above already - # confines scheduling to GPU nodes, so the "Exists" toleration only - # matters when those nodes carry extra taints (dedicated=gpu:NoSchedule, - # nvidia.com/gpu:NoSchedule from the GPU Operator's default policy, - # etc.). This mirrors the stance of the upstream nvidia-driver-daemonset - # and nvidia-device-plugin DaemonSets — blanket tolerations plus a - # narrow nodeSelector. - tolerations: - - operator: Exists - initContainers: - - name: create-driver-tree - image: busybox:1.37 - command: - - sh - - -c - - | - set -e - GLIBC_LIB="/host/usr/local/glibc/usr/lib" - SRC_BIN="/host/usr/local/bin" - DST="/host/var/nvidia-driver" - - mkdir -p "$DST/bin" - - [ -f "$GLIBC_LIB/libnvidia-ml.so.1" ] || { - echo "missing $GLIBC_LIB/libnvidia-ml.so.1" >&2 - exit 1 - } - [ -f "$SRC_BIN/nvidia-smi" ] || { - echo "missing $SRC_BIN/nvidia-smi" >&2 - exit 1 - } - - cp -f "$GLIBC_LIB/libnvidia-ml.so.1" "$DST/libnvidia-ml.so.1" - echo "Copied libnvidia-ml.so.1" - - cp -f "$SRC_BIN/nvidia-smi" "$DST/bin/nvidia-smi" - chmod +x "$DST/bin/nvidia-smi" - echo "Copied nvidia-smi" - - mkdir -p /host/run/nvidia/validations - touch /host/run/nvidia/validations/.driver-ctr-ready - echo "Created driver-ctr-ready flag" - echo "Done" - securityContext: - privileged: true - resources: - requests: - cpu: 10m - memory: 16Mi - limits: - cpu: 100m - memory: 64Mi - volumeMounts: - - name: host-root - mountPath: /host - containers: - - name: pause - image: registry.k8s.io/pause:3.10 - resources: - requests: - cpu: 10m - memory: 8Mi - limits: - cpu: 50m - memory: 16Mi - volumes: - - name: host-root - hostPath: - path: / diff --git a/packages/system/gpu-operator/examples/values-native-talos.yaml b/packages/system/gpu-operator/examples/values-native-talos.yaml deleted file mode 100644 index ef7b2891..00000000 --- a/packages/system/gpu-operator/examples/values-native-talos.yaml +++ /dev/null @@ -1,52 +0,0 @@ -# Cozystack Package values for running GPU workloads natively in pods -# on Talos. This is the counterpart to values-talos.yaml, which targets -# the sandbox (VFIO passthrough) scenario. -# -# Prerequisites: -# - NVIDIA Talos system extension installed on GPU nodes. -# - examples/nvidia-driver-compat.yaml deployed to stage driver files -# where the gpu-operator validator looks for them. -# - examples/dcgm-custom-metrics.yaml applied so DCGM Exporter exports -# the full set of metrics used by the dashboard and recording rules. -apiVersion: cozystack.io/v1alpha1 -kind: Package -metadata: - name: cozystack.gpu-operator -spec: - variant: default - components: - gpu-operator: - values: - gpu-operator: - # The compat DaemonSet stages libnvidia-ml.so.1 and nvidia-smi - # under /var/nvidia-driver. Point the validator at that path. - hostPaths: - driverInstallDir: "/var/nvidia-driver" - # Disable the sandbox path — workloads run in pods, not VMs. - sandboxWorkloads: - enabled: false - vfioManager: - enabled: false - # The Talos extension provides the driver and runtime hooks, - # so the operator's own toolkit and driver components must be - # switched off to avoid conflicting installations. - toolkit: - enabled: false - devicePlugin: - enabled: true - # Export full set of DCGM metrics using the ConfigMap in - # examples/dcgm-custom-metrics.yaml. - dcgmExporter: - serviceMonitor: - enabled: true - # Matches the 1m evaluation cadence of GPU recording rules with - # 4x oversampling headroom. DCGM_FI_PROF_* counters have - # documented overhead concerns below 1s; 15s is safe. - interval: "15s" - honorLabels: true - relabelings: - - sourceLabels: [__meta_kubernetes_pod_node_name] - targetLabel: node - action: replace - config: - name: dcgm-custom-metrics diff --git a/packages/system/grafana-operator/images/grafana-dashboards.tag b/packages/system/grafana-operator/images/grafana-dashboards.tag index cde83a70..08f6bd0b 100644 --- a/packages/system/grafana-operator/images/grafana-dashboards.tag +++ b/packages/system/grafana-operator/images/grafana-dashboards.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/grafana-dashboards:v1.3.0@sha256:2c9aa0b48e2bf6167db198f4d15882bfe51700108edf2e9f6d0942940a2c1204 +ghcr.io/cozystack/cozystack/grafana-dashboards:v1.2.1@sha256:2c9aa0b48e2bf6167db198f4d15882bfe51700108edf2e9f6d0942940a2c1204 diff --git a/packages/system/hami/Chart.yaml b/packages/system/hami/Chart.yaml deleted file mode 100644 index 3fcf4c5d..00000000 --- a/packages/system/hami/Chart.yaml +++ /dev/null @@ -1,3 +0,0 @@ -apiVersion: v2 -name: cozy-hami -version: 0.0.0 # Placeholder, the actual version will be automatically set during the build process diff --git a/packages/system/hami/Makefile b/packages/system/hami/Makefile deleted file mode 100644 index 83663a66..00000000 --- a/packages/system/hami/Makefile +++ /dev/null @@ -1,43 +0,0 @@ -export NAME=hami -export NAMESPACE=cozy-$(NAME) - -include ../../../hack/common-envs.mk -include ../../../hack/package.mk - -# When bumping the HAMi version, run `make update` and then review -# the resulting diff in `charts/hami/`. The recipe below reproduces the -# top-level vendoring overrides automatically: -# -# 1. Removes the broken hami-dra subchart. Upstream's NVIDIA DRA driver -# path requires kubelet DRA support that cozystack does not enable -# and has no upstream fix tracked. See commit 3c5521e. -# 2. Empties Chart.yaml dependencies and drops Chart.lock so Helm does -# not try to re-pull hami-dra at build time. See commit 2734dc0. -# 3. Strips dra/hami-dra/podSecurityPolicy blocks from the upstream -# values.yaml since the corresponding code paths are gone. PSP is -# removed from Kubernetes 1.25+ and is unused by cozystack. -# -# Template-level patches are NOT reproduced automatically: -# -# * Scheduler templates have `{{- if .Values.dra.enabled }}` blocks -# that need to be removed because the dra value is gone (commit -# 2734dc0 stripped them). -# * device-plugin/monitorservice.yaml uses `indent` with leading -# whitespace; it must be rewritten to `nindent` for the labels block -# to render correctly when devicePlugin.service.labels is set -# (commit 3685254). -# -# After `make update`, run `git diff -- charts/hami/templates/` and -# replay these template patches against the new upstream version, then -# verify with `helm unittest`. If upstream restructured the affected -# files, the patches may need to be redesigned rather than reapplied. - -update: - rm -rf charts - helm repo add hami-charts https://project-hami.github.io/HAMi/ - helm repo update hami-charts - helm pull hami-charts/hami --untar --untardir charts - rm -rf charts/hami/charts/hami-dra - yq --inplace '.dependencies = []' charts/hami/Chart.yaml - rm -f charts/hami/Chart.lock - yq --inplace 'del(.dra) | del(.["hami-dra"]) | del(.podSecurityPolicy)' charts/hami/values.yaml diff --git a/packages/system/hami/README.md b/packages/system/hami/README.md deleted file mode 100644 index 68669ac7..00000000 --- a/packages/system/hami/README.md +++ /dev/null @@ -1,82 +0,0 @@ -# HAMi — GPU Virtualization Middleware - -[HAMi](https://github.com/Project-HAMi/HAMi) (Heterogeneous AI Computing Virtualization Middleware) is a CNCF Sandbox project that enables fractional GPU sharing in Kubernetes. It allows workloads to request specific amounts of GPU memory and compute cores instead of claiming entire GPUs. - -## Architecture - -HAMi consists of four components: - -- **MutatingWebhook** — intercepts pod creation, injects `schedulerName: hami-scheduler` -- **Scheduler Extender** — extends kube-scheduler with GPU-aware Filter and Bind logic -- **Device Plugin** (DaemonSet) — registers vGPU resources via the Kubernetes Device Plugin API -- **HAMi-core** (`libvgpu.so`) — `LD_PRELOAD` library injected into workload containers, intercepts CUDA API calls to enforce memory and compute isolation - -## Prerequisites - -- GPU Operator must be enabled (`addons.gpuOperator.enabled: true`) -- NVIDIA driver >= 440 on host nodes -- nvidia-container-toolkit configured as the default container runtime -- GPU nodes labeled with `gpu=on` - -## Known Limitations - -### glibc < 2.34 requirement for workload containers - -HAMi-core uses `LD_PRELOAD` to intercept `dlsym()` for CUDA symbol resolution. The fallback code path relies on `_dl_sym`, a private glibc internal symbol that was removed in glibc 2.34 when libdl and libpthread were merged into libc.so. - -**This limitation affects workload containers only**, not the host OS or HAMi's own components. - -| Distribution | glibc | Result | -| --------------- | ----- | -------------------------------------------- | -| Ubuntu 18.04 | 2.27 | Full isolation (memory + compute) | -| Ubuntu 20.04 | 2.31 | Full isolation (memory + compute) | -| Ubuntu 22.04 | 2.35 | Memory isolation works, compute breaks | -| Ubuntu 24.04 | 2.39 | Both memory and compute isolation break | -| Alpine (musl) | N/A | Completely incompatible (`dlvsym` absent) | - -Most modern ML/AI base images (CUDA 12.x, PyTorch 2.x, TensorFlow 2.x) use Ubuntu 22.04+ with glibc >= 2.35, which means compute isolation will not work with these images until the upstream fix is merged. - -**Upstream tracking issues:** - -- [HAMi-core#174](https://github.com/Project-HAMi/HAMi-core/issues/174) — `_dl_sym` removal in glibc 2.34 breaks HAMi-core's CUDA symbol resolution at the symbol level -- [HAMi#1190](https://github.com/Project-HAMi/HAMi/issues/1190) — maintainer thread confirming the empirical per-glibc-version isolation behavior shown in the table above - -### musl libc (Alpine) incompatibility - -HAMi-core is completely incompatible with musl libc. The `dlvsym()` function used by HAMi-core is a glibc extension not available in musl. Only glibc-based container images (Debian, Ubuntu, RHEL, etc.) can use HAMi GPU isolation. - -## Usage - -Enable HAMi in your tenant Kubernetes cluster values: - -```yaml -addons: - gpuOperator: - enabled: true - hami: - enabled: true -``` - -When HAMi is enabled, GPU Operator's built-in device plugin is automatically disabled to avoid conflicts. This default is preserved by setting `addons.gpuOperator.valuesOverride.gpu-operator.devicePlugin.enabled: false`; advanced topologies that partition GPU pools (e.g. some nodes use HAMi while others run the standard NVIDIA device plugin via node selectors) can re-enable it explicitly through `valuesOverride`. - -### Requesting fractional GPU resources - -```yaml -resources: - limits: - nvidia.com/gpu: 1 - nvidia.com/gpumem: 3000 # 3000 MB of GPU memory - nvidia.com/gpucores: 30 # 30% of GPU compute cores -``` - -## Parameters - -Default values shown below are inherited from the upstream HAMi chart and may change with upstream updates. - -| Name | Description | Default | -| --- | --- | --- | -| `hami.devicePlugin.runtimeClassName` | RuntimeClass for device plugin pods | `nvidia` | -| `hami.devicePlugin.deviceSplitCount` | Max virtual GPUs per physical GPU | `10` | -| `hami.devicePlugin.deviceMemoryScaling` | Memory overcommit factor (> 1.0 enables overcommit) | `1` | -| `hami.scheduler.defaultSchedulerPolicy.nodeSchedulerPolicy` | Node packing strategy (`binpack` or `spread`) | `binpack` | -| `hami.scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy` | GPU packing strategy (`binpack` or `spread`) | `spread` | diff --git a/packages/system/hami/charts/hami/Chart.yaml b/packages/system/hami/charts/hami/Chart.yaml deleted file mode 100644 index 55f32ab6..00000000 --- a/packages/system/hami/charts/hami/Chart.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: v2 -appVersion: 2.8.1 -dependencies: [] -description: Heterogeneous AI Computing Virtualization Middleware -keywords: -- vgpu -- gpu -kubeVersion: '>= 1.18.0-0' -maintainers: -- email: archlitchi@gmail.com - name: limengxuan -- email: xiaozhang0210@hotmail.com - name: zhangxiao -name: hami -sources: -- https://github.com/Project-HAMi/HAMi -type: application -version: 2.8.1 diff --git a/packages/system/hami/charts/hami/README.md b/packages/system/hami/charts/hami/README.md deleted file mode 100644 index e8217290..00000000 --- a/packages/system/hami/charts/hami/README.md +++ /dev/null @@ -1,237 +0,0 @@ -# HAMi Helm Chart Values Documentation - -This document provides detailed descriptions of all configurable values parameters for the HAMi Helm Chart. - -## Global Configuration - -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `global.imageRegistry` | Global Docker image registry | `""` | -| `global.imagePullSecrets` | Global Docker image pull secrets | `[]` | -| `global.imageTag` | Image tag | `"v2.8.1"` | -| `global.gpuHookPath` | GPU Hook path | `/usr/local` | -| `global.labels` | Global labels | `{}` | -| `global.annotations` | Global annotations | `{}` | -| `global.managedNodeSelectorEnable` | Whether to enable managed node selector | `false` | -| `global.managedNodeSelector.usage` | Managed node selector usage | `"gpu"` | -| `nameOverride` | Name override | `""` | -| `fullnameOverride` | Full name override | `""` | -| `namespaceOverride` | Namespace override | `""` | - -## Resource Name Configuration - -### NVIDIA GPU Resources -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `resourceName` | GPU resource name | `"nvidia.com/gpu"` | -| `resourceMem` | GPU memory resource name | `"nvidia.com/gpumem"` | -| `resourceMemPercentage` | GPU memory percentage resource name | `"nvidia.com/gpumem-percentage"` | -| `resourceCores` | GPU core resource name | `"nvidia.com/gpucores"` | -| `resourcePriority` | GPU priority resource name | `"nvidia.com/priority"` | - -### Cambricon MLU Resources -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `mluResourceName` | MLU resource name | `"cambricon.com/vmlu"` | -| `mluResourceMem` | MLU memory resource name | `"cambricon.com/mlu.smlu.vmemory"` | -| `mluResourceCores` | MLU core resource name | `"cambricon.com/mlu.smlu.vcore"` | - -### Hygon DCU Resources -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `dcuResourceName` | DCU resource name | `"hygon.com/dcunum"` | -| `dcuResourceMem` | DCU memory resource name | `"hygon.com/dcumem"` | -| `dcuResourceCores` | DCU core resource name | `"hygon.com/dcucores"` | - -### Metax GPU Resources -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `metaxResourceName` | GPU resource name | `"metax-tech.com/sgpu"` | -| `metaxResourceCore` | GPU core resource name | `"metax-tech.com/vcore"` | -| `metaxResourceMem` | GPU memory resource name | `"metax-tech.com/vmemory"` | -| `metaxsGPUTopologyAware` | GPU topology awareness | `"false"` | - -### Enflame GCU Resources -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `enflameResourceNameVGCU` | vGCU resource name | `"enflame.com/vgcu"` | -| `enflameResourceNameVGCUPercentage` | vGCU percentage resource name | `"enflame.com/vgcu-percentage"` | - -### Kunlunxin XPU Resources -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `kunlunResourceName` | XPU resource name | `"kunlunxin.com/xpu"` | - -## Scheduler Configuration - -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `schedulerName` | Scheduler name | `"hami-scheduler"` | -| `scheduler.nodeName` | Define node name, scheduler will schedule to this node | `""` | -| `scheduler.overwriteEnv` | Whether to overwrite environment variables | `"false"` | -| `scheduler.defaultSchedulerPolicy.nodeSchedulerPolicy` | Node scheduler policy | `binpack` | -| `scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy` | GPU scheduler policy | `spread` | -| `scheduler.metricsBindAddress` | Metrics bind address | `":9395"` | -| `scheduler.forceOverwriteDefaultScheduler` | Whether to force overwrite default scheduler | `true` | -| `scheduler.livenessProbe` | Whether to enable liveness probe | `false` | -| `scheduler.leaderElect` | Whether to enable leader election | `true` | -| `scheduler.replicas` | Number of replicas | `1` | - -### Kube Scheduler Configuration - -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `scheduler.kubeScheduler.enabled` | Whether to run kube-scheduler container in scheduler pod | `true` | -| `scheduler.kubeScheduler.image.registry` | Kube scheduler image registry | `"registry.cn-hangzhou.aliyuncs.com"` | -| `scheduler.kubeScheduler.image.repository` | Kube scheduler image repository | `"google_containers/kube-scheduler"` | -| `scheduler.kubeScheduler.image.tag` | Kube scheduler image tag | `""` | -| `scheduler.kubeScheduler.image.pullPolicy` | Kube scheduler image pull policy | `IfNotPresent` | -| `scheduler.kubeScheduler.image.pullSecrets` | Kube scheduler image pull secrets | `[]` | -| `scheduler.kubeScheduler.extraNewArgs` | Extra new arguments | `["--config=/config/config.yaml", "-v=4"]` | -| `scheduler.kubeScheduler.extraArgs` | Extra arguments | `["--policy-config-file=/config/config.json", "-v=4"]` | - -### Extender Configuration - -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `scheduler.extender.image.registry` | Scheduler extender image registry | `"docker.io"` | -| `scheduler.extender.image.repository` | Scheduler extender image repository | `"projecthami/hami"` | -| `scheduler.extender.image.tag` | Scheduler extender image tag | `""` | -| `scheduler.extender.image.pullPolicy` | Scheduler extender image pull policy | `IfNotPresent` | -| `scheduler.extender.image.pullSecrets` | Scheduler extender image pull secrets | `[]` | -| `scheduler.extender.extraArgs` | Scheduler extender extra arguments | `["--debug", "-v=4"]` | - -### Admission Webhook Configuration - -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `scheduler.admissionWebhook.enabled` | Whether to enable admission webhook | `true` | -| `scheduler.admissionWebhook.customURL.enabled` | Whether to enable custom URL | `false` | -| `scheduler.admissionWebhook.customURL.host` | Custom URL host | `127.0.0.1` | -| `scheduler.admissionWebhook.customURL.port` | Custom URL port | `31998` | -| `scheduler.admissionWebhook.customURL.path` | Custom URL path | `/webhook` | -| `scheduler.admissionWebhook.reinvocationPolicy` | Reinvocation policy | `Never` | -| `scheduler.admissionWebhook.failurePolicy` | Failure policy | `Ignore` | - -### TLS Certificate Configuration - -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `scheduler.certManager.enabled` | Whether to use cert-manager to generate self-signed certificates | `false` | -| `scheduler.patch.enabled` | Whether to use kube-webhook-certgen to generate self-signed certificates | `true` | -| `scheduler.patch.image.registry` | Certgen image registry | `"docker.io"` | -| `scheduler.patch.image.repository` | Certgen image repository | `"jettech/kube-webhook-certgen"` | -| `scheduler.patch.image.tag` | Certgen image tag | `"v1.5.2"` | -| `scheduler.patch.image.pullPolicy` | Certgen image pull policy | `IfNotPresent` | -| `scheduler.patch.imageNew.registry` | New certgen image registry | `"docker.io"` | -| `scheduler.patch.imageNew.repository` | New certgen image repository | `"liangjw/kube-webhook-certgen"` | -| `scheduler.patch.imageNew.tag` | New certgen image tag | `"v1.1.1"` | - -### Scheduler Service Configuration - -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `scheduler.service.type` | Service type | `NodePort` | -| `scheduler.service.httpPort` | HTTP port | `443` | -| `scheduler.service.schedulerPort` | Scheduler NodePort | `31998` | -| `scheduler.service.monitorPort` | Monitor port | `31993` | -| `scheduler.service.monitorTargetPort` | Monitor target port | `9395` | - -## Device Plugin Configuration - -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `devicePlugin.image.registry` | Device plugin image registry | `"docker.io"` | -| `devicePlugin.image.repository` | Device plugin image repository | `"projecthami/hami"` | -| `devicePlugin.image.tag` | Device plugin image tag | `""` | -| `devicePlugin.image.pullPolicy` | Device plugin image pull policy | `IfNotPresent` | -| `devicePlugin.image.pullSecrets` | Device plugin image pull secrets | `[]` | - -### Monitor Configuration - -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `devicePlugin.monitor.image.registry` | Monitor image registry | `"docker.io"` | -| `devicePlugin.monitor.image.repository` | Monitor image repository | `"projecthami/hami"` | -| `devicePlugin.monitor.image.tag` | Monitor image tag | `""` | -| `devicePlugin.monitor.image.pullPolicy` | Monitor image pull policy | `IfNotPresent` | -| `devicePlugin.monitor.image.pullSecrets` | Monitor image pull secrets | `[]` | -| `devicePlugin.monitor.ctrPath` | Container path | `/usr/local/vgpu/containers` | -| `devicePlugin.monitor.extraArgs` | Monitor extra arguments | `["-v=4"]` | -| `devicePlugin.monitor.extraEnvs` | Monitor extra environments | `{}` | - -### Device Plugin Other Configuration - -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `devicePlugin.deviceSplitCount` | Integer type, default value: 10. Maximum number of tasks assigned to a single GPU device | `10` | -| `devicePlugin.deviceMemoryScaling` | Device memory scaling ratio | `1` | -| `devicePlugin.deviceCoreScaling` | Device core scaling ratio | `1` | -| `devicePlugin.runtimeClassName` | Runtime class name | `""` | -| `devicePlugin.createRuntimeClass` | Whether to create runtime class | `false` | -| `devicePlugin.migStrategy` | String type, "none" means ignore MIG functionality, "mixed" means allocate MIG devices through independent resources | `"none"` | -| `devicePlugin.disablecorelimit` | String type, "true" means disable core limit, "false" means enable core limit | `"false"` | -| `devicePlugin.passDeviceSpecsEnabled` | Whether to enable passing device specs | `false` | -| `devicePlugin.extraArgs` | Device plugin extra arguments | `["-v=4"]` | -| `devicePlugin.nodeConfiguration.config` | Node configuration for device plugin by json | An example of default configuration. | -| `devicePlugin.nodeConfiguration.externalConfigName` | Node configuration for device plugin by external congimap | `""` | -| `devicePlugin.extraEnvs` | Device plugin extra environments | `{}` | - -### Device Plugin Service Configuration - -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `devicePlugin.service.type` | Service type | `NodePort` | -| `devicePlugin.service.httpPort` | HTTP port | `31992` | - -### Device Plugin Deployment Configuration - -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `devicePlugin.pluginPath` | Plugin path | `/var/lib/kubelet/device-plugins` | -| `devicePlugin.libPath` | Library path | `/usr/local/vgpu` | -| `devicePlugin.nvidiaNodeSelector` | NVIDIA node selector | `{"gpu": "on"}` | -| `devicePlugin.updateStrategy.type` | Update strategy type | `RollingUpdate` | -| `devicePlugin.updateStrategy.rollingUpdate.maxUnavailable` | Maximum unavailable count | `1` | - -## Device Configuration - -### AWS Neuron -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `devices.awsneuron.customresources` | Custom resources | `["aws.amazon.com/neuron", "aws.amazon.com/neuroncore"]` | - -### Kunlunxin -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `devices.kunlun.enabled` | Whether to enable | `true` | -| `devices.kunlun.customresources` | Custom resources | `["kunlunxin.com/xpu"]` | - -### Mthreads -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `devices.mthreads.enabled` | Whether to enable | `true` | -| `devices.mthreads.customresources` | Custom resources | `["mthreads.com/vgpu"]` | - -### NVIDIA -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `devices.nvidia.gpuCorePolicy` | GPU core policy | `default` | -| `devices.nvidia.libCudaLogLevel` | CUDA library log level | `1` | - -### Huawei Ascend -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `devices.ascend.enabled` | Whether to enable | `false` | -| `devices.ascend.image` | Image | `""` | -| `devices.ascend.imagePullPolicy` | Image pull policy | `IfNotPresent` | -| `devices.ascend.extraArgs` | Extra arguments | `[]` | -| `devices.ascend.nodeSelector` | Node selector | `{"ascend": "on"}` | -| `devices.ascend.tolerations` | Tolerations | `[]` | -| `devices.ascend.customresources` | Custom resources | `["huawei.com/Ascend910A", "huawei.com/Ascend910A-memory", ...]` | - -### Iluvatar -| Parameter | Description | Default Value | -|-----------|-------------|---------------| -| `devices.iluvatar.enabled` | Whether to enable | `false` | -| `devices.iluvatar.customresources` | Custom resources | `["iluvatar.ai/BI-V150-vgpu", "iluvatar.ai/BI-V150.vMem","iluvatar.ai/BI-V150.vCore", ...]` | diff --git a/packages/system/hami/charts/hami/templates/NOTES.txt b/packages/system/hami/charts/hami/templates/NOTES.txt deleted file mode 100644 index 15bb1218..00000000 --- a/packages/system/hami/charts/hami/templates/NOTES.txt +++ /dev/null @@ -1,3 +0,0 @@ -** Please be patient while the chart is being deployed ** -Resource name: {{ .Values.resourceName }} - diff --git a/packages/system/hami/charts/hami/templates/_commons.tpl b/packages/system/hami/charts/hami/templates/_commons.tpl deleted file mode 100644 index b68018e4..00000000 --- a/packages/system/hami/charts/hami/templates/_commons.tpl +++ /dev/null @@ -1,49 +0,0 @@ -{{/* -Return the proper image name -{{ include "common.images.image" ( dict "imageRoot" .Values.path.to.the.image "global" $) }} -*/}} -{{- define "common.images.image" -}} -{{- $registryName := .imageRoot.registry -}} -{{- $repositoryName := .imageRoot.repository -}} -{{- $tag := .imageRoot.tag | toString -}} -{{- if .global }} - {{- if .global.imageRegistry }} - {{- $registryName = .global.imageRegistry -}} - {{- end -}} -{{- end -}} -{{- if .tag }} - {{- $tag = .tag | toString -}} -{{- end -}} -{{- if $registryName }} -{{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} -{{- else -}} -{{- printf "%s:%s" $repositoryName $tag -}} -{{- end -}} -{{- end -}} - -{{/* -Return the proper Docker Image Registry Secret Names (deprecated: use common.images.renderPullSecrets instead) -{{ include "common.images.pullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global) }} -*/}} -{{- define "common.images.pullSecrets" -}} - {{- $pullSecrets := list }} - - {{- if .global }} - {{- range .global.imagePullSecrets -}} - {{- $pullSecrets = append $pullSecrets . -}} - {{- end -}} - {{- end -}} - - {{- range .images -}} - {{- range .pullSecrets -}} - {{- $pullSecrets = append $pullSecrets . -}} - {{- end -}} - {{- end -}} - - {{- if (not (empty $pullSecrets)) }} -imagePullSecrets: - {{- range $pullSecrets }} - - name: {{ . }} - {{- end }} - {{- end }} -{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/_helpers.tpl b/packages/system/hami/charts/hami/templates/_helpers.tpl deleted file mode 100644 index ffcc61da..00000000 --- a/packages/system/hami/charts/hami/templates/_helpers.tpl +++ /dev/null @@ -1,163 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "hami-vgpu.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "hami-vgpu.fullname" -}} -{{- if .Values.fullnameOverride -}} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Allow the release namespace to be overridden for multi-namespace deployments in combined charts -*/}} -{{- define "hami-vgpu.namespace" -}} - {{- if .Values.namespaceOverride -}} - {{- .Values.namespaceOverride -}} - {{- else -}} - {{- .Release.Namespace -}} - {{- end -}} -{{- end -}} - -{{/* -The app name for Scheduler -*/}} -{{- define "hami-vgpu.scheduler" -}} -{{- printf "%s-scheduler" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -The app name for DevicePlugin -*/}} -{{- define "hami-vgpu.device-plugin" -}} -{{- printf "%s-device-plugin" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* - The app name for MockDevicePlugin - */}} -{{- define "hami-vgpu.mock-device-plugin" -}} -{{- printf "%s-mock-device-plugin" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -The tls secret name for Scheduler -*/}} -{{- define "hami-vgpu.scheduler.tls" -}} -{{- printf "%s-scheduler-tls" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -The webhook name -*/}} -{{- define "hami-vgpu.scheduler.webhook" -}} -{{- printf "%s-webhook" ( include "hami-vgpu.fullname" . ) | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "hami-vgpu.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "hami-vgpu.labels" -}} -helm.sh/chart: {{ include "hami-vgpu.chart" . }} -{{ include "hami-vgpu.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "hami-vgpu.selectorLabels" -}} -app.kubernetes.io/name: {{ include "hami-vgpu.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - - -{{/* - Resolve the tag for kubeScheduler. -*/}} -{{- define "resolvedKubeSchedulerTag" -}} -{{- if .Values.scheduler.kubeScheduler.image.tag }} -{{- .Values.scheduler.kubeScheduler.image.tag | trim -}} -{{- else }} -{{- include "strippedKubeVersion" . | trim -}} -{{- end }} -{{- end }} - -{{/* - Return the stripped Kubernetes version string by removing extra parts after semantic version number. - v1.31.1+k3s1 -> v1.31.1 - v1.30.8-eks-2d5f260 -> v1.30.8 - v1.31.1 -> v1.31.1 -*/}} -{{- define "strippedKubeVersion" -}} -{{ regexReplaceAll "^(v[0-9]+\\.[0-9]+\\.[0-9]+)(.*)$" .Capabilities.KubeVersion.Version "$1" }} -{{- end -}} - -{{- define "hami.scheduler.kubeScheduler.image" -}} -{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.kubeScheduler.image "global" .Values.global "tag" (include "resolvedKubeSchedulerTag" .)) }} -{{- end -}} - -{{- define "hami.scheduler.extender.image" -}} -{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.extender.image "global" .Values.global "tag" .Values.global.imageTag) }} -{{- end -}} - -{{- define "hami.devicePlugin.image" -}} -{{ include "common.images.image" (dict "imageRoot" .Values.devicePlugin.image "global" .Values.global "tag" .Values.global.imageTag) }} -{{- end -}} - -{{- define "hami.mockDevicePlugin.image" -}} -{{ include "common.images.image" (dict "imageRoot" .Values.mockDevicePlugin.image "global" .Values.global "tag" .Values.mockDevicePlugin.tag) }} -{{- end -}} - -{{- define "hami.devicePlugin.monitor.image" -}} -{{ include "common.images.image" (dict "imageRoot" .Values.devicePlugin.monitor.image "global" .Values.global "tag" .Values.global.imageTag) }} -{{- end -}} - -{{- define "hami.scheduler.patch.image" -}} -{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.patch.image "global" .Values.global) }} -{{- end -}} - -{{- define "hami.scheduler.patch.new.image" -}} -{{ include "common.images.image" (dict "imageRoot" .Values.scheduler.patch.imageNew "global" .Values.global) }} -{{- end -}} - -{{- define "hami.scheduler.extender.imagePullSecrets" -}} -{{ include "common.images.pullSecrets" (dict "images" (list .Values.scheduler.extender.image) "global" .Values.global) }} -{{- end -}} - -{{- define "hami.devicePlugin.imagePullSecrets" -}} -{{ include "common.images.pullSecrets" (dict "images" (list .Values.devicePlugin.image) "global" .Values.global) }} -{{- end -}} - -{{- define "hami.scheduler.patch.imagePullSecrets" -}} -{{ include "common.images.pullSecrets" (dict "images" (list .Values.scheduler.patch.image) "global" .Values.global) }} -{{- end -}} - -{{- define "hami.scheduler.patch.new.imagePullSecrets" -}} -{{ include "common.images.pullSecrets" (dict "images" (list .Values.scheduler.patch.imageNew) "global" .Values.global) }} -{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml b/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml deleted file mode 100644 index 74631e24..00000000 --- a/packages/system/hami/charts/hami/templates/device-plugin/configmap.yaml +++ /dev/null @@ -1,13 +0,0 @@ -{{- if and .Values.devicePlugin.enabled (not .Values.devicePlugin.nodeConfiguration.externalConfigName) -}} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "hami-vgpu.device-plugin" . }} - namespace: {{ include "hami-vgpu.namespace" . }} - labels: - app.kubernetes.io/component: hami-device-plugin - {{- include "hami-vgpu.labels" . | nindent 4 }} -data: - config.json: | -{{- .Values.devicePlugin.nodeConfiguration.config | nindent 4 }} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/daemonsetmock.yaml b/packages/system/hami/charts/hami/templates/device-plugin/daemonsetmock.yaml deleted file mode 100644 index a0dc4ff3..00000000 --- a/packages/system/hami/charts/hami/templates/device-plugin/daemonsetmock.yaml +++ /dev/null @@ -1,55 +0,0 @@ -{{- if .Values.mockDevicePlugin.enabled }} -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: {{ include "hami-vgpu.mock-device-plugin" . }} - namespace: {{ include "hami-vgpu.namespace" . }} -spec: - selector: - matchLabels: - app.kubernetes.io/component: hami-mock-device-plugin - {{- include "hami-vgpu.selectorLabels" . | nindent 6 }} - template: - metadata: - annotations: - scheduler.alpha.kubernetes.io/critical-pod: "" - labels: - app.kubernetes.io/component: hami-mock-device-plugin - {{- include "hami-vgpu.selectorLabels" . | nindent 8 }} - spec: - serviceAccountName: {{ include "hami-vgpu.mock-device-plugin" . }} - tolerations: - - key: CriticalAddonsOnly - operator: Exists - containers: - - image: {{ include "hami.mockDevicePlugin.image" . }} - imagePullPolicy: {{ .Values.mockDevicePlugin.image.pullPolicy }} - name: hami-mock-dp-cntr - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - command: - - ./k8s-device-plugin - - -v=5 - - --device-config-file=/device-config.yaml - volumeMounts: - - name: dp - mountPath: /var/lib/kubelet/device-plugins - - name: sys - mountPath: /sys - - name: device-config - mountPath: /device-config.yaml - subPath: device-config.yaml - volumes: - - name: dp - hostPath: - path: /var/lib/kubelet/device-plugins - - name: sys - hostPath: - path: /sys - - name: device-config - configMap: - name: {{ include "hami-vgpu.scheduler" . }}-device -{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml b/packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml deleted file mode 100644 index 1f4c24f3..00000000 --- a/packages/system/hami/charts/hami/templates/device-plugin/daemonsetnvidia.yaml +++ /dev/null @@ -1,262 +0,0 @@ -{{- if .Values.devicePlugin.enabled }} -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: {{ include "hami-vgpu.device-plugin" . }} - namespace: {{ include "hami-vgpu.namespace" . }} - labels: - app.kubernetes.io/component: hami-device-plugin - {{- include "hami-vgpu.labels" . | nindent 4 }} - {{- with .Values.global.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} - {{- if .Values.global.annotations }} - annotations: {{ toYaml .Values.global.annotations | nindent 4}} - {{- end }} -spec: - updateStrategy: - {{- with .Values.devicePlugin.updateStrategy }} - {{- toYaml . | nindent 4 }} - {{- end }} - selector: - matchLabels: - app.kubernetes.io/component: hami-device-plugin - {{- include "hami-vgpu.selectorLabels" . | nindent 6 }} - template: - metadata: - labels: - app.kubernetes.io/component: hami-device-plugin - hami.io/webhook: ignore - {{- include "hami-vgpu.selectorLabels" . | nindent 8 }} - annotations: - {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} - checksum/hami-scheduler-newversion-config: {{ include (print $.Template.BasePath "/scheduler/configmapnew.yaml") . | sha256sum }} - {{- else }} - checksum/hami-scheduler-config: {{ include (print $.Template.BasePath "/scheduler/configmap.yaml") . | sha256sum }} - {{- end }} - checksum/hami-scheduler-device-config: {{ include (print $.Template.BasePath "/scheduler/device-configmap.yaml") . | sha256sum }} - {{- if .Values.devicePlugin.podAnnotations }} - {{- toYaml .Values.devicePlugin.podAnnotations | nindent 8 }} - {{- end }} - spec: - {{- if .Values.devicePlugin.runtimeClassName }} - runtimeClassName: {{ .Values.devicePlugin.runtimeClassName }} - {{- end }} - serviceAccountName: {{ include "hami-vgpu.device-plugin" . }} - priorityClassName: system-node-critical - hostPID: true - hostNetwork: true - {{- include "hami.devicePlugin.imagePullSecrets" . | nindent 6 }} - {{- if .Values.devicePlugin.gpuOperatorToolkitReady.enabled }} - initContainers: - - name: toolkit-validation - image: {{ include "hami.devicePlugin.image" . }} - imagePullPolicy: {{ .Values.devicePlugin.image.pullPolicy }} - securityContext: - privileged: true - runAsUser: 0 - command: ["sh", "-c"] - args: - - | - echo "Waiting for NVIDIA Toolkit to be ready..." - until [ -f {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath }}/toolkit-ready ]; do - echo "Waiting for {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath }}/toolkit-ready..." - sleep 5 - done - echo "NVIDIA Toolkit is ready!" - volumeMounts: - - name: nvidia-validations - mountPath: {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath | quote }} - mountPropagation: HostToContainer - readOnly: true - {{- end }} - containers: - - name: device-plugin - image: {{ include "hami.devicePlugin.image" . }} - imagePullPolicy: {{ .Values.devicePlugin.image.pullPolicy }} - lifecycle: - postStart: - exec: - command: ["/bin/sh","-c", {{ printf "/k8s-vgpu/bin/vgpu-init.sh %s/vgpu/" .Values.global.gpuHookPath | quote }}] - command: - - nvidia-device-plugin - - --config-file=/device-config.yaml - - --mig-strategy={{ .Values.devicePlugin.migStrategy }} - - --disable-core-limit={{ .Values.devicePlugin.disablecorelimit }} - {{- range .Values.devicePlugin.extraArgs }} - - {{ . }} - {{- end }} - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: NVIDIA_MIG_MONITOR_DEVICES - value: all - - name: DEVICE_LIST_STRATEGY - value: {{ .Values.devicePlugin.deviceListStrategy }} - - name: HOOK_PATH - value: {{ .Values.global.gpuHookPath }} - {{- if typeIs "bool" .Values.devicePlugin.passDeviceSpecsEnabled }} - - name: PASS_DEVICE_SPECS - value: {{ .Values.devicePlugin.passDeviceSpecsEnabled | quote }} - {{- end }} - {{- if typeIs "string" .Values.devicePlugin.nvidiaDriverRoot }} - - name: NVIDIA_DRIVER_ROOT - value: {{ .Values.devicePlugin.nvidiaDriverRoot }} - {{- end }} - {{- if typeIs "string" .Values.devicePlugin.nvidiaHookPath }} - - name: NVIDIA_CDI_HOOK_PATH - value: {{ .Values.devicePlugin.nvidiaHookPath }} - {{- end }} - {{- if typeIs "bool" .Values.devicePlugin.gdrcopyEnabled }} - - name: GDRCOPY_ENABLED - value: {{ .Values.devicePlugin.gdrcopyEnabled | quote }} - {{- end }} - {{- if typeIs "bool" .Values.devicePlugin.gdsEnabled }} - - name: GDS_ENABLED - value: {{ .Values.devicePlugin.gdsEnabled | quote }} - {{- end }} - {{- if typeIs "bool" .Values.devicePlugin.mofedEnabled }} - - name: MOFED_ENABLED - value: {{ .Values.devicePlugin.mofedEnabled | quote }} - {{- end }} - {{- if eq (.Values.scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy | default "spread") "topology-aware" }} - - name: ENABLE_TOPOLOGY_SCORE - value: "true" - {{- end }} - {{- with .Values.devicePlugin.extraEnvs }} - {{- . | toYaml | nindent 12 }} - {{- end }} - securityContext: - privileged: true - allowPrivilegeEscalation: true - capabilities: - drop: ["ALL"] - add: ["SYS_ADMIN"] - resources: - {{- toYaml .Values.devicePlugin.resources | nindent 12 }} - volumeMounts: - - name: device-plugin - mountPath: /var/lib/kubelet/device-plugins - - name: lib - mountPath: {{ printf "%s%s" .Values.global.gpuHookPath "/vgpu" }} - - name: usrbin - mountPath: /usrbin - - name: deviceconfig - mountPath: /config - - name: hosttmp - mountPath: /tmp - - name: device-config - mountPath: /device-config.yaml - subPath: device-config.yaml - - name: cdi-root - mountPath: /var/run/cdi - {{- if typeIs "string" .Values.devicePlugin.nvidiaDriverRoot }} - # We always mount the driver root at /driver-root in the container. - # This is required for CDI detection to work correctly. - - name: driver-root - mountPath: /driver-root - readOnly: true - {{- end }} - - name: vgpu-monitor - image: {{ include "hami.devicePlugin.monitor.image" . }} - imagePullPolicy: {{ .Values.devicePlugin.monitor.image.pullPolicy }} - command: - - "vGPUmonitor" - {{- range .Values.devicePlugin.monitor.extraArgs }} - - {{ . }} - {{- end }} - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: ["ALL"] - add: ["SYS_ADMIN"] - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: NVIDIA_VISIBLE_DEVICES - value: "all" - - name: NVIDIA_MIG_MONITOR_DEVICES - value: "all" - - name: HOOK_PATH - value: "{{ .Values.global.gpuHookPath }}/vgpu" - - name: HAMI_RESYNC_INTERVAL - value: {{ .Values.devicePlugin.monitor.resyncInterval | default "5m" | quote }} - {{- with .Values.devicePlugin.monitor.extraEnvs }} - {{- . | toYaml | nindent 12 }} - {{- end }} - resources: - {{- toYaml .Values.devicePlugin.monitor.resources | nindent 12 }} - volumeMounts: - - name: ctrs - mountPath: {{ .Values.devicePlugin.monitor.ctrPath }} - - name: dockers - mountPath: /run/docker - - name: containerds - mountPath: /run/containerd - - name: sysinfo - mountPath: /sysinfo - - name: hostvar - mountPath: /hostvar - - name: hosttmp - mountPath: /tmp - volumes: - - name: ctrs - hostPath: - path: {{ .Values.devicePlugin.monitor.ctrPath }} - - name: hosttmp - hostPath: - path: /tmp - - name: dockers - hostPath: - path: /run/docker - - name: containerds - hostPath: - path: /run/containerd - - name: device-plugin - hostPath: - path: {{ .Values.devicePlugin.pluginPath }} - - name: lib - hostPath: - path: {{ .Values.devicePlugin.libPath }} - {{- if .Values.devicePlugin.gpuOperatorToolkitReady.enabled }} - - name: nvidia-validations - hostPath: - path: {{ .Values.devicePlugin.gpuOperatorToolkitReady.hostPath }} - type: DirectoryOrCreate - {{- end }} - {{- if typeIs "string" .Values.devicePlugin.nvidiaDriverRoot }} - - name: driver-root - hostPath: - path: {{ .Values.devicePlugin.nvidiaDriverRoot }} - type: Directory - {{- end }} - - name: cdi-root - hostPath: - path: /var/run/cdi - type: DirectoryOrCreate - - name: usrbin - hostPath: - path: /usr/bin - - name: sysinfo - hostPath: - path: /sys - - name: hostvar - hostPath: - path: /var - - name: deviceconfig - configMap: - name: {{ .Values.devicePlugin.nodeConfiguration.externalConfigName | default (include "hami-vgpu.device-plugin" .) }} - - name: device-config - configMap: - name: {{ include "hami-vgpu.scheduler" . }}-device - {{- if .Values.devicePlugin.nvidiaNodeSelector }} - nodeSelector: {{ toYaml .Values.devicePlugin.nvidiaNodeSelector | nindent 8 }} - {{- end }} - {{- if .Values.devicePlugin.tolerations }} - tolerations: {{ toYaml .Values.devicePlugin.tolerations | nindent 8 }} - {{- end }} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml deleted file mode 100644 index 6ac757f2..00000000 --- a/packages/system/hami/charts/hami/templates/device-plugin/monitorrole.yaml +++ /dev/null @@ -1,28 +0,0 @@ -{{- if .Values.devicePlugin.enabled -}} -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "hami-vgpu.device-plugin" . }}-monitor -rules: - - apiGroups: - - "" - resources: - - pods - verbs: - - get - - create - - watch - - list - - update - - patch - - apiGroups: - - "" - resources: - - nodes - verbs: - - get - - update - - list - - patch -{{- end -}} - diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml deleted file mode 100644 index 2f0a14ba..00000000 --- a/packages/system/hami/charts/hami/templates/device-plugin/monitorrolebinding.yaml +++ /dev/null @@ -1,17 +0,0 @@ -{{- if .Values.devicePlugin.enabled -}} -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "hami-vgpu.device-plugin" . }} - labels: - app.kubernetes.io/component: "hami-device-plugin" - {{- include "hami-vgpu.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ include "hami-vgpu.device-plugin" . }}-monitor -subjects: - - kind: ServiceAccount - name: {{ include "hami-vgpu.device-plugin" . }} - namespace: {{ include "hami-vgpu.namespace" . }} -{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml deleted file mode 100644 index 88f1b214..00000000 --- a/packages/system/hami/charts/hami/templates/device-plugin/monitorservice.yaml +++ /dev/null @@ -1,29 +0,0 @@ -{{- if .Values.devicePlugin.enabled -}} -apiVersion: v1 -kind: Service -metadata: - name: {{ include "hami-vgpu.device-plugin" . }}-monitor - namespace: {{ include "hami-vgpu.namespace" . }} - labels: - app.kubernetes.io/component: hami-device-plugin - {{- include "hami-vgpu.labels" . | nindent 4 }} - {{- with .Values.devicePlugin.service.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} - {{- if .Values.devicePlugin.service.annotations }} # Use devicePlugin instead of scheduler - annotations: {{ toYaml .Values.devicePlugin.service.annotations | nindent 4 }} - {{- end }} -spec: - type: {{ .Values.devicePlugin.service.type | default "NodePort" }} # Default type is NodePort - ports: - - name: monitorport - port: {{ .Values.devicePlugin.service.httpPort | default 31992 }} # Default HTTP port is 31992 - targetPort: 9394 - {{- if eq (.Values.devicePlugin.service.type | default "NodePort") "NodePort" }} # If type is NodePort, set nodePort - nodePort: {{ .Values.devicePlugin.service.httpPort | default 31992 }} - {{- end }} - protocol: TCP - selector: - app.kubernetes.io/component: hami-device-plugin - {{- include "hami-vgpu.selectorLabels" . | nindent 4 }} -{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml b/packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml deleted file mode 100644 index 6e3c2a44..00000000 --- a/packages/system/hami/charts/hami/templates/device-plugin/monitorserviceaccount.yaml +++ /dev/null @@ -1,10 +0,0 @@ -{{- if .Values.devicePlugin.enabled -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "hami-vgpu.device-plugin" . }} - namespace: {{ include "hami-vgpu.namespace" . }} - labels: - app.kubernetes.io/component: "hami-device-plugin" - {{- include "hami-vgpu.labels" . | nindent 4 }} -{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml b/packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml deleted file mode 100644 index ebcc2c9c..00000000 --- a/packages/system/hami/charts/hami/templates/device-plugin/runtime-class.yaml +++ /dev/null @@ -1,9 +0,0 @@ -{{- if and .Values.devicePlugin.enabled .Values.devicePlugin.createRuntimeClass .Values.devicePlugin.runtimeClassName -}} -apiVersion: node.k8s.io/v1 -kind: RuntimeClass -metadata: - name: {{ .Values.devicePlugin.runtimeClassName }} - annotations: - helm.sh/hook: pre-install,pre-upgrade -handler: nvidia -{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml b/packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml deleted file mode 100644 index e6d28721..00000000 --- a/packages/system/hami/charts/hami/templates/scheduler/certmanager.yaml +++ /dev/null @@ -1,31 +0,0 @@ -{{- if .Values.scheduler.admissionWebhook.enabled -}} -{{- if .Values.scheduler.certManager.enabled }} -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: {{ include "hami-vgpu.scheduler" . }}-serving-cert - namespace: {{ include "hami-vgpu.namespace" . }} - labels: - app.kubernetes.io/component: hami-scheduler - {{- include "hami-vgpu.labels" . | nindent 4 }} -spec: - dnsNames: - - {{ include "hami-vgpu.scheduler" . }}.{{ include "hami-vgpu.namespace" . }}.svc - - {{ include "hami-vgpu.scheduler" . }}.{{ include "hami-vgpu.namespace" . }}.svc.cluster.local - issuerRef: - kind: Issuer - name: {{ include "hami-vgpu.scheduler" . }}-selfsigned-issuer - secretName: {{ include "hami-vgpu.scheduler.tls" . }} ---- -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - name: {{ include "hami-vgpu.scheduler" . }}-selfsigned-issuer - namespace: {{ include "hami-vgpu.namespace" . }} - labels: - app.kubernetes.io/component: hami-scheduler - {{- include "hami-vgpu.labels" . | nindent 4 }} -spec: - selfSigned: {} -{{- end }} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml b/packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml deleted file mode 100644 index 81c4fddf..00000000 --- a/packages/system/hami/charts/hami/templates/scheduler/clusterrole.yaml +++ /dev/null @@ -1,34 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "hami-vgpu.scheduler" . }} - labels: - app.kubernetes.io/component: "hami-scheduler" - {{- include "hami-vgpu.labels" . | nindent 4 }} -rules: - - apiGroups: [""] - resources: ["pods", "configmaps"] - verbs: ["get", "list", "watch", "patch"] - - apiGroups: [""] - resources: ["pods/binding"] - verbs: ["create"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "patch", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["create", "get", "list"] - - apiGroups: [""] - resources: ["resourcequotas"] - verbs: ["get", "list", "watch"] ---- -{{- if .Values.mockDevicePlugin.enabled }} -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "hami-vgpu.mock-device-plugin" . }} -rules: - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "update", "list", "patch"] -{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml deleted file mode 100644 index a81d425c..00000000 --- a/packages/system/hami/charts/hami/templates/scheduler/clusterrolebinding.yaml +++ /dev/null @@ -1,47 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "hami-vgpu.scheduler" . }}-kube - labels: - app.kubernetes.io/component: "hami-scheduler" - {{- include "hami-vgpu.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:kube-scheduler -subjects: - - kind: ServiceAccount - name: {{ include "hami-vgpu.scheduler" . }} - namespace: {{ include "hami-vgpu.namespace" . }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "hami-vgpu.scheduler" . }}-volume - labels: - app.kubernetes.io/component: "hami-scheduler" - {{- include "hami-vgpu.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:volume-scheduler -subjects: - - kind: ServiceAccount - name: {{ include "hami-vgpu.scheduler" . }} - namespace: {{ include "hami-vgpu.namespace" . }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "hami-vgpu.scheduler" . }} - labels: - app.kubernetes.io/component: "hami-scheduler" - {{- include "hami-vgpu.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ include "hami-vgpu.scheduler" . }} -subjects: - - kind: ServiceAccount - name: {{ include "hami-vgpu.scheduler" . }} - namespace: {{ include "hami-vgpu.namespace" . }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/configmap.yaml b/packages/system/hami/charts/hami/templates/scheduler/configmap.yaml deleted file mode 100644 index 109ecbce..00000000 --- a/packages/system/hami/charts/hami/templates/scheduler/configmap.yaml +++ /dev/null @@ -1,142 +0,0 @@ -{{- if .Values.scheduler.kubeScheduler.enabled -}} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "hami-vgpu.scheduler" . }} - namespace: {{ include "hami-vgpu.namespace" . }} - labels: - app.kubernetes.io/component: hami-scheduler - {{- include "hami-vgpu.labels" . | nindent 4 }} -data: - config.json: | - { - "kind": "Policy", - "apiVersion": "v1", - "extenders": [ - { - {{- if .Values.scheduler.admissionWebhook.enabled }} - "urlPrefix": "https://127.0.0.1:443", - "enableHttps": true, - "tlsConfig": { - "insecure": true - }, - {{- else }} - "urlPrefix": "http://127.0.0.1:80", - "enableHttps": false, - {{- end }} - "filterVerb": "filter", - "bindVerb": "bind", - "weight": 1, - "nodeCacheCapable": true, - "httpTimeout": 30000000000, - "managedResources": [ - {{- range .Values.devices.amd.customresources }} - { - "name": "{{ . }}", - "ignoredByScheduler": true - }, - {{- end }} - {{- if .Values.devices.ascend.enabled }} - {{- range .Values.devices.ascend.customresources }} - { - "name": "{{ . }}", - "ignoredByScheduler": true - }, - {{- end }} - {{- end }} - {{- if .Values.devices.mthreads.enabled }} - {{- range .Values.devices.mthreads.customresources }} - { - "name": "{{ . }}", - "ignoredByScheduler": true - }, - {{- end }} - {{- end }} - {{- if .Values.devices.enflame.enabled }} - {{- range .Values.devices.enflame.customresources }} - { - "name": "{{ . }}", - "ignoredByScheduler": true - }, - {{- end }} - {{- end }} - {{- if .Values.devices.kunlun.enabled }} - {{- range .Values.devices.kunlun.customresources }} - { - "name": "{{ . }}", - "ignoredByScheduler": true - }, - {{- end }} - {{- end }} - {{- range .Values.devices.awsneuron.customresources }} - { - "name": "{{ . }}", - "ignoredByScheduler": true - }, - {{- end }} - {{- if .Values.devices.iluvatar.enabled }} - {{- range .Values.devices.iluvatar.customresources }} - { - "name": "{{ . }}", - "ignoredByScheduler": true - }, - {{- end }} - {{- end }} - { - "name": "{{ .Values.resourceName }}", - "ignoredByScheduler": true - }, - { - "name": "{{ .Values.resourceMem }}", - "ignoredByScheduler": true - }, - { - "name": "{{ .Values.resourceCores }}", - "ignoredByScheduler": true - }, - { - "name": "{{ .Values.resourceMemPercentage }}", - "ignoredByScheduler": true - }, - { - "name": "{{ .Values.resourcePriority }}", - "ignoredByScheduler": true - }, - { - "name": "{{ .Values.mluResourceName }}", - "ignoredByScheduler": true - }, - { - "name": "{{ .Values.dcuResourceName }}", - "ignoredByScheduler": true - }, - { - "name": "{{ .Values.dcuResourceMem }}", - "ignoredByScheduler": true - }, - { - "name": "{{ .Values.dcuResourceCores }}", - "ignoredByScheduler": true - }, - { - "name": "metax-tech.com/gpu", - "ignoredByScheduler": true - }, - { - "name": "{{ .Values.metaxResourceName }}", - "ignoredByScheduler": true - }, - { - "name": "{{ .Values.metaxResourceCore }}", - "ignoredByScheduler": true - }, - { - "name": "{{ .Values.metaxResourceMem }}", - "ignoredByScheduler": true - } - ], - "ignoreable": false - } - ] - } -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml b/packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml deleted file mode 100644 index 6f6db097..00000000 --- a/packages/system/hami/charts/hami/templates/scheduler/configmapnew.yaml +++ /dev/null @@ -1,102 +0,0 @@ -{{- if .Values.scheduler.kubeScheduler.enabled -}} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "hami-vgpu.scheduler" . }}-newversion - namespace: {{ include "hami-vgpu.namespace" . }} - labels: - app.kubernetes.io/component: hami-scheduler - {{- include "hami-vgpu.labels" . | nindent 4 }} -data: - config.yaml: | - {{- if gt (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 25}} - apiVersion: kubescheduler.config.k8s.io/v1 - {{- else }} - apiVersion: kubescheduler.config.k8s.io/v1beta2 - {{- end }} - kind: KubeSchedulerConfiguration - leaderElection: - leaderElect: false - profiles: - - schedulerName: {{ .Values.schedulerName }} - extenders: - {{- if .Values.scheduler.admissionWebhook.enabled }} - - urlPrefix: "https://127.0.0.1:443" - enableHTTPS: true - tlsConfig: - insecure: true - {{- else }} - - urlPrefix: "http://127.0.0.1:80" - enableHTTPS: false - {{- end }} - filterVerb: filter - bindVerb: bind - nodeCacheCapable: true - weight: 1 - httpTimeout: 30s - managedResources: - - name: {{ .Values.resourceName }} - ignoredByScheduler: true - - name: {{ .Values.resourceMem }} - ignoredByScheduler: true - - name: {{ .Values.resourceCores }} - ignoredByScheduler: true - - name: {{ .Values.resourceMemPercentage }} - ignoredByScheduler: true - - name: {{ .Values.resourcePriority }} - ignoredByScheduler: true - - name: {{ .Values.mluResourceName }} - ignoredByScheduler: true - - name: {{ .Values.dcuResourceName }} - ignoredByScheduler: true - - name: {{ .Values.dcuResourceMem }} - ignoredByScheduler: true - - name: {{ .Values.dcuResourceCores }} - ignoredByScheduler: true - - name: "metax-tech.com/gpu" - ignoredByScheduler: true - - name: {{ .Values.metaxResourceName }} - ignoredByScheduler: true - - name: {{ .Values.metaxResourceCore }} - ignoredByScheduler: true - - name: {{ .Values.metaxResourceMem }} - ignoredByScheduler: true - {{- if .Values.devices.ascend.enabled }} - {{- range .Values.devices.ascend.customresources }} - - name: {{ . }} - ignoredByScheduler: true - {{- end }} - {{- end }} - {{- if .Values.devices.mthreads.enabled }} - {{- range .Values.devices.mthreads.customresources }} - - name: {{ . }} - ignoredByScheduler: true - {{- end }} - {{- end }} - {{- if .Values.devices.enflame.enabled }} - {{- range .Values.devices.enflame.customresources }} - - name: {{ . }} - ignoredByScheduler: true - {{- end }} - {{- end }} - {{- if .Values.devices.kunlun.enabled }} - {{- range .Values.devices.kunlun.customresources }} - - name: {{ . }} - ignoredByScheduler: true - {{- end }} - {{- end }} - {{- range .Values.devices.awsneuron.customresources }} - - name: {{ . }} - ignoredByScheduler: true - {{- end }} - {{- if .Values.devices.iluvatar.enabled }} - {{- range .Values.devices.iluvatar.customresources }} - - name: {{ . }} - ignoredByScheduler: true - {{- end }} - {{- end }} - {{- range .Values.devices.amd.customresources }} - - name: {{ . }} - ignoredByScheduler: true - {{- end }} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml b/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml deleted file mode 100644 index 1d89e189..00000000 --- a/packages/system/hami/charts/hami/templates/scheduler/deployment.yaml +++ /dev/null @@ -1,225 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "hami-vgpu.scheduler" . }} - namespace: {{ include "hami-vgpu.namespace" . }} - labels: - app.kubernetes.io/component: hami-scheduler - {{- include "hami-vgpu.labels" . | nindent 4 }} - {{- with .Values.global.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} - {{- if .Values.global.annotations }} - annotations: {{ toYaml .Values.global.annotations | nindent 4}} - {{- end }} -spec: - {{- if .Values.scheduler.leaderElect }} - replicas: {{ .Values.scheduler.replicas }} - {{- else }} - replicas: 1 - {{- end }} - selector: - matchLabels: - app.kubernetes.io/component: hami-scheduler - {{- include "hami-vgpu.selectorLabels" . | nindent 6 }} - template: - metadata: - labels: - app.kubernetes.io/component: hami-scheduler - {{- include "hami-vgpu.selectorLabels" . | nindent 8 }} - hami.io/webhook: ignore - annotations: - {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} - checksum/hami-scheduler-newversion-config: {{ include (print $.Template.BasePath "/scheduler/configmapnew.yaml") . | sha256sum }} - {{- else }} - checksum/hami-scheduler-config: {{ include (print $.Template.BasePath "/scheduler/configmap.yaml") . | sha256sum }} - {{- end }} - checksum/hami-scheduler-device-config: {{ include (print $.Template.BasePath "/scheduler/device-configmap.yaml") . | sha256sum }} - {{- if .Values.scheduler.podAnnotations }} - {{- toYaml .Values.scheduler.podAnnotations | nindent 8 }} - {{- end }} - spec: - serviceAccountName: {{ include "hami-vgpu.scheduler" . }} - priorityClassName: system-node-critical - {{- include "hami.scheduler.extender.imagePullSecrets" . | nindent 6 }} - containers: - {{- if .Values.scheduler.kubeScheduler.enabled }} - - name: kube-scheduler - image: {{ include "hami.scheduler.kubeScheduler.image" . }} - imagePullPolicy: {{ .Values.scheduler.kubeScheduler.image.pullPolicy }} - command: - - kube-scheduler - {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} - {{- range .Values.scheduler.kubeScheduler.extraNewArgs }} - - {{ . }} - {{- end }} - {{- else }} - - --scheduler-name={{ .Values.schedulerName }} - {{- range .Values.scheduler.kubeScheduler.extraArgs }} - - {{ . }} - {{- end }} - {{- end }} - - --leader-elect={{ .Values.scheduler.leaderElect }} - - --leader-elect-resource-name={{ .Values.schedulerName }} - - --leader-elect-resource-namespace={{ include "hami-vgpu.namespace" . }} - resources: - {{- toYaml .Values.scheduler.kubeScheduler.resources | nindent 12 }} - volumeMounts: - - name: scheduler-config - mountPath: /config - {{- end }} - {{- if .Values.scheduler.livenessProbe }} - livenessProbe: - failureThreshold: 8 - httpGet: - path: /healthz - port: 10259 - scheme: HTTPS - initialDelaySeconds: 10 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 15 - {{- end }} - - name: vgpu-scheduler-extender - image: {{ include "hami.scheduler.extender.image" . }} - imagePullPolicy: {{ .Values.scheduler.extender.image.pullPolicy }} - env: - {{- if .Values.scheduler.nodeLockExpire }} - - name: HAMI_NODELOCK_EXPIRE - value: "{{ .Values.scheduler.nodeLockExpire }}" - {{- end }} - {{- if .Values.global.managedNodeSelectorEnable }} - {{- range $key, $value := .Values.global.managedNodeSelector }} - - name: NODE_SELECTOR_{{ $key | upper | replace "-" "_" }} - value: "{{ $value }}" - {{- end }} - {{- end }} - command: - - scheduler - {{- if .Values.scheduler.admissionWebhook.enabled }} - - --http_bind=0.0.0.0:443 - - --cert_file=/tls/tls.crt - - --key_file=/tls/tls.key - {{- else }} - - --http_bind=0.0.0.0:80 - {{- end }} - - --scheduler-name={{ .Values.schedulerName }} - - --metrics-bind-address={{ .Values.scheduler.metricsBindAddress }} - - --node-scheduler-policy={{ .Values.scheduler.defaultSchedulerPolicy.nodeSchedulerPolicy }} - - --gpu-scheduler-policy={{ .Values.scheduler.defaultSchedulerPolicy.gpuSchedulerPolicy }} - - --force-overwrite-default-scheduler={{ .Values.scheduler.forceOverwriteDefaultScheduler}} - - --device-config-file=/device-config.yaml - - --leader-elect={{ .Values.scheduler.leaderElect }} - - --leader-elect-resource-name={{ .Values.schedulerName }} - - --leader-elect-resource-namespace={{ include "hami-vgpu.namespace" . }} - {{- if .Values.devices.ascend.enabled }} - - --enable-ascend=true - {{- end }} - {{- if .Values.devices.iluvatar.enabled }} - - --enable-iluvatar=true - {{- end }} - {{- if .Values.scheduler.nodeLabelSelector }} - - --node-label-selector={{- $first := true -}} - {{- range $key, $value := .Values.scheduler.nodeLabelSelector -}} - {{- if not $first }},{{ end -}} - {{- $key }}={{ $value -}} - {{- $first = false -}} - {{- end -}} - {{- end }} - {{- range .Values.scheduler.extender.extraArgs }} - - {{ . }} - {{- end }} - ports: - {{- if .Values.scheduler.admissionWebhook.enabled }} - - name: https - containerPort: 443 - protocol: TCP - {{- else }} - - name: http - containerPort: 80 - protocol: TCP - {{- end }} - - name: metrics - containerPort: {{ last (splitList ":" .Values.scheduler.metricsBindAddress) | int }} - protocol: TCP - resources: - {{- toYaml .Values.scheduler.extender.resources | nindent 12 }} - volumeMounts: - - name: device-config - mountPath: /device-config.yaml - subPath: device-config.yaml - {{- if .Values.scheduler.admissionWebhook.enabled }} - - name: tls-config - mountPath: /tls - {{- end }} - {{- if .Values.scheduler.livenessProbe }} - livenessProbe: - httpGet: - path: /healthz - {{- if .Values.scheduler.admissionWebhook.enabled }} - port: https - scheme: HTTPS - {{- else }} - port: http - scheme: HTTP - {{- end }} - initialDelaySeconds: 10 - periodSeconds: 10 - failureThreshold: 3 - timeoutSeconds: 5 - {{- end }} - {{- if .Values.scheduler.leaderElect }} - readinessProbe: - httpGet: - path: /readyz - {{- if .Values.scheduler.admissionWebhook.enabled }} - port: https - scheme: HTTPS - {{- else }} - port: http - scheme: HTTP - {{- end }} - initialDelaySeconds: 10 - periodSeconds: 10 - failureThreshold: 3 - timeoutSeconds: 5 - {{- end }} - {{- if .Values.scheduler.leaderElect }} - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: app.kubernetes.io/component - operator: In - values: - - hami-scheduler - topologyKey: "kubernetes.io/hostname" - {{- end }} - volumes: - {{- if .Values.scheduler.admissionWebhook.enabled }} - - name: tls-config - secret: - secretName: {{ template "hami-vgpu.scheduler.tls" . }} - {{- end }} - {{- if .Values.scheduler.kubeScheduler.enabled }} - - name: scheduler-config - configMap: - {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} - name: {{ template "hami-vgpu.scheduler" . }}-newversion - {{- else }} - name: {{ template "hami-vgpu.scheduler" . }} - {{- end }} - {{- end }} - - name: device-config - configMap: - name: {{ include "hami-vgpu.scheduler" . }}-device - {{- if .Values.scheduler.nodeSelector }} - nodeSelector: {{ toYaml .Values.scheduler.nodeSelector | nindent 8 }} - {{- end }} - {{- if .Values.scheduler.tolerations }} - tolerations: {{ toYaml .Values.scheduler.tolerations | nindent 8 }} - {{- end }} - {{- if .Values.scheduler.nodeName }} - nodeName: {{ .Values.scheduler.nodeName }} - {{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml b/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml deleted file mode 100644 index 873b813b..00000000 --- a/packages/system/hami/charts/hami/templates/scheduler/device-configmap.yaml +++ /dev/null @@ -1,408 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "hami-vgpu.scheduler" . }}-device - namespace: {{ include "hami-vgpu.namespace" . }} - labels: - app.kubernetes.io/component: hami-scheduler - {{- include "hami-vgpu.labels" . | nindent 4 }} -data: - device-config.yaml: |- - {{- if .Files.Glob "files/device-config.yaml" }} - {{- .Files.Get "files/device-config.yaml" | nindent 4}} - {{- else }} - nvidia: - resourceCountName: {{ .Values.resourceName }} - resourceMemoryName: {{ .Values.resourceMem }} - resourceMemoryPercentageName: {{ .Values.resourceMemPercentage }} - resourceCoreName: {{ .Values.resourceCores }} - resourcePriorityName: {{ .Values.resourcePriority }} - overwriteEnv: false - defaultMemory: 0 - defaultCores: 0 - defaultGPUNum: 1 - memoryFactor: 1 - deviceSplitCount: {{ .Values.devicePlugin.deviceSplitCount }} - deviceMemoryScaling: {{ .Values.devicePlugin.deviceMemoryScaling }} - deviceCoreScaling: {{ .Values.devicePlugin.deviceCoreScaling }} - gpuCorePolicy: {{ .Values.devices.nvidia.gpuCorePolicy }} - libCudaLogLevel: {{ .Values.devices.nvidia.libCudaLogLevel }} - runtimeClassName: "{{ .Values.devicePlugin.runtimeClassName }}" - knownMigGeometries: - - models: [ "A30" ] - allowedGeometries: - - - - name: 1g.6gb - core: 25 - memory: 6144 - count: 4 - - - - name: 2g.12gb - core: 50 - memory: 12288 - count: 2 - - - - name: 4g.24gb - core: 100 - memory: 24576 - count: 1 - - models: [ "A100-SXM4-40GB", "A100-40GB-PCIe", "A100-PCIE-40GB"] - allowedGeometries: - - - - name: 1g.5gb - core: 14 - memory: 5120 - count: 7 - - - - name: 1g.5gb - core: 14 - memory: 5120 - count: 1 - - name: 2g.10gb - core: 28 - memory: 10240 - count: 3 - - - - name: 3g.20gb - core: 42 - memory: 20480 - count: 2 - - - - name: 7g.40gb - core: 100 - memory: 40960 - count: 1 - - models: [ "A100-SXM4-80GB", "A100-80GB-PCIe", "A100-PCIE-80GB"] - allowedGeometries: - - - - name: 1g.10gb - core: 14 - memory: 10240 - count: 7 - - - - name: 1g.10gb - core: 14 - memory: 10240 - count: 1 - - name: 2g.20gb - core: 28 - memory: 20480 - count: 3 - - - - name: 3g.40gb - core: 42 - memory: 40960 - count: 2 - - - - name: 7g.79gb - core: 100 - memory: 80896 - count: 1 - - models: [ "H100-PCIE-80GB", "H100-SXM5-80GB"] - allowedGeometries: - - - - name: 1g.10gb - core: 14 - memory: 10240 - count: 7 - - - - name: 1g.10gb - core: 14 - memory: 10240 - count: 1 - - name: 2g.20gb - core: 28 - memory: 20480 - count: 3 - - - - name: 3g.40gb - core: 42 - memory: 40960 - count: 2 - - - - name: 7g.80gb - core: 100 - memory: 81920 - count: 1 - - models: [ "H100-PCIE-94GB", "H100-SXM5-94GB"] - allowedGeometries: - - - - name: 1g.12gb - core: 14 - memory: 12288 - count: 7 - - - - name: 1g.12gb - core: 14 - memory: 12288 - count: 1 - - name: 2g.24gb - core: 28 - memory: 24576 - count: 3 - - - - name: 3g.47gb - core: 42 - memory: 48128 - count: 2 - - - - name: 7g.94gb - core: 100 - memory: 96256 - count: 1 - - models: [ "H20", "H100 on GH200"] - allowedGeometries: - - - - name: 1g.12gb - core: 14 - memory: 12288 - count: 7 - - - - name: 1g.12gb - core: 14 - memory: 12288 - count: 1 - - name: 2g.24gb - core: 28 - memory: 24576 - count: 3 - - - - name: 3g.48gb - core: 42 - memory: 49152 - count: 2 - - - - name: 7g.96gb - core: 100 - memory: 98304 - count: 1 - - models: [ "H200 NVL", "H200-SXM5"] - allowedGeometries: - - - - name: 1g.18gb - core: 14 - memory: 18432 - count: 7 - - - - name: 1g.18gb - core: 14 - memory: 18432 - count: 1 - - name: 2g.35gb - core: 28 - memory: 35840 - count: 3 - - - - name: 3g.71gb - core: 42 - memory: 72704 - count: 2 - - - - name: 7g.141gb - core: 100 - memory: 144384 - count: 1 - - models: [ "B200" ] - allowedGeometries: - - - - name: 1g.23gb - core: 14 - memory: 23552 - count: 7 - - - - name: 1g.23gb - core: 14 - memory: 23552 - count: 1 - - name: 2g.45gb - core: 28 - memory: 46080 - count: 3 - - - - name: 3g.90gb - core: 42 - memory: 92160 - count: 2 - - - - name: 7g.180gb - core: 100 - memory: 184320 - count: 1 - cambricon: - resourceCountName: {{ .Values.mluResourceName }} - resourceMemoryName: {{ .Values.mluResourceMem }} - resourceCoreName: {{ .Values.mluResourceCores }} - hygon: - resourceCountName: {{ .Values.dcuResourceName }} - resourceMemoryName: {{ .Values.dcuResourceMem }} - resourceCoreName: {{ .Values.dcuResourceCores }} - memoryFactor: 1 - metax: - resourceCountName: "metax-tech.com/gpu" - resourceVCountName: {{ .Values.metaxResourceName }} - resourceVMemoryName: {{ .Values.metaxResourceMem }} - resourceVCoreName: {{ .Values.metaxResourceCore }} - sgpuTopologyAware: {{ .Values.metaxsGPUTopologyAware }} - enflame: - resourceNameGCU: "enflame.com/gcu" - resourceNameVGCU: {{ .Values.enflameResourceNameVGCU }} - resourceNameVGCUPercentage: {{ .Values.enflameResourceNameVGCUPercentage }} - mthreads: - resourceCountName: "mthreads.com/vgpu" - resourceMemoryName: "mthreads.com/sgpu-memory" - resourceCoreName: "mthreads.com/sgpu-core" - iluvatars: - - chipName: MR-V100 - commonWord: MR-V100 - resourceCountName: iluvatar.ai/MR-V100-vgpu - resourceMemoryName: iluvatar.ai/MR-V100.vMem - resourceCoreName: iluvatar.ai/MR-V100.vCore - - chipName: MR-V50 - commonWord: MR-V50 - resourceCountName: iluvatar.ai/MR-V50-vgpu - resourceMemoryName: iluvatar.ai/MR-V50.vMem - resourceCoreName: iluvatar.ai/MR-V50.vCore - - chipName: BI-V150 - commonWord: BI-V150 - resourceCountName: iluvatar.ai/BI-V150-vgpu - resourceMemoryName: iluvatar.ai/BI-V150.vMem - resourceCoreName: iluvatar.ai/BI-V150.vCore - - chipName: BI-V100 - commonWord: BI-V100 - resourceCountName: iluvatar.ai/BI-V100-vgpu - resourceMemoryName: iluvatar.ai/BI-V100.vMem - resourceCoreName: iluvatar.ai/BI-V100.vCore - kunlun: - resourceCountName: {{ .Values.kunlunResourceName }} - resourceVCountName: {{ .Values.kunlunResourceVCountName }} - resourceVMemoryName: {{ .Values.kunlunResourceVMemoryName }} - awsneuron: - resourceCountName: "aws.amazon.com/neuron" - resourceCoreName: "aws.amazon.com/neuroncore" - amd: - resourceCountName: "amd.com/gpu" - vnpus: - - chipName: 910A - commonWord: Ascend910A - resourceName: huawei.com/Ascend910A - resourceMemoryName: huawei.com/Ascend910A-memory - memoryAllocatable: 32768 - memoryCapacity: 32768 - memoryFactor: 1 - aiCore: 30 - templates: - - name: vir02 - memory: 2184 - aiCore: 2 - - name: vir04 - memory: 4369 - aiCore: 4 - - name: vir08 - memory: 8738 - aiCore: 8 - - name: vir16 - memory: 17476 - aiCore: 16 - - chipName: 910B2 - commonWord: Ascend910B2 - resourceName: huawei.com/Ascend910B2 - resourceMemoryName: huawei.com/Ascend910B2-memory - memoryAllocatable: 65536 - memoryCapacity: 65536 - memoryFactor: 1 - aiCore: 24 - aiCPU: 6 - templates: - - name: vir03_1c_8g - memory: 8192 - aiCore: 3 - aiCPU: 1 - - name: vir06_1c_16g - memory: 16384 - aiCore: 6 - aiCPU: 1 - - name: vir12_3c_32g - memory: 32768 - aiCore: 12 - aiCPU: 3 - - chipName: 910B3 - commonWord: Ascend910B3 - resourceName: huawei.com/Ascend910B3 - resourceMemoryName: huawei.com/Ascend910B3-memory - memoryAllocatable: 65536 - memoryCapacity: 65536 - memoryFactor: 1 - aiCore: 20 - aiCPU: 7 - templates: - - name: vir05_1c_16g - memory: 16384 - aiCore: 5 - aiCPU: 1 - - name: vir10_3c_32g - memory: 32768 - aiCore: 10 - aiCPU: 3 - - chipName: 910B4-1 - commonWord: Ascend910B4-1 - resourceName: huawei.com/Ascend910B4-1 - resourceMemoryName: huawei.com/Ascend910B4-1-memory - memoryAllocatable: 65536 - memoryCapacity: 65536 - memoryFactor: 1 - aiCore: 20 - aiCPU: 7 - templates: - # NOTE: Names of vnpu template for 910B4-1 are fixed by Ascend runtime and must not be changed. - # The memory is used for scheduling so the correct values must be set. - # Template vir05_1c_8g actually provides 16GB memory, - - name: vir05_1c_8g - memory: 16384 - aiCore: 5 - aiCPU: 1 - # Template vir10_3c_16g actually provides 32GB memory - - name: vir10_3c_16g - memory: 32768 - aiCore: 10 - aiCPU: 3 - - chipName: 910B4 - commonWord: Ascend910B4 - resourceName: huawei.com/Ascend910B4 - resourceMemoryName: huawei.com/Ascend910B4-memory - memoryAllocatable: 32768 - memoryCapacity: 32768 - memoryFactor: 1 - aiCore: 20 - aiCPU: 7 - templates: - - name: vir05_1c_8g - memory: 8192 - aiCore: 5 - aiCPU: 1 - - name: vir10_3c_16g - memory: 16384 - aiCore: 10 - aiCPU: 3 - - chipName: 310P3 - commonWord: Ascend310P - resourceName: huawei.com/Ascend310P - resourceMemoryName: huawei.com/Ascend310P-memory - memoryAllocatable: 21527 - memoryCapacity: 24576 - memoryFactor: 1 - aiCore: 8 - aiCPU: 7 - templates: - - name: vir01 - memory: 3072 - aiCore: 1 - aiCPU: 1 - - name: vir02 - memory: 6144 - aiCore: 2 - aiCPU: 2 - - name: vir04 - memory: 12288 - aiCore: 4 - aiCPU: 4 - {{ end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml deleted file mode 100644 index 77e891cc..00000000 --- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrole.yaml +++ /dev/null @@ -1,21 +0,0 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "hami-vgpu.fullname" . }}-admission - annotations: - "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded - labels: - {{- include "hami-vgpu.labels" . | nindent 4 }} - app.kubernetes.io/component: admission-webhook -rules: - - apiGroups: - - admissionregistration.k8s.io - resources: - #- validatingwebhookconfigurations - - mutatingwebhookconfigurations - verbs: - - get - - update -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml deleted file mode 100644 index 2b82f926..00000000 --- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/clusterrolebinding.yaml +++ /dev/null @@ -1,20 +0,0 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "hami-vgpu.fullname" . }}-admission - annotations: - "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded - labels: - {{- include "hami-vgpu.labels" . | nindent 4 }} - app.kubernetes.io/component: admission-webhook -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ include "hami-vgpu.fullname" . }}-admission -subjects: - - kind: ServiceAccount - name: {{ include "hami-vgpu.fullname" . }}-admission - namespace: {{ include "hami-vgpu.namespace" . }} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml deleted file mode 100644 index 0e36d95f..00000000 --- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-createSecret.yaml +++ /dev/null @@ -1,68 +0,0 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} -apiVersion: batch/v1 -kind: Job -metadata: - name: {{ include "hami-vgpu.fullname" . }}-admission-create - namespace: {{ include "hami-vgpu.namespace" . }} - annotations: - "helm.sh/hook": pre-install,pre-upgrade - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded - labels: - {{- include "hami-vgpu.labels" . | nindent 4 }} - app.kubernetes.io/component: admission-webhook -spec: - {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} - # Alpha feature since k8s 1.12 - ttlSecondsAfterFinished: 0 - {{- end }} - template: - metadata: - name: {{ include "hami-vgpu.fullname" . }}-admission-create - {{- if .Values.scheduler.patch.podAnnotations }} - annotations: {{ toYaml .Values.scheduler.patch.podAnnotations | nindent 8 }} - {{- end }} - labels: - {{- include "hami-vgpu.labels" . | nindent 8 }} - app.kubernetes.io/component: admission-webhook - hami.io/webhook: ignore - spec: - {{- if .Values.scheduler.patch.priorityClassName }} - priorityClassName: {{ .Values.scheduler.patch.priorityClassName }} - {{- end }} - {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} - {{- include "hami.scheduler.patch.new.imagePullSecrets" . | nindent 6 }} - {{- else }} - {{- include "hami.scheduler.patch.imagePullSecrets" . | nindent 6 }} - {{- end }} - containers: - - name: create - {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} - image: {{ include "hami.scheduler.patch.new.image" . }} - imagePullPolicy: {{ .Values.scheduler.patch.imageNew.pullPolicy }} - {{- else }} - image: {{ include "hami.scheduler.patch.image" . }} - imagePullPolicy: {{ .Values.scheduler.patch.image.pullPolicy }} - {{- end }} - args: - - create - - --cert-name=tls.crt - - --key-name=tls.key - {{- if .Values.scheduler.admissionWebhook.customURL.enabled }} - - --host={{ printf "%s.%s.svc,127.0.0.1,%s" (include "hami-vgpu.scheduler" .) (include "hami-vgpu.namespace" .) .Values.scheduler.admissionWebhook.customURL.host}} - {{- else }} - - --host={{ printf "%s.%s.svc,127.0.0.1" (include "hami-vgpu.scheduler" .) (include "hami-vgpu.namespace" .) }} - {{- end }} - - --namespace={{ include "hami-vgpu.namespace" . }} - - --secret-name={{ include "hami-vgpu.scheduler.tls" . }} - restartPolicy: OnFailure - serviceAccountName: {{ include "hami-vgpu.fullname" . }}-admission - {{- if .Values.scheduler.patch.nodeSelector }} - nodeSelector: {{ toYaml .Values.scheduler.patch.nodeSelector | nindent 8 }} - {{- end }} - {{- if .Values.scheduler.patch.tolerations }} - tolerations: {{ toYaml .Values.scheduler.patch.tolerations | nindent 8 }} - {{- end }} - securityContext: - runAsNonRoot: true - runAsUser: {{ .Values.scheduler.patch.runAsUser }} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml deleted file mode 100644 index ce52042c..00000000 --- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/job-patchWebhook.yaml +++ /dev/null @@ -1,63 +0,0 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} -apiVersion: batch/v1 -kind: Job -metadata: - name: {{ include "hami-vgpu.fullname" . }}-admission-patch - namespace: {{ include "hami-vgpu.namespace" . }} - annotations: - "helm.sh/hook": post-install,post-upgrade - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded - labels: - {{- include "hami-vgpu.labels" . | nindent 4 }} - app.kubernetes.io/component: admission-webhook -spec: - {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} - # Alpha feature since k8s 1.12 - ttlSecondsAfterFinished: 0 - {{- end }} - template: - metadata: - name: {{ include "hami-vgpu.fullname" . }}-admission-patch - {{- if .Values.scheduler.patch.podAnnotations }} - annotations: {{ toYaml .Values.scheduler.patch.podAnnotations | nindent 8 }} - {{- end }} - labels: - {{- include "hami-vgpu.labels" . | nindent 8 }} - app.kubernetes.io/component: admission-webhook - hami.io/webhook: ignore - spec: - {{- if .Values.scheduler.patch.priorityClassName }} - priorityClassName: {{ .Values.scheduler.patch.priorityClassName }} - {{- end }} - {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} - {{- include "hami.scheduler.patch.new.imagePullSecrets" . | nindent 6 }} - {{- else }} - {{- include "hami.scheduler.patch.imagePullSecrets" . | nindent 6 }} - {{- end }} - containers: - - name: patch - {{- if ge (regexReplaceAll "[^0-9]" .Capabilities.KubeVersion.Minor "" | int) 22 }} - image: {{ include "hami.scheduler.patch.new.image" . }} - imagePullPolicy: {{ .Values.scheduler.patch.imageNew.pullPolicy }} - {{- else }} - image: {{ include "hami.scheduler.patch.image" . }} - imagePullPolicy: {{ .Values.scheduler.patch.image.pullPolicy }} - {{- end }} - args: - - patch - - --webhook-name={{ include "hami-vgpu.scheduler.webhook" . }} - - --namespace={{ include "hami-vgpu.namespace" . }} - - --patch-validating=false - - --secret-name={{ include "hami-vgpu.scheduler.tls" . }} - restartPolicy: OnFailure - serviceAccountName: {{ include "hami-vgpu.fullname" . }}-admission - {{- if .Values.scheduler.patch.nodeSelector }} - nodeSelector: {{ toYaml .Values.scheduler.patch.nodeSelector | nindent 8 }} - {{- end }} - {{- if .Values.scheduler.patch.tolerations }} - tolerations: {{ toYaml .Values.scheduler.patch.tolerations | nindent 8 }} - {{- end }} - securityContext: - runAsNonRoot: true - runAsUser: {{ .Values.scheduler.patch.runAsUser }} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml deleted file mode 100644 index 56682f8e..00000000 --- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/role.yaml +++ /dev/null @@ -1,21 +0,0 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: {{ include "hami-vgpu.fullname" . }}-admission - namespace: {{ include "hami-vgpu.namespace" . }} - annotations: - "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded - labels: - {{- include "hami-vgpu.labels" . | nindent 4 }} - app.kubernetes.io/component: admission-webhook -rules: - - apiGroups: - - "" - resources: - - secrets - verbs: - - get - - create -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml deleted file mode 100644 index 7239b128..00000000 --- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/rolebinding.yaml +++ /dev/null @@ -1,21 +0,0 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) -}} -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ include "hami-vgpu.fullname" . }}-admission - namespace: {{ include "hami-vgpu.namespace" . }} - annotations: - "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded - labels: - {{- include "hami-vgpu.labels" . | nindent 4 }} - app.kubernetes.io/component: admission-webhook -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: {{ include "hami-vgpu.fullname" . }}-admission -subjects: - - kind: ServiceAccount - name: {{ include "hami-vgpu.fullname" . }}-admission - namespace: {{ include "hami-vgpu.namespace" . }} -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml b/packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml deleted file mode 100644 index 857c6a8d..00000000 --- a/packages/system/hami/charts/hami/templates/scheduler/job-patch/serviceaccount.yaml +++ /dev/null @@ -1,13 +0,0 @@ -{{- if and .Values.scheduler.admissionWebhook.enabled (.Values.scheduler.patch.enabled) (not .Values.scheduler.certManager.enabled) }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "hami-vgpu.fullname" . }}-admission - namespace: {{ include "hami-vgpu.namespace" . }} - annotations: - "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded - labels: - {{- include "hami-vgpu.labels" . | nindent 4 }} - app.kubernetes.io/component: admission-webhook -{{- end }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/role.yaml b/packages/system/hami/charts/hami/templates/scheduler/role.yaml deleted file mode 100644 index 5f7d7e44..00000000 --- a/packages/system/hami/charts/hami/templates/scheduler/role.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: {{ include "hami-vgpu.scheduler" . }} - namespace: {{ include "hami-vgpu.namespace" . }} - labels: - app.kubernetes.io/component: "hami-scheduler" - {{- include "hami-vgpu.labels" . | nindent 4 }} -rules: - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["create", "list", "watch", "get", "update", "patch"] diff --git a/packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml b/packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml deleted file mode 100644 index 96e175a1..00000000 --- a/packages/system/hami/charts/hami/templates/scheduler/rolebinding.yaml +++ /dev/null @@ -1,31 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ include "hami-vgpu.scheduler" . }} - namespace: {{ include "hami-vgpu.namespace" . }} - labels: - app.kubernetes.io/component: "hami-scheduler" - {{- include "hami-vgpu.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: {{ include "hami-vgpu.scheduler" . }} -subjects: - - kind: ServiceAccount - name: {{ include "hami-vgpu.scheduler" . }} - namespace: {{ include "hami-vgpu.namespace" . }} ---- -{{- if .Values.mockDevicePlugin.enabled }} -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "hami-vgpu.mock-device-plugin" . }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ include "hami-vgpu.mock-device-plugin" . }} -subjects: - - kind: ServiceAccount - name: {{ include "hami-vgpu.mock-device-plugin" . }} - namespace: {{ include "hami-vgpu.namespace" . }} -{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/scheduler/service.yaml b/packages/system/hami/charts/hami/templates/scheduler/service.yaml deleted file mode 100644 index d7538fed..00000000 --- a/packages/system/hami/charts/hami/templates/scheduler/service.yaml +++ /dev/null @@ -1,34 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ include "hami-vgpu.scheduler" . }} - namespace: {{ include "hami-vgpu.namespace" . }} - labels: - app.kubernetes.io/component: hami-scheduler - {{- include "hami-vgpu.labels" . | nindent 4 }} - {{- if .Values.scheduler.service.labels }} - {{ toYaml .Values.scheduler.service.labels | indent 4 }} - {{- end }} - {{- if .Values.scheduler.service.annotations }} - annotations: {{ toYaml .Values.scheduler.service.annotations | nindent 4 }} - {{- end }} -spec: - type: {{ .Values.scheduler.service.type | default "NodePort" }} # Default type is NodePort - ports: - - name: http - port: {{ .Values.scheduler.service.httpPort | default 443 }} # Default HTTP port is 443 - targetPort: {{ .Values.scheduler.service.httpTargetPort | default 443 }} - {{- if eq (.Values.scheduler.service.type | default "NodePort") "NodePort" }} # If type is NodePort, set nodePort - nodePort: {{ .Values.scheduler.service.schedulerPort | default 31998 }} - {{- end }} - protocol: TCP - - name: monitor - port: {{ .Values.scheduler.service.monitorPort | default 31993 }} # Default monitoring port is 31993 - targetPort: {{ .Values.scheduler.service.monitorTargetPort | default 9395 }} - {{- if eq (.Values.scheduler.service.type | default "NodePort") "NodePort" }} # If type is NodePort, set nodePort - nodePort: {{ .Values.scheduler.service.monitorPort | default 31993 }} - {{- end }} - protocol: TCP - selector: - app.kubernetes.io/component: hami-scheduler - {{- include "hami-vgpu.selectorLabels" . | nindent 4 }} diff --git a/packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml b/packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml deleted file mode 100644 index 0435c003..00000000 --- a/packages/system/hami/charts/hami/templates/scheduler/serviceaccount.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "hami-vgpu.scheduler" . }} - namespace: {{ include "hami-vgpu.namespace" . }} - labels: - app.kubernetes.io/component: "hami-scheduler" - {{- include "hami-vgpu.labels" . | nindent 4 }} ---- -{{- if .Values.mockDevicePlugin.enabled }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "hami-vgpu.mock-device-plugin" . }} - namespace: {{ include "hami-vgpu.namespace" . }} -{{- end -}} diff --git a/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml b/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml deleted file mode 100644 index db9f8029..00000000 --- a/packages/system/hami/charts/hami/templates/scheduler/webhook.yaml +++ /dev/null @@ -1,57 +0,0 @@ -{{- if .Values.scheduler.admissionWebhook.enabled -}} -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - {{- if .Values.scheduler.certManager.enabled }} - annotations: - cert-manager.io/inject-ca-from: {{ include "hami-vgpu.namespace" . }}/{{ include "hami-vgpu.scheduler" . }}-serving-cert - {{- end }} - name: {{ include "hami-vgpu.scheduler.webhook" . }} -webhooks: - - admissionReviewVersions: - - v1beta1 - clientConfig: - {{- if .Values.scheduler.admissionWebhook.customURL.enabled }} - url: https://{{ .Values.scheduler.admissionWebhook.customURL.host}}:{{.Values.scheduler.admissionWebhook.customURL.port}}{{.Values.scheduler.admissionWebhook.customURL.path}} - {{- else }} - service: - name: {{ include "hami-vgpu.scheduler" . }} - namespace: {{ include "hami-vgpu.namespace" . }} - path: /webhook - port: {{ .Values.scheduler.service.httpPort }} - {{- end }} - failurePolicy: {{ .Values.scheduler.admissionWebhook.failurePolicy }} - matchPolicy: Equivalent - name: vgpu.hami.io - namespaceSelector: - matchExpressions: - - key: hami.io/webhook - operator: NotIn - values: - - ignore - {{- if .Values.scheduler.admissionWebhook.whitelistNamespaces }} - - key: kubernetes.io/metadata.name - operator: NotIn - values: - {{- toYaml .Values.scheduler.admissionWebhook.whitelistNamespaces | nindent 10 }} - {{- end }} - objectSelector: - matchExpressions: - - key: hami.io/webhook - operator: NotIn - values: - - ignore - reinvocationPolicy: {{ .Values.scheduler.admissionWebhook.reinvocationPolicy }} - rules: - - apiGroups: - - "" - apiVersions: - - v1 - operations: - - CREATE - resources: - - pods - scope: '*' - sideEffects: None - timeoutSeconds: 10 -{{- end }} diff --git a/packages/system/hami/charts/hami/values.yaml b/packages/system/hami/charts/hami/values.yaml deleted file mode 100644 index 01caca0b..00000000 --- a/packages/system/hami/charts/hami/values.yaml +++ /dev/null @@ -1,455 +0,0 @@ -## @section Global configuration -## @param global.imageRegistry Global Docker image registry -## @param global.imagePullSecrets Global Docker image pull secrets -global: - ## @param global.imageRegistry Global Docker image registry - imageRegistry: "" - ## E.g. - ## imagePullSecrets: - ## - myRegistryKeySecretName - ## @param global.imagePullSecrets Global Docker image pull secrets - imagePullSecrets: [] - imageTag: "v2.8.1" - gpuHookPath: /usr/local - labels: {} - annotations: {} - managedNodeSelectorEnable: false - managedNodeSelector: - usage: "gpu" - -nameOverride: "" -fullnameOverride: "" -namespaceOverride: "" - -#Nvidia GPU Parameters -resourceName: "nvidia.com/gpu" -resourceMem: "nvidia.com/gpumem" -resourceMemPercentage: "nvidia.com/gpumem-percentage" -resourceCores: "nvidia.com/gpucores" -resourcePriority: "nvidia.com/priority" - -#MLU Parameters -mluResourceName: "cambricon.com/vmlu" -mluResourceMem: "cambricon.com/mlu.smlu.vmemory" -mluResourceCores: "cambricon.com/mlu.smlu.vcore" - -#Hygon DCU Parameters -dcuResourceName: "hygon.com/dcunum" -dcuResourceMem: "hygon.com/dcumem" -dcuResourceCores: "hygon.com/dcucores" - -#Metax sGPU Parameters -metaxResourceName: "metax-tech.com/sgpu" -metaxResourceCore: "metax-tech.com/vcore" -metaxResourceMem: "metax-tech.com/vmemory" -metaxsGPUTopologyAware: "false" - -#Enflame VGCU Parameters -enflameResourceNameVGCU: "enflame.com/vgcu" -enflameResourceNameVGCUPercentage: "enflame.com/vgcu-percentage" - -#Kunlun XPU Parameters -kunlunResourceName: "kunlunxin.com/xpu" -kunlunResourceVCountName: "kunlunxin.com/vxpu" -kunlunResourceVMemoryName: "kunlunxin.com/vxpu-memory" - -schedulerName: "hami-scheduler" - -scheduler: - # @param nodeName defines the node name and the nvidia-vgpu-scheduler-scheduler will schedule to the node. - # if we install the nvidia-vgpu-scheduler-scheduler as default scheduler, we need to remove the k8s default - # scheduler pod from the cluster first, we must specify node name to skip the schedule workflow. - nodeName: "" - #nodeLabelSelector: - # "gpu": "on" - overwriteEnv: "false" - defaultSchedulerPolicy: - nodeSchedulerPolicy: binpack - gpuSchedulerPolicy: spread - metricsBindAddress: ":9395" - # If set to false, When Pod.Spec.SchedulerName equals to the const DefaultSchedulerName in k8s.io/api/core/v1 package, webhook will not overwrite it - forceOverwriteDefaultScheduler: true - livenessProbe: false - leaderElect: true - # when leaderElect is true, replicas is available, otherwise replicas is 1. - replicas: 1 - kubeScheduler: - # @param enabled indicate whether to run kube-scheduler container in the scheduler pod, it's true by default. - enabled: true - ## @param image.registry kube scheduler image registry - ## @param image.repository kube scheduler image repository - ## @param image.tag kube scheduler image tag (immutable tags are recommended) - ## @param image.pullPolicy kube scheduler image pull policy - ## @param image.pullSecrets Specify docker-registry secret names as an array - image: - registry: "registry.cn-hangzhou.aliyuncs.com" - repository: "google_containers/kube-scheduler" - tag: "" - ## Specify a imagePullPolicy - ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' - ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images - ## - pullPolicy: IfNotPresent - ## Optionally specify an array of imagePullSecrets. - ## Secrets must be manually created in the namespace. - ## Example: - ## pullSecrets: - ## - myRegistryKeySecretName - ## - pullSecrets: [] - resources: {} - # If you do want to specify resources, uncomment the following lines, adjust them as necessary. - # and remove the curly braces after 'resources:'. -# limits: -# cpu: 1000m -# memory: 1000Mi -# requests: -# cpu: 100m -# memory: 100Mi - extraNewArgs: - - --config=/config/config.yaml - - -v=4 - extraArgs: - - --policy-config-file=/config/config.json - - -v=4 - extender: - ## @param image.registry scheduler extender image registry - ## @param image.repository scheduler extender image repository - ## @param image.tag scheduler extender image tag (immutable tags are recommended) - ## @param image.pullPolicy scheduler extender image pull policy - ## @param image.pullSecrets Specify docker-registry secret names as an array - image: - registry: "docker.io" - repository: "projecthami/hami" - tag: "" - ## Specify a imagePullPolicy - ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' - ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images - ## - pullPolicy: IfNotPresent - ## Optionally specify an array of imagePullSecrets. - ## Secrets must be manually created in the namespace. - ## Example: - ## pullSecrets: - ## - myRegistryKeySecretName - ## - pullSecrets: [] - resources: {} - # If you do want to specify resources, uncomment the following lines, adjust them as necessary, - # and remove the curly braces after 'resources:'. -# limits: -# cpu: 1000m -# memory: 1000Mi -# requests: -# cpu: 100m -# memory: 100Mi - extraArgs: - - --debug - - -v=4 - nodeLockExpire: "5m" - podAnnotations: {} - tolerations: [] - #serviceAccountName: "hami-vgpu-scheduler-sa" - admissionWebhook: - # If set to false, the admission webhook is not installed and any pods that should use HAMi must be - # configured to use the right 'schedulerName' and other device-specific configurations. - enabled: true - customURL: - enabled: false - # must be an endpoint using https. - # should generate host certs here - host: 127.0.0.1 # hostname or ip, can be your node'IP if you want to use https://:/ - port: 31998 - path: /webhook - whitelistNamespaces: - # Specify the namespaces that the webhook will not be applied to. - # - default - # - kube-system - # - istio-system - reinvocationPolicy: Never - failurePolicy: Ignore - ## TLS Certificate Option 1: Use cert-manager to generate self-signed certificate. - ## If enabled, always takes precedence over options 2. - certManager: - enabled: false - ## TLS Certificate Option 2: Use kube-webhook-certgen to generate self-signed certificate. - ## If true and certManager.enabled is false, Helm will automatically create a self-signed cert and secret for you. - patch: - enabled: true - ## @param image.registry scheduler extender image registry - ## @param image.repository scheduler extender image repository - ## @param image.tag scheduler extender image tag (immutable tags are recommended) - ## @param image.pullPolicy scheduler extender image pull policy - ## @param image.pullSecrets Specify docker-registry secret names as an array - image: - registry: "docker.io" - repository: "jettech/kube-webhook-certgen" - tag: "v1.5.2" - ## Specify a imagePullPolicy - ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' - ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images - ## - pullPolicy: IfNotPresent - ## Optionally specify an array of imagePullSecrets. - ## Secrets must be manually created in the namespace. - ## Example: - ## pullSecrets: - ## - myRegistryKeySecretName - ## - pullSecrets: [] - ## @param image.registry scheduler extender image registry - ## @param image.repository scheduler extender image repository - ## @param image.tag scheduler extender image tag (immutable tags are recommended) - ## @param image.pullPolicy scheduler extender image pull policy - ## @param image.pullSecrets Specify docker-registry secret names as an array - imageNew: - registry: "docker.io" - repository: "liangjw/kube-webhook-certgen" - tag: "v1.1.1" - ## Specify a imagePullPolicy - ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' - ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images - ## - pullPolicy: IfNotPresent - ## Optionally specify an array of imagePullSecrets. - ## Secrets must be manually created in the namespace. - ## Example: - ## pullSecrets: - ## - myRegistryKeySecretName - ## - pullSecrets: [] - priorityClassName: "" - podAnnotations: {} - nodeSelector: {} - tolerations: [] - runAsUser: 2000 - service: - type: NodePort # Default type is NodePort, can be changed to ClusterIP - httpPort: 443 # HTTP port - schedulerPort: 31998 # NodePort for HTTP - monitorPort: 31993 # Monitoring port - monitorTargetPort: 9395 - httpTargetPort: 443 - labels: {} - annotations: {} - -devicePlugin: - enabled: true - gpuOperatorToolkitReady: - enabled: false - hostPath: "/run/nvidia/validations" - ## @param image.registry devicePlugin image registry - ## @param image.repository devicePlugin image repository - ## @param image.tag devicePlugin image tag (immutable tags are recommended) - ## @param image.pullPolicy devicePlugin image pull policy - ## @param image.pullSecrets Specify docker-registry secret names as an array - image: - registry: "docker.io" - repository: "projecthami/hami" - tag: "" - ## Specify a imagePullPolicy - ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' - ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images - ## - pullPolicy: IfNotPresent - ## Optionally specify an array of imagePullSecrets. - ## Secrets must be manually created in the namespace. - ## Example: - ## pullSecrets: - ## - myRegistryKeySecretName - ## - pullSecrets: [] - monitor: - ## @param image.registry monitor image registry - ## @param image.repository monitor image repository - ## @param image.tag monitor image tag (immutable tags are recommended) - ## @param image.pullPolicy monitor image pull policy - ## @param image.pullSecrets Specify docker-registry secret names as an array - image: - registry: "docker.io" - repository: "projecthami/hami" - tag: "" - ## Specify a imagePullPolicy - ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' - ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images - ## - pullPolicy: IfNotPresent - ## Optionally specify an array of imagePullSecrets. - ## Secrets must be manually created in the namespace. - ## Example: - ## pullSecrets: - ## - myRegistryKeySecretName - ## - pullSecrets: [] - ctrPath: /usr/local/vgpu/containers - resyncInterval: "5m" - extraArgs: - - -v=4 - extraEnvs: {} - resources: {} - # If you do want to specify resources, uncomment the following lines, adjust them as necessary. - # and remove the curly braces after 'resources:'. -# limits: -# cpu: 1000m -# memory: 1000Mi -# requests: -# cpu: 100m -# memory: 100Mi - - deviceSplitCount: 10 - deviceMemoryScaling: 1 - deviceCoreScaling: 1 - # Node configuration for device plugin, Priority: externalConfigName > config > default config - nodeConfiguration: - # If you want to use a custom config.json, you can set the content here. - # If this is set, it will override the default config.json(An example is as follows). - config: | - { - "nodeconfig": [ - { - "name": "your-node-name", - "operatingmode": "hami-core", - "devicememoryscaling": 1, - "devicesplitcount": 10, - "migstrategy": "none", - "filterdevices": { - "uuid": [], - "index": [] - } - } - ] - } - # If you want to use an existing ConfigMap, you can set the name here. - # If this is set, the chart will not create the ConfigMap and will use the existing one. - externalConfigName: "" - # The runtime class name to be used by the device plugin, and added to the pod.spec.runtimeClassName of applications utilizing NVIDIA GPUs - runtimeClassName: "" - # Whether to create runtime class, name comes from runtimeClassName when it is set - createRuntimeClass: false - migStrategy: "none" - disablecorelimit: "false" - passDeviceSpecsEnabled: false - deviceListStrategy: "envvar" - nvidiaHookPath: null - nvidiaDriverRoot: null - gdrcopyEnabled: null - gdsEnabled: null - mofedEnabled: null - - extraArgs: - - -v=4 - extraEnvs: {} - service: - type: NodePort # Default type is NodePort, can be changed to ClusterIP - httpPort: 31992 - labels: {} - annotations: {} - - pluginPath: /var/lib/kubelet/device-plugins - libPath: /usr/local/vgpu - - podAnnotations: {} - nvidiaNodeSelector: - gpu: "on" - tolerations: [] - # The updateStrategy for DevicePlugin DaemonSet. - # If you want to update the DaemonSet by manual, set type as "OnDelete". - # We recommend use OnDelete update strategy because DevicePlugin pod restart will cause business pod restart, this behavior is destructive. - # Otherwise, you can use RollingUpdate update strategy to rolling update DevicePlugin pod. - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 1 - - resources: {} - # If you do want to specify resources, uncomment the following lines, adjust them as necessary. - # and remove the curly braces after 'resources:'. -# limits: -# cpu: 1000m -# memory: 1000Mi -# requests: -# cpu: 100m -# memory: 100Mi - -mockDevicePlugin: - enabled: false - image: - registry: "docker.io" - repository: "projecthami/mock-device-plugin" - tag: "1.0.1" - ## Specify a imagePullPolicy - ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' - ## ref: https://kubernetes.io/docs/user-guide/images/#pre-pulling-images - ## - pullPolicy: IfNotPresent - ## Optionally specify an array of imagePullSecrets. - ## Secrets must be manually created in the namespace. - ## Example: - ## pullSecrets: - ## - myRegistryKeySecretName - ## - pullSecrets: [] -devices: - amd: - customresources: - - amd.com/gpu - - amd.com/gpu-memory - awsneuron: - customresources: - - aws.amazon.com/neuron - - aws.amazon.com/neuroncore - kunlun: - enabled: true - customresources: - - kunlunxin.com/xpu - - kunlunxin.com/vxpu - - kunlunxin.com/vxpu-memory - enflame: - enabled: true - customresources: - - enflame.com/vgcu - - enflame.com/vgcu-percentage - - enflame.com/gcu - mthreads: - enabled: true - customresources: - - mthreads.com/vgpu - nvidia: - gpuCorePolicy: default - libCudaLogLevel: 1 - ascend: - enabled: false - image: "" - imagePullPolicy: IfNotPresent - extraArgs: [] - nodeSelector: - ascend: "on" - tolerations: [] - customresources: - - huawei.com/Ascend910A - - huawei.com/Ascend910A-memory - - huawei.com/Ascend910B2 - - huawei.com/Ascend910B2-memory - - huawei.com/Ascend910B3 - - huawei.com/Ascend910B3-memory - - huawei.com/Ascend910B4 - - huawei.com/Ascend910B4-memory - - huawei.com/Ascend910B4-1 - - huawei.com/Ascend910B4-1-memory - - huawei.com/Ascend310P - - huawei.com/Ascend310P-memory - iluvatar: - enabled: false - customresources: - - iluvatar.ai/BI-V100-vgpu - - iluvatar.ai/BI-V100.vCore - - iluvatar.ai/BI-V100.vMem - - iluvatar.ai/BI-V150-vgpu - - iluvatar.ai/BI-V150.vCore - - iluvatar.ai/BI-V150.vMem - - iluvatar.ai/MR-V100-vgpu - - iluvatar.ai/MR-V100.vCore - - iluvatar.ai/MR-V100.vMem - - iluvatar.ai/MR-V50-vgpu - - iluvatar.ai/MR-V50.vCore - - iluvatar.ai/MR-V50.vMem diff --git a/packages/system/hami/values.yaml b/packages/system/hami/values.yaml deleted file mode 100644 index 76e8d631..00000000 --- a/packages/system/hami/values.yaml +++ /dev/null @@ -1,17 +0,0 @@ -hami: - scheduler: - kubeScheduler: - image: - registry: registry.k8s.io - repository: kube-scheduler - devicePlugin: - runtimeClassName: nvidia - updateStrategy: - type: RollingUpdate - # See https://github.com/Project-HAMi/HAMi/blob/v2.8.1/deployments/charts/hami/values.yaml - # Each entry: {"name": "node-name", "operatingmode": "hami-core", "devicesplitcount": N, ...} - nodeConfiguration: - config: | - { - "nodeconfig": [] - } diff --git a/packages/system/harbor-rd/cozyrds/harbor.yaml b/packages/system/harbor-rd/cozyrds/harbor.yaml index 5682c053..5abc725f 100644 --- a/packages/system/harbor-rd/cozyrds/harbor.yaml +++ b/packages/system/harbor-rd/cozyrds/harbor.yaml @@ -13,6 +13,7 @@ spec: prefix: "harbor-" labels: sharding.fluxcd.io/key: tenants + internal.cozystack.io/tenantmodule: "true" chartRef: kind: ExternalArtifact name: cozystack-harbor-application-default-harbor @@ -23,6 +24,7 @@ spec: plural: Harbor name: harbor description: Managed Harbor container registry + module: true tags: - registry - container diff --git a/packages/system/harbor/templates/database.yaml b/packages/system/harbor/templates/database.yaml index 9a948601..02e11faa 100644 --- a/packages/system/harbor/templates/database.yaml +++ b/packages/system/harbor/templates/database.yaml @@ -5,9 +5,7 @@ metadata: name: {{ .Values.harbor.fullnameOverride }}-db spec: instances: {{ .Values.db.replicas }} - {{- $existingCluster := lookup "postgresql.cnpg.io/v1" "Cluster" .Release.Namespace (printf "%s-db" .Values.harbor.fullnameOverride) }} - {{- $image := dig "spec" "imageName" "ghcr.io/cloudnative-pg/postgresql:17.7-standard-trixie" $existingCluster }} - imageName: {{ if regexMatch ":17\\." $image }}ghcr.io/cloudnative-pg/postgresql:17.7-standard-trixie{{ else if regexMatch ":[0-9]+\\.[0-9]+$" $image }}{{ printf "%s-standard-trixie" $image }}{{ else }}{{ $image }}{{ end }} + imageName: ghcr.io/cloudnative-pg/postgresql:17.7 storage: size: {{ .Values.db.size }} {{- with .Values.db.storageClass }} diff --git a/packages/system/hetzner-robotlb/charts/robotlb/Chart.yaml b/packages/system/hetzner-robotlb/charts/robotlb/Chart.yaml index 192470dd..743f255d 100644 --- a/packages/system/hetzner-robotlb/charts/robotlb/Chart.yaml +++ b/packages/system/hetzner-robotlb/charts/robotlb/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -appVersion: 0.0.6 +appVersion: 0.0.5 description: A Helm chart for robotlb (loadbalancer on hetzner cloud). name: robotlb type: application diff --git a/packages/system/hetzner-robotlb/charts/robotlb/templates/deployment.yaml b/packages/system/hetzner-robotlb/charts/robotlb/templates/deployment.yaml index 41b4661a..4fd71366 100644 --- a/packages/system/hetzner-robotlb/charts/robotlb/templates/deployment.yaml +++ b/packages/system/hetzner-robotlb/charts/robotlb/templates/deployment.yaml @@ -5,7 +5,7 @@ metadata: labels: {{- include "robotlb.labels" . | nindent 4 }} spec: - replicas: 1 + replicas: {{ .Values.replicas }} selector: matchLabels: {{- include "robotlb.selectorLabels" . | nindent 6 }} diff --git a/packages/system/hetzner-robotlb/charts/robotlb/templates/role.yaml b/packages/system/hetzner-robotlb/charts/robotlb/templates/role.yaml index 76bac249..3a7b9334 100644 --- a/packages/system/hetzner-robotlb/charts/robotlb/templates/role.yaml +++ b/packages/system/hetzner-robotlb/charts/robotlb/templates/role.yaml @@ -3,8 +3,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: {{ include "robotlb.fullname" . }}-cr -rules: - {{- toYaml .Values.serviceAccount.permissions | nindent 2 }} +rules: {{- toYaml .Values.serviceAccount.permissions | nindent 2 }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/packages/system/hetzner-robotlb/charts/robotlb/values.yaml b/packages/system/hetzner-robotlb/charts/robotlb/values.yaml index 4365f677..739c7b73 100644 --- a/packages/system/hetzner-robotlb/charts/robotlb/values.yaml +++ b/packages/system/hetzner-robotlb/charts/robotlb/values.yaml @@ -36,9 +36,6 @@ serviceAccount: - apiGroups: [""] resources: [nodes, pods] verbs: [get, list, watch] - - apiGroups: [discovery.k8s.io] - resources: [endpointslices] - verbs: [get, list, watch] podAnnotations: {} podLabels: {} diff --git a/packages/system/kamaji/values.yaml b/packages/system/kamaji/values.yaml index 6ba649f0..9c6eea0e 100644 --- a/packages/system/kamaji/values.yaml +++ b/packages/system/kamaji/values.yaml @@ -3,37 +3,14 @@ kamaji: deploy: false image: pullPolicy: IfNotPresent - tag: v1.3.0@sha256:8c9af083b60600c0800eb56a2cda75f26007b7272a1cf019140de003bfce1a4d + tag: v1.2.1@sha256:34b9af510bbad3e64ea5cd7afdffa2b31c0751009c467355c8f67f8f9c5b3513 repository: ghcr.io/cozystack/cozystack/kamaji resources: limits: cpu: 200m - memory: 512Mi + memory: 500Mi requests: cpu: 100m - memory: 256Mi - startupProbe: - httpGet: - path: /healthz - port: healthcheck - initialDelaySeconds: 5 - periodSeconds: 5 - timeoutSeconds: 3 - successThreshold: 1 - failureThreshold: 12 - livenessProbe: - httpGet: - path: /healthz - port: healthcheck - initialDelaySeconds: 30 - periodSeconds: 20 - timeoutSeconds: 1 - readinessProbe: - httpGet: - path: /readyz - port: healthcheck - initialDelaySeconds: 30 - periodSeconds: 10 - timeoutSeconds: 1 + memory: 100Mi extraArgs: - - --migrate-image=ghcr.io/cozystack/cozystack/kamaji:v1.3.0@sha256:8c9af083b60600c0800eb56a2cda75f26007b7272a1cf019140de003bfce1a4d + - --migrate-image=ghcr.io/cozystack/cozystack/kamaji:v1.2.1@sha256:34b9af510bbad3e64ea5cd7afdffa2b31c0751009c467355c8f67f8f9c5b3513 diff --git a/packages/system/keycloak/templates/db.yaml b/packages/system/keycloak/templates/db.yaml index 6a57e93a..fb649a43 100644 --- a/packages/system/keycloak/templates/db.yaml +++ b/packages/system/keycloak/templates/db.yaml @@ -4,9 +4,7 @@ metadata: name: keycloak-db spec: instances: 2 - {{- $existingCluster := lookup "postgresql.cnpg.io/v1" "Cluster" .Release.Namespace "keycloak-db" }} - {{- $image := dig "spec" "imageName" "ghcr.io/cloudnative-pg/postgresql:17.7-standard-trixie" $existingCluster }} - imageName: {{ if regexMatch ":17\\." $image }}ghcr.io/cloudnative-pg/postgresql:17.7-standard-trixie{{ else if regexMatch ":[0-9]+\\.[0-9]+$" $image }}{{ printf "%s-standard-trixie" $image }}{{ else }}{{ $image }}{{ end }} + imageName: ghcr.io/cloudnative-pg/postgresql:17.7 storage: size: 20Gi {{- if .Values._cluster.scheduling }} diff --git a/packages/system/keycloak/templates/ingress.yaml b/packages/system/keycloak/templates/ingress.yaml index 7f9fd476..aa282f71 100644 --- a/packages/system/keycloak/templates/ingress.yaml +++ b/packages/system/keycloak/templates/ingress.yaml @@ -11,7 +11,7 @@ metadata: {{- with .Values.ingress.annotations }} annotations: {{- if eq $solver "http01" }} - acme.cert-manager.io/http01-ingress-ingressclassname: {{ $exposeIngress }} + acme.cert-manager.io/http01-ingress-class: {{ $exposeIngress }} {{- end }} cert-manager.io/cluster-issuer: {{ $clusterIssuer }} {{- toYaml . | nindent 4 }} diff --git a/packages/system/kubeovn-plunger/templates/deployment.yaml b/packages/system/kubeovn-plunger/templates/deployment.yaml index 37b49620..183e8c5e 100644 --- a/packages/system/kubeovn-plunger/templates/deployment.yaml +++ b/packages/system/kubeovn-plunger/templates/deployment.yaml @@ -29,8 +29,6 @@ spec: {{- end }} - --metrics-bind-address=:8080 - --metrics-secure=false - - --kube-ovn-namespace={{ .Release.Namespace }} - - --ovn-central-name={{ .Values.ovnCentralName }} ports: - name: metrics containerPort: 8080 diff --git a/packages/system/kubeovn-plunger/templates/rbac.yaml b/packages/system/kubeovn-plunger/templates/rbac.yaml index 3ca22740..a0c5527b 100644 --- a/packages/system/kubeovn-plunger/templates/rbac.yaml +++ b/packages/system/kubeovn-plunger/templates/rbac.yaml @@ -22,6 +22,8 @@ rules: - get - list - watch + resourceNames: + - {{ .Values.ovnCentralName }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding diff --git a/packages/system/kubeovn-plunger/values.yaml b/packages/system/kubeovn-plunger/values.yaml index 02620593..7691803e 100644 --- a/packages/system/kubeovn-plunger/values.yaml +++ b/packages/system/kubeovn-plunger/values.yaml @@ -1,4 +1,4 @@ portSecurity: true routes: "" -image: ghcr.io/cozystack/cozystack/kubeovn-plunger:v1.3.0@sha256:b75a0facb99c3b0fe8090414b3425f5c4858fff632d1de122ce9944c4daa89c0 +image: ghcr.io/cozystack/cozystack/kubeovn-plunger:v1.2.1@sha256:71d487b9da211061921c7183d5a1e19a32c64dffe33a3fac0557c4cb25764dfe ovnCentralName: ovn-central diff --git a/packages/system/kubeovn-webhook/values.yaml b/packages/system/kubeovn-webhook/values.yaml index 0c6ce81a..df5215d1 100644 --- a/packages/system/kubeovn-webhook/values.yaml +++ b/packages/system/kubeovn-webhook/values.yaml @@ -1,3 +1,3 @@ portSecurity: true routes: "" -image: ghcr.io/cozystack/cozystack/kubeovn-webhook:v1.3.0@sha256:e18f9fd679e38f65362a8d0042f25468272f6d081136ad47027168d8e7e07a4a +image: ghcr.io/cozystack/cozystack/kubeovn-webhook:v1.2.1@sha256:e6334c29d3aaf0dea766c88e3e05b53ad623d1bb497b3c836e6f76adade45b29 diff --git a/packages/system/kubeovn/charts/kube-ovn/Chart.yaml b/packages/system/kubeovn/charts/kube-ovn/Chart.yaml index 9c01c123..f0295b87 100644 --- a/packages/system/kubeovn/charts/kube-ovn/Chart.yaml +++ b/packages/system/kubeovn/charts/kube-ovn/Chart.yaml @@ -15,12 +15,12 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: v1.15.10 +version: v1.15.3 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "1.15.10" +appVersion: "1.15.3" kubeVersion: ">= 1.29.0-0" diff --git a/packages/system/kubeovn/charts/kube-ovn/templates/kube-ovn-crd.yaml b/packages/system/kubeovn/charts/kube-ovn/templates/kube-ovn-crd.yaml index 62ae328e..5b093be6 100644 --- a/packages/system/kubeovn/charts/kube-ovn/templates/kube-ovn-crd.yaml +++ b/packages/system/kubeovn/charts/kube-ovn/templates/kube-ovn-crd.yaml @@ -1353,77 +1353,6 @@ spec: type: string type: object type: array - resources: - description: |- - `resources` are the compute resources required by this container. - For more information, see https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - - This field depends on the DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - bandwidth: - type: object - description: Optional bandwidth limit for each egress gateway instance in both ingress and egress directions. - properties: - ingress: - type: integer - format: int64 - description: ingress bandwidth limit in Mbps - egress: - type: integer - format: int64 - description: egress bandwidth limit in Mbps --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition diff --git a/packages/system/kubeovn/charts/kube-ovn/templates/ovncni-ds.yaml b/packages/system/kubeovn/charts/kube-ovn/templates/ovncni-ds.yaml index 06971d01..8be8eba6 100644 --- a/packages/system/kubeovn/charts/kube-ovn/templates/ovncni-ds.yaml +++ b/packages/system/kubeovn/charts/kube-ovn/templates/ovncni-ds.yaml @@ -126,9 +126,6 @@ spec: - --enable-ovn-ipsec={{- .Values.func.ENABLE_OVN_IPSEC }} - --set-vxlan-tx-off={{- .Values.func.SET_VXLAN_TX_OFF }} - --non-primary-cni-mode={{- .Values.cni_conf.NON_PRIMARY_CNI }} - {{- with .Values.mtu }} - - --mtu={{ . }} - {{- end }} securityContext: runAsUser: 0 privileged: false diff --git a/packages/system/kubeovn/charts/kube-ovn/templates/pre-upgrade-ovs-ovn.yaml b/packages/system/kubeovn/charts/kube-ovn/templates/pre-upgrade-ovs-ovn.yaml deleted file mode 100644 index a18a4111..00000000 --- a/packages/system/kubeovn/charts/kube-ovn/templates/pre-upgrade-ovs-ovn.yaml +++ /dev/null @@ -1,129 +0,0 @@ -{{- if include "kubeovn.ovn.versionCompatibility" . -}} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: ovs-ovn-pre-upgrade - namespace: {{ .Values.namespace }} - annotations: - "helm.sh/hook": pre-upgrade - "helm.sh/hook-weight": "1" - "helm.sh/hook-delete-policy": hook-succeeded -automountServiceAccountToken: false ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - annotations: - rbac.authorization.k8s.io/system-only: "true" - "helm.sh/hook": pre-upgrade - "helm.sh/hook-weight": "2" - "helm.sh/hook-delete-policy": hook-succeeded - name: system:ovs-ovn-pre-upgrade -rules: - - apiGroups: - - "" - resources: - - pods - verbs: - - list - - get - - apiGroups: - - "" - resources: - - pods/exec - verbs: - - create ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: ovs-ovn-pre-upgrade - annotations: - "helm.sh/hook": pre-upgrade - "helm.sh/hook-weight": "3" - "helm.sh/hook-delete-policy": hook-succeeded -roleRef: - name: system:ovs-ovn-pre-upgrade - kind: ClusterRole - apiGroup: rbac.authorization.k8s.io -subjects: - - kind: ServiceAccount - name: ovs-ovn-pre-upgrade - namespace: {{ .Values.namespace }} ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: "{{ .Chart.Name }}-pre-upgrade-hook" - namespace: {{ .Values.namespace }} - labels: - app.kubernetes.io/managed-by: {{ .Release.Service | quote }} - app.kubernetes.io/instance: {{ .Release.Name | quote }} - app.kubernetes.io/version: {{ .Chart.AppVersion }} - helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - annotations: - "helm.sh/hook": pre-upgrade - "helm.sh/hook-weight": "4" - "helm.sh/hook-delete-policy": hook-succeeded -spec: - completions: 1 - template: - metadata: - name: "{{ .Release.Name }}" - labels: - app.kubernetes.io/managed-by: {{ .Release.Service | quote }} - app.kubernetes.io/instance: {{ .Release.Name | quote }} - helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - app: pre-upgrade - component: job - spec: - tolerations: - - key: "" - operator: "Exists" - effect: "NoSchedule" - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - topologyKey: kubernetes.io/hostname - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - pre-upgrade - - key: component - operator: In - values: - - job - restartPolicy: Never - hostNetwork: true - nodeSelector: - kubernetes.io/os: "linux" - serviceAccountName: ovs-ovn-pre-upgrade - automountServiceAccountToken: true - securityContext: - seccompProfile: - type: RuntimeDefault - containers: - - name: ovs-ovn-pre-upgrade - image: "{{ .Values.global.registry.address}}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }}" - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - command: - - bash - - -eo - - pipefail - - -c - - bash /kube-ovn/pre-upgrade-ovs.sh 2>&1 | tee -a /var/log/kube-ovn/pre-upgrade-ovs.log - volumeMounts: - - mountPath: /var/log/kube-ovn - name: kube-ovn-log - volumes: - - name: kube-ovn-log - hostPath: - path: {{ .Values.log_conf.LOG_DIR }}/kube-ovn -{{- end -}} diff --git a/packages/system/kubeovn/charts/kube-ovn/values.yaml b/packages/system/kubeovn/charts/kube-ovn/values.yaml index fd3472f6..3ffb7750 100644 --- a/packages/system/kubeovn/charts/kube-ovn/values.yaml +++ b/packages/system/kubeovn/charts/kube-ovn/values.yaml @@ -8,11 +8,11 @@ global: images: kubeovn: repository: kube-ovn - tag: v1.15.10 + tag: v1.15.3 natgateway: repository: vpc-nat-gateway # Falls back to the same tag as kubeovn if empty - tag: v1.15.10 + tag: v1.15.3 image: pullPolicy: IfNotPresent diff --git a/packages/system/kubeovn/values.yaml b/packages/system/kubeovn/values.yaml index e9d2d10f..6911f8aa 100644 --- a/packages/system/kubeovn/values.yaml +++ b/packages/system/kubeovn/values.yaml @@ -65,4 +65,4 @@ global: images: kubeovn: repository: kubeovn - tag: v1.15.10@sha256:741299cbd0081a786a6b60c460fa3156b3a42a38141c559dd8ac031f50c5504f + tag: v1.15.3@sha256:fa53d5f254f640cb626329ad35d9e7aad647dd8e1e645e68f3f13c3659472a30 diff --git a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml index f81eaff0..5e9e8f94 100644 --- a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml +++ b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml @@ -2,27 +2,13 @@ apiVersion: cozystack.io/v1alpha1 kind: ApplicationDefinition metadata: name: kubernetes - annotations: - # Kubernetes tenants boot a Kamaji control plane whose admin-kubeconfig - # Secret is provisioned asynchronously. Cold Kamaji start (image pull + - # etcd + apiserver Ready) plus admin-kubeconfig generation can exceed - # Flux helm-controller's default wait budget, causing remediation loops - # that uninstall the Cluster CR. This override is applied by cozystack-api - # to the HelmRelease Spec.Install.Timeout and Spec.Upgrade.Timeout. - # - # Coupling: the wait-for-kubeconfig init container in - # packages/apps/kubernetes/templates/_helpers.tpl hard-codes a 10m - # deadline chosen to stay strictly below this value so the pod's - # CrashLoopBackOff surfaces before flux remediation fires. If this - # annotation is raised, update that init deadline correspondingly. - release.cozystack.io/helm-install-timeout: "15m" spec: application: kind: Kubernetes singular: kubernetes plural: kuberneteses openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"replicated"},"nodeGroups":{"description":"Worker nodes configuration map.","type":"object","default":{"md0":{"ephemeralStorage":"20Gi","gpus":[],"instanceType":"u1.medium","maxReplicas":10,"minReplicas":0,"resources":{},"roles":["ingress-nginx"]}},"additionalProperties":{"type":"object","required":["ephemeralStorage","instanceType","maxReplicas","minReplicas","resources"],"properties":{"ephemeralStorage":{"description":"Ephemeral storage size.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string"}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium"},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"CPU and memory resources for each worker node.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles.","type":"array","items":{"type":"string"}}}}},"version":{"description":"Kubernetes major.minor version to deploy","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31","v1.30"]},"host":{"description":"External hostname for Kubernetes cluster. Defaults to `.` if empty.","type":"string","default":""},"addons":{"description":"Cluster addons configuration.","type":"object","default":{},"required":["certManager","cilium","coredns","fluxcd","gatewayAPI","gpuOperator","hami","ingressNginx","monitoringAgents","velero","verticalPodAutoscaler"],"properties":{"certManager":{"description":"Cert-manager addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable cert-manager.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"cilium":{"description":"Cilium CNI plugin.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"coredns":{"description":"CoreDNS addon.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"fluxcd":{"description":"FluxCD GitOps operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable FluxCD.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"gatewayAPI":{"description":"Gateway API addon.","type":"object","default":{},"required":["enabled"],"properties":{"enabled":{"description":"Enable Gateway API.","type":"boolean","default":false}}},"gpuOperator":{"description":"NVIDIA GPU Operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable GPU Operator.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"hami":{"description":"HAMi GPU virtualization middleware.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable HAMi (requires GPU Operator).","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ingressNginx":{"description":"Ingress-NGINX controller.","type":"object","default":{},"required":["enabled","exposeMethod","valuesOverride"],"properties":{"enabled":{"description":"Enable the controller (requires nodes labeled `ingress-nginx`).","type":"boolean","default":false},"exposeMethod":{"description":"Method to expose the controller. Allowed values: `Proxied`, `LoadBalancer`.","type":"string","default":"Proxied","enum":["Proxied","LoadBalancer"]},"hosts":{"description":"Domains routed to this tenant cluster when `exposeMethod` is `Proxied`.","type":"array","default":[],"items":{"type":"string"}},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"monitoringAgents":{"description":"Monitoring agents.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable monitoring agents.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"velero":{"description":"Velero backup/restore addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable Velero.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"verticalPodAutoscaler":{"description":"Vertical Pod Autoscaler.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}}}},"controlPlane":{"description":"Kubernetes control-plane configuration.","type":"object","default":{},"required":["apiServer","controllerManager","konnectivity","replicas","scheduler"],"properties":{"apiServer":{"description":"API Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for API Server.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"large","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"controllerManager":{"description":"Controller Manager configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Controller Manager.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"konnectivity":{"description":"Konnectivity configuration.","type":"object","default":{},"required":["server"],"properties":{"server":{"description":"Konnectivity Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Konnectivity.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"replicas":{"description":"Number of control-plane replicas.","type":"integer","default":2},"scheduler":{"description":"Scheduler configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Scheduler.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"images":{"description":"Optional image overrides for air-gapped or rate-limited registries.","type":"object","default":{},"properties":{"waitForKubeconfig":{"description":"Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag.","type":"string","default":""}}}}} + {"title":"Chart Values","type":"object","properties":{"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"replicated"},"nodeGroups":{"description":"Worker nodes configuration map.","type":"object","default":{"md0":{"ephemeralStorage":"20Gi","gpus":[],"instanceType":"u1.medium","maxReplicas":10,"minReplicas":0,"resources":{},"roles":["ingress-nginx"]}},"additionalProperties":{"type":"object","required":["ephemeralStorage","instanceType","maxReplicas","minReplicas","resources"],"properties":{"ephemeralStorage":{"description":"Ephemeral storage size.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string"}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium"},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"CPU and memory resources for each worker node.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles.","type":"array","items":{"type":"string"}}}}},"version":{"description":"Kubernetes major.minor version to deploy","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31","v1.30"]},"host":{"description":"External hostname for Kubernetes cluster. Defaults to `.` if empty.","type":"string","default":""},"addons":{"description":"Cluster addons configuration.","type":"object","default":{},"required":["certManager","cilium","coredns","fluxcd","gatewayAPI","gpuOperator","ingressNginx","monitoringAgents","velero","verticalPodAutoscaler"],"properties":{"certManager":{"description":"Cert-manager addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable cert-manager.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"cilium":{"description":"Cilium CNI plugin.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"coredns":{"description":"CoreDNS addon.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"fluxcd":{"description":"FluxCD GitOps operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable FluxCD.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"gatewayAPI":{"description":"Gateway API addon.","type":"object","default":{},"required":["enabled"],"properties":{"enabled":{"description":"Enable Gateway API.","type":"boolean","default":false}}},"gpuOperator":{"description":"NVIDIA GPU Operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable GPU Operator.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ingressNginx":{"description":"Ingress-NGINX controller.","type":"object","default":{},"required":["enabled","exposeMethod","valuesOverride"],"properties":{"enabled":{"description":"Enable the controller (requires nodes labeled `ingress-nginx`).","type":"boolean","default":false},"exposeMethod":{"description":"Method to expose the controller. Allowed values: `Proxied`, `LoadBalancer`.","type":"string","default":"Proxied","enum":["Proxied","LoadBalancer"]},"hosts":{"description":"Domains routed to this tenant cluster when `exposeMethod` is `Proxied`.","type":"array","default":[],"items":{"type":"string"}},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"monitoringAgents":{"description":"Monitoring agents.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable monitoring agents.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"velero":{"description":"Velero backup/restore addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable Velero.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"verticalPodAutoscaler":{"description":"Vertical Pod Autoscaler.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}}}},"controlPlane":{"description":"Kubernetes control-plane configuration.","type":"object","default":{},"required":["apiServer","controllerManager","konnectivity","replicas","scheduler"],"properties":{"apiServer":{"description":"API Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for API Server.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"large","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"controllerManager":{"description":"Controller Manager configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Controller Manager.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}},"konnectivity":{"description":"Konnectivity configuration.","type":"object","default":{},"required":["server"],"properties":{"server":{"description":"Konnectivity Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Konnectivity.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"replicas":{"description":"Number of control-plane replicas.","type":"integer","default":2},"scheduler":{"description":"Scheduler configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Scheduler.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]}}}}}}} release: prefix: kubernetes- labels: @@ -40,7 +26,7 @@ spec: tags: - compute icon: PHN2ZyB3aWR0aD0iMTQ0IiBoZWlnaHQ9IjE0NCIgdmlld0JveD0iMCAwIDE0NCAxNDQiIGZpbGw9Im5vbmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CjxyZWN0IHdpZHRoPSIxNDQiIGhlaWdodD0iMTQ0IiByeD0iMjQiIGZpbGw9InVybCgjcGFpbnQwX2xpbmVhcl82ODFfMjg0NSkiLz4KPHBhdGggZD0iTTcxLjk5NjggMTlDNzAuMzAzOSAxOS4wMDAyIDY4LjkzMTIgMjAuNTMyMiA2OC45MzE0IDIyLjQyMjFDNjguOTMxNCAyMi40NTExIDY4LjkzNzMgMjIuNDc4OCA2OC45Mzc5IDIyLjUwNzZDNjguOTM1NCAyMi43NjQ0IDY4LjkyMzEgMjMuMDczNyA2OC45MzE0IDIzLjI5NzNDNjguOTcxNyAyNC4zODczIDY5LjIwODIgMjUuMjIxNiA2OS4zNTA2IDI2LjIyNThDNjkuNjA4NCAyOC4zNzUyIDY5LjgyNDUgMzAuMTU2OSA2OS42OTEyIDMxLjgxM0M2OS41NjE1IDMyLjQzNzUgNjkuMTAzNyAzMy4wMDg2IDY4LjY5NTYgMzMuNDA1Nkw2OC42MjM1IDM0LjcwODZDNjYuNzgzOSAzNC44NjE3IDY0LjkzMTkgMzUuMTQyMSA2My4wODIxIDM1LjU2NDFDNTUuMTIyNiAzNy4zNzk4IDQ4LjI2OTUgNDEuNDk5MSA0My4wNTIgNDcuMDYwOUM0Mi43MTM0IDQ2LjgyODggNDIuMTIxMSA0Ni40MDE5IDQxLjk0NSA0Ni4yNzEyQzQxLjM5NzcgNDYuMzQ1NCA0MC44NDQ1IDQ2LjUxNTEgNDAuMTI0MSA0Ni4wOTM1QzM4Ljc1MjIgNDUuMTY1NyAzNy41MDI4IDQzLjg4NTEgMzUuOTkxIDQyLjM0MjRDMzUuMjk4MiA0MS42MDQ0IDM0Ljc5NjYgNDAuOTAxOCAzMy45NzM1IDQwLjE5MDRDMzMuNzg2NiA0MC4wMjg5IDMzLjUwMTQgMzkuODEwNCAzMy4yOTIzIDM5LjY0NDJDMzIuNjQ4OSAzOS4xMjg4IDMxLjg5IDM4Ljg2IDMxLjE1NyAzOC44MzQ4QzMwLjIxNDcgMzguODAyNCAyOS4zMDc1IDM5LjE3MjUgMjguNzEzOCAzOS45MjA2QzI3LjY1ODQgNDEuMjUwNiAyNy45OTYzIDQzLjI4MzMgMjkuNDY3MSA0NC40NjE0QzI5LjQ4MiA0NC40NzM0IDI5LjQ5NzkgNDQuNDgyNyAyOS41MTI5IDQ0LjQ5NDNDMjkuNzE1IDQ0LjY1ODkgMjkuOTYyNSA0NC44Njk4IDMwLjE0ODMgNDUuMDA3NkMzMS4wMjE3IDQ1LjY1NTUgMzEuODE5NSA0NS45ODcyIDMyLjY4OTcgNDYuNTAxNUMzNC41MjMxIDQ3LjYzOTEgMzYuMDQzIDQ4LjU4MjMgMzcuMjQ4NiA0OS43MTk2QzM3LjcxOTQgNTAuMjIzNyAzNy44MDE2IDUxLjExMjIgMzcuODY0MyA1MS40OTY0TDM4Ljg0NjggNTIuMzc4MkMzMy41ODcyIDYwLjMzMDggMzEuMTUzIDcwLjE1MzkgMzIuNTkxNSA4MC4xNjI3TDMxLjMwNzcgODAuNTM3OEMzMC45NjkzIDgwLjk3NjggMzAuNDkxMiA4MS42Njc2IDI5Ljk5MTEgODEuODczOEMyOC40MTM4IDgyLjM3MjkgMjYuNjM4NyA4Mi41NTYyIDI0LjQ5NTYgODIuNzgxOUMyMy40ODk0IDgyLjg2NiAyMi42MjEzIDgyLjgxNTggMjEuNTU0NiA4My4wMTg4QzIxLjMxOTggODMuMDYzNSAyMC45OTI3IDgzLjE0OTEgMjAuNzM1OCA4My4yMDk3QzIwLjcyNjkgODMuMjExNiAyMC43MTg2IDgzLjIxNDIgMjAuNzA5NiA4My4yMTYyQzIwLjY5NTYgODMuMjE5NSAyMC42NzcyIDgzLjIyNjMgMjAuNjYzOCA4My4yMjk0QzE4Ljg1NyA4My42NjggMTcuNjk2MyA4NS4zMzY1IDE4LjA2OTkgODYuOTgwNUMxOC40NDM3IDg4LjYyNDggMjAuMjA4NiA4OS42MjQ4IDIyLjAyNjIgODkuMjMxMkMyMi4wMzkzIDg5LjIyODIgMjIuMDU4NCA4OS4yMjc3IDIyLjA3MiA4OS4yMjQ2QzIyLjA5MjYgODkuMjE5OSAyMi4xMTA2IDg5LjIwOTkgMjIuMTMxIDg5LjIwNDlDMjIuMzg0NCA4OS4xNDkgMjIuNzAxOSA4OS4wODY4IDIyLjkyMzYgODkuMDI3MkMyMy45NzIzIDg4Ljc0NTEgMjQuNzMxOCA4OC4zMzA2IDI1LjY3NDYgODcuOTY3N0MyNy43MDI5IDg3LjIzNjggMjkuMzgyOCA4Ni42MjYyIDMxLjAxOTUgODYuMzg4M0MzMS43MDMgODYuMzM0NSAzMi40MjMyIDg2LjgxMiAzMi43ODE0IDg3LjAxMzRMMzQuMTE3NyA4Ni43ODMxQzM3LjE5MjYgOTYuMzYxMyA0My42MzY2IDEwNC4xMDMgNTEuNzk2MyAxMDguOTYxTDUxLjIzOTYgMTEwLjMwM0M1MS40NDAzIDExMC44MjQgNTEuNjYxNiAxMTEuNTMgNTEuNTEyMSAxMTIuMDQ1QzUwLjkxNzEgMTEzLjU5NSA0OS44OTggMTE1LjIzMSA0OC43Mzc0IDExNy4wNTVDNDguMTc1NSAxMTcuODk4IDQ3LjYwMDQgMTE4LjU1MiA0Ny4wOTM0IDExOS41MTZDNDYuOTcyIDExOS43NDcgNDYuODE3NSAxMjAuMTAyIDQ2LjcwMDQgMTIwLjM0NkM0NS45MTI1IDEyMi4wMzkgNDYuNDkwNCAxMjMuOTkgNDguMDAzOCAxMjQuNzIyQzQ5LjUyNjggMTI1LjQ1OSA1MS40MTcxIDEyNC42ODIgNTIuMjM1MiAxMjIuOTg1QzUyLjIzNjQgMTIyLjk4MiA1Mi4yNDA2IDEyMi45OCA1Mi4yNDE3IDEyMi45NzhDNTIuMjQyNiAxMjIuOTc2IDUyLjI0MDkgMTIyLjk3MyA1Mi4yNDE3IDEyMi45NzFDNTIuMzU4MiAxMjIuNzMxIDUyLjUyMzMgMTIyLjQxNSA1Mi42MjE2IDEyMi4xODhDNTMuMDU2IDEyMS4xODkgNTMuMjAwNSAxMjAuMzMyIDUzLjUwNTkgMTE5LjM2NUM1NC4zMTcgMTE3LjMxOCA1NC43NjI2IDExNS4xNyA1NS44NzkxIDExMy44MzJDNTYuMTg0OSAxMTMuNDY2IDU2LjY4MzMgMTEzLjMyNSA1Ny4yMDAxIDExMy4xODZMNTcuODk0NCAxMTEuOTIyQzY1LjAwOCAxMTQuNjY1IDcyLjk3MDUgMTE1LjQwMiA4MC45MjQ1IDExMy41ODdDODIuNzM5MSAxMTMuMTczIDg0LjQ5MDggMTEyLjYzNyA4Ni4xODQzIDExMS45OTRDODYuMzc5NCAxMTIuMzQyIDg2Ljc0MiAxMTMuMDExIDg2LjgzOTMgMTEzLjE3OUM4Ny4zNjQ0IDExMy4zNTEgODcuOTM3NyAxMTMuNDM5IDg4LjQwNDcgMTE0LjEzM0M4OS4yNDAxIDExNS41NjcgODkuODExNCAxMTcuMjYzIDkwLjUwNzMgMTE5LjMxMkM5MC44MTI4IDEyMC4yNzkgOTAuOTYzOCAxMjEuMTM2IDkxLjM5ODEgMTIyLjEzNkM5MS40OTcxIDEyMi4zNjMgOTEuNjYxNCAxMjIuNjg0IDkxLjc3OCAxMjIuOTI1QzkyLjU5NDQgMTI0LjYyOCA5NC40OTA3IDEyNS40MDcgOTYuMDE1OSAxMjQuNjY5Qzk3LjUyOTIgMTIzLjkzNyA5OC4xMDc3IDEyMS45ODYgOTcuMzE5NCAxMjAuMjkzQzk3LjIwMjMgMTIwLjA0OSA5Ny4wNDEyIDExOS42OTUgOTYuOTE5OCAxMTkuNDY0Qzk2LjQxMjcgMTE4LjQ5OSA5NS44Mzc3IDExNy44NTIgOTUuMjc1OCAxMTcuMDA5Qzk0LjExNTIgMTE1LjE4NSA5My4xNTI2IDExMy42NyA5Mi41NTc1IDExMi4xMkM5Mi4zMDg3IDExMS4zMiA5Mi41OTk1IDExMC44MjMgOTIuNzkzMyAxMTAuMzAzQzkyLjY3NzIgMTEwLjE3IDkyLjQyODggMTA5LjQxNCA5Mi4yODI0IDEwOS4wNTlDMTAwLjc2MiAxMDQuMDI5IDEwNy4wMTcgOTUuOTk4NSAxMDkuOTU1IDg2LjcyMzlDMTEwLjM1MSA4Ni43ODY1IDExMS4wNDEgODYuOTA5MSAxMTEuMjY1IDg2Ljk1NDJDMTExLjcyNiA4Ni42NDg3IDExMi4xNDkgODYuMjUwMSAxMTIuOTgxIDg2LjMxNTlDMTE0LjYxNyA4Ni41NTM3IDExNi4yOTcgODcuMTY0NSAxMTguMzI2IDg3Ljg5NTNDMTE5LjI2OCA4OC4yNTgxIDEyMC4wMjggODguNjc5MyAxMjEuMDc3IDg4Ljk2MTRDMTIxLjI5OCA4OS4wMjEgMTIxLjYxNiA4OS4wNzY2IDEyMS44NjkgODkuMTMyNUMxMjEuODg5IDg5LjEzNzUgMTIxLjkwOCA4OS4xNDc1IDEyMS45MjggODkuMTUyMkMxMjEuOTQyIDg5LjE1NTMgMTIxLjk2MSA4OS4xNTU4IDEyMS45NzQgODkuMTU4OEMxMjMuNzkyIDg5LjU1MiAxMjUuNTU3IDg4LjU1MjYgMTI1LjkzIDg2LjkwODFDMTI2LjMwMyA4NS4yNjQxIDEyNS4xNDMgODMuNTk1MiAxMjMuMzM2IDgzLjE1N0MxMjMuMDc0IDgzLjA5NyAxMjIuNzAxIDgyLjk5NSAxMjIuNDQ2IDgyLjk0NjVDMTIxLjM3OSA4Mi43NDM1IDEyMC41MTEgODIuNzkzNSAxMTkuNTA1IDgyLjcwOTVDMTE3LjM2MSA4Mi40ODM5IDExNS41ODYgODIuMzAwNCAxMTQuMDA5IDgxLjgwMTRDMTEzLjM2NiA4MS41NTA3IDExMi45MDggODAuNzgxOSAxMTIuNjg2IDgwLjQ2NTVMMTExLjQ0OCA4MC4xMDM1QzExMi4wOSA3NS40MzggMTExLjkxNyA3MC41ODI1IDExMC44MDYgNjUuNzI0M0MxMDkuNjg1IDYwLjgyMDggMTA3LjcwNCA1Ni4zMzYxIDEwNS4wNjIgNTIuMzg0OEMxMDUuMzc5IDUyLjA5NDggMTA1Ljk3OSA1MS41NjEyIDEwNi4xNDkgNTEuNDA0M0MxMDYuMTk5IDUwLjg1MTcgMTA2LjE1NiA1MC4yNzIyIDEwNi43MjUgNDkuNjYwM0MxMDcuOTMxIDQ4LjUyMyAxMDkuNDUxIDQ3LjU3OTkgMTExLjI4NCA0Ni40NDIzQzExMi4xNTQgNDUuOTI3OSAxMTIuOTU5IDQ1LjU5NjQgMTEzLjgzMiA0NC45NDg0QzExNC4wMyA0NC44MDE5IDExNC4yOTkgNDQuNTY5OSAxMTQuNTA3IDQ0LjQwMjJDMTE1Ljk3NyA0My4yMjM3IDExNi4zMTYgNDEuMTkxMSAxMTUuMjYgMzkuODYxNEMxMTQuMjA0IDM4LjUzMTcgMTEyLjE1OSAzOC40MDY1IDExMC42ODggMzkuNTg1QzExMC40NzkgMzkuNzUxNiAxMTAuMTk1IDM5Ljk2ODggMTEwLjAwNyA0MC4xMzEyQzEwOS4xODQgNDAuODQyNiAxMDguNjc2IDQxLjU0NTIgMTA3Ljk4MyA0Mi4yODMyQzEwNi40NzEgNDMuODI1OSAxMDUuMjIyIDQ1LjExMyAxMDMuODUgNDYuMDQwOUMxMDMuMjU1IDQ2LjM4ODUgMTAyLjM4NSA0Ni4yNjgyIDEwMS45OSA0Ni4yNDQ5TDEwMC44MjQgNDcuMDgwNkM5NC4xNzUzIDQwLjA3NjMgODUuMTIzNSAzNS41OTgyIDc1LjM3NjYgMzQuNzI4M0M3NS4zNDk0IDM0LjMxNzkgNzUuMzEzNyAzMy41NzYxIDc1LjMwNDYgMzMuMzUyOUM3NC45MDU2IDMyLjk2OTMgNzQuNDIzNSAzMi42NDE4IDc0LjMwMjQgMzEuODEzQzc0LjE2OTEgMzAuMTU2OSA3NC4zOTE3IDI4LjM3NTIgNzQuNjQ5NiAyNi4yMjU4Qzc0Ljc5MTkgMjUuMjIxNiA3NS4wMjg0IDI0LjM4NzMgNzUuMDY4OCAyMy4yOTczQzc1LjA3OCAyMy4wNDk1IDc1LjA2MzIgMjIuNjkgNzUuMDYyMiAyMi40MjIxQzc1LjA2MiAyMC41MzIyIDczLjY4OTggMTguOTk5OCA3MS45OTY4IDE5Wk02OC4xNTg1IDQyLjg4ODZMNjcuMjQ4IDU5LjA0NDdMNjcuMTgyNSA1OS4wNzc2QzY3LjEyMTQgNjAuNTIyOSA2NS45Mzc1IDYxLjY3NyA2NC40ODM5IDYxLjY3N0M2My44ODg0IDYxLjY3NyA2My4zMzg4IDYxLjQ4NDkgNjIuODkyMiA2MS4xNTcxTDYyLjg2NiA2MS4xNzAzTDQ5LjY4MDcgNTEuNzc5NEM1My43MzMxIDQ3Ljc3NTkgNTguOTE2NCA0NC44MTcyIDY0Ljg5IDQzLjQ1NDZDNjUuOTgxMiA0My4yMDU2IDY3LjA3MTkgNDMuMDIwOSA2OC4xNTg1IDQyLjg4ODZaTTc1Ljg0MTcgNDIuODg4NkM4Mi44MTU5IDQzLjc1MDQgODkuMjY1NyA0Ni45MjMyIDk0LjIwODEgNTEuNzg2TDgxLjEwOCA2MS4xMTc2TDgxLjA2MjEgNjEuMDk3OUM3OS44OTk0IDYxLjk1MTIgNzguMjYxMSA2MS43Mzk0IDc3LjM1NDggNjAuNTk3OEM3Ni45ODM1IDYwLjEzMDEgNzYuNzg4NyA1OS41ODAxIDc2Ljc2NTMgNTkuMDI0OUw3Ni43NTIyIDU5LjAxODRMNzUuODQxNyA0Mi44ODg2Wk00NC44OTkxIDU3LjgxNEw1Ni45MzgyIDY4LjYzM0w1Ni45MjUxIDY4LjY5ODhDNTguMDExNyA2OS42NDc5IDU4LjE3MiA3MS4yOTQ5IDU3LjI2NTcgNzIuNDM2OEM1Ni44OTQ0IDcyLjkwNDUgNTYuMzk3NSA3My4yMTgyIDU1Ljg2MzkgNzMuMzY0N0w1NS44NTA4IDczLjQxNzNMNDAuNDE4OCA3Ny44OTIzQzM5LjYzMzQgNzAuNjc2NSA0MS4zMjYxIDYzLjY2MjEgNDQuODk5MSA1Ny44MTRaTTk5LjAwOTQgNTcuODIwNkMxMDAuNzk4IDYwLjczMzYgMTAyLjE1MyA2My45ODcxIDEwMi45NTkgNjcuNTE0M0MxMDMuNzU2IDcwLjk5OTEgMTAzLjk1NiA3NC40Nzc4IDEwMy42MjcgNzcuODM5N0w4OC4xMTY2IDczLjM1MTVMODguMTAzNSA3My4yODU3Qzg2LjcxNDUgNzIuOTA0MyA4NS44NjA5IDcxLjQ4NDggODYuMTg0MyA3MC4wNjExQzg2LjMxNjggNjkuNDc3OCA4Ni42MjQ5IDY4Ljk4NDQgODcuMDQyMyA2OC42MTk4TDg3LjAzNTggNjguNTg2OUw5OS4wMDk0IDU3LjgyMDZaTTY5LjUyNzQgNjkuNDY4OEg3NC40NTk2TDc3LjUyNTEgNzMuMzE4Nkw3Ni40MjQ3IDc4LjEyMjZMNzEuOTk2OCA4MC4yNjE0TDY3LjU1NTggNzguMTE2MUw2Ni40NTU0IDczLjMxMkw2OS41Mjc0IDY5LjQ2ODhaTTg1LjMzOTMgODIuNjQzN0M4NS41NDg5IDgyLjYzMzEgODUuNzU3NiA4Mi42NTIgODUuOTYxNiA4Mi42ODk4TDg1Ljk4NzggODIuNjU2OUwxMDEuOTUgODUuMzY4MkM5OS42MTQyIDkxLjk2MjQgOTUuMTQ0IDk3LjY3NSA4OS4xNzExIDEwMS40OThMODIuOTc0NyA4Ni40NjA2TDgyLjk5NDQgODYuNDM0M0M4Mi40MjUyIDg1LjEwNTUgODIuOTk0OCA4My41NDcyIDg0LjMwNDQgODIuOTEzNUM4NC42Mzk3IDgyLjc1MTMgODQuOTkgODIuNjYxNCA4NS4zMzkzIDgyLjY0MzdaTTU4LjUyOTggODIuNzA5NUM1OS43NDggODIuNzI2NyA2MC44NDA2IDgzLjU3NjEgNjEuMTIzNyA4NC44MjJDNjEuMjU2MiA4NS40MDUyIDYxLjE5MTcgODUuOTgzMSA2MC45NzMgODYuNDkzNUw2MS4wMTg5IDg2LjU1MjdMNTQuODg4IDEwMS40MzlDNDkuMTU1OSA5Ny43NDMyIDQ0LjU5MDQgOTIuMjA5OSA0Mi4xNDgxIDg1LjQyMDhMNTcuOTczMSA4Mi43MjI3TDU3Ljk5OTMgODIuNzU1NkM1OC4xNzYzIDgyLjcyMjkgNTguMzU1OCA4Mi43MDcxIDU4LjUyOTggODIuNzA5NVpNNzEuODk4NiA4OS4yMzEyQzcyLjMyMjkgODkuMjE1NSA3Mi43NTM0IDg5LjMwMyA3My4xNjI3IDg5LjUwMUM3My42OTkyIDg5Ljc2MDYgNzQuMTEzNiA5MC4xNjkyIDc0LjM3NDUgOTAuNjU5Mkg3NC40MzM0TDgyLjIzNDYgMTA0LjgyMUM4MS4yMjIxIDEwNS4xNjIgODAuMTgxMyAxMDUuNDU0IDc5LjExNjcgMTA1LjY5N0M3My4xNTA1IDEwNy4wNTggNjcuMjAzMiAxMDYuNjQ1IDYxLjgxOCAxMDQuODAyTDY5LjU5OTUgOTAuNjY1OEg2OS42MTI2QzcwLjA3OTUgODkuNzg4OCA3MC45NjUgODkuMjY1NiA3MS44OTg2IDg5LjIzMTJaIiBmaWxsPSJ3aGl0ZSIgc3Ryb2tlPSJ3aGl0ZSIgc3Ryb2tlLXdpZHRoPSIwLjI1Ii8+CjxkZWZzPgo8bGluZWFyR3JhZGllbnQgaWQ9InBhaW50MF9saW5lYXJfNjgxXzI4NDUiIHgxPSIxMCIgeTE9IjE1LjUiIHgyPSIxNDQiIHkyPSIxMzEuNSIgZ3JhZGllbnRVbml0cz0idXNlclNwYWNlT25Vc2UiPgo8c3RvcCBzdG9wLWNvbG9yPSIjNEQ4N0ZGIi8+CjxzdG9wIG9mZnNldD0iMSIgc3RvcC1jb2xvcj0iIzA1NDdEMCIvPgo8L2xpbmVhckdyYWRpZW50Pgo8L2RlZnM+Cjwvc3ZnPgo= - keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "storageClass"], ["spec", "nodeGroups"], ["spec", "nodeGroups", "md0"], ["spec", "nodeGroups", "md0", "minReplicas"], ["spec", "nodeGroups", "md0", "maxReplicas"], ["spec", "nodeGroups", "md0", "instanceType"], ["spec", "nodeGroups", "md0", "ephemeralStorage"], ["spec", "nodeGroups", "md0", "roles"], ["spec", "nodeGroups", "md0", "resources"], ["spec", "nodeGroups", "md0", "gpus"], ["spec", "version"], ["spec", "host"], ["spec", "addons"], ["spec", "addons", "certManager"], ["spec", "addons", "certManager", "enabled"], ["spec", "addons", "certManager", "valuesOverride"], ["spec", "addons", "cilium"], ["spec", "addons", "cilium", "valuesOverride"], ["spec", "addons", "gatewayAPI"], ["spec", "addons", "gatewayAPI", "enabled"], ["spec", "addons", "ingressNginx"], ["spec", "addons", "ingressNginx", "enabled"], ["spec", "addons", "ingressNginx", "exposeMethod"], ["spec", "addons", "ingressNginx", "hosts"], ["spec", "addons", "ingressNginx", "valuesOverride"], ["spec", "addons", "gpuOperator"], ["spec", "addons", "gpuOperator", "enabled"], ["spec", "addons", "gpuOperator", "valuesOverride"], ["spec", "addons", "hami"], ["spec", "addons", "hami", "enabled"], ["spec", "addons", "hami", "valuesOverride"], ["spec", "addons", "fluxcd"], ["spec", "addons", "fluxcd", "enabled"], ["spec", "addons", "fluxcd", "valuesOverride"], ["spec", "addons", "monitoringAgents"], ["spec", "addons", "monitoringAgents", "enabled"], ["spec", "addons", "monitoringAgents", "valuesOverride"], ["spec", "addons", "verticalPodAutoscaler"], ["spec", "addons", "verticalPodAutoscaler", "valuesOverride"], ["spec", "addons", "velero"], ["spec", "addons", "velero", "enabled"], ["spec", "addons", "velero", "valuesOverride"], ["spec", "addons", "coredns"], ["spec", "addons", "coredns", "valuesOverride"], ["spec", "controlPlane"], ["spec", "controlPlane", "replicas"], ["spec", "controlPlane", "apiServer"], ["spec", "controlPlane", "apiServer", "resources"], ["spec", "controlPlane", "apiServer", "resourcesPreset"], ["spec", "controlPlane", "controllerManager"], ["spec", "controlPlane", "controllerManager", "resources"], ["spec", "controlPlane", "controllerManager", "resourcesPreset"], ["spec", "controlPlane", "scheduler"], ["spec", "controlPlane", "scheduler", "resources"], ["spec", "controlPlane", "scheduler", "resourcesPreset"], ["spec", "controlPlane", "konnectivity"], ["spec", "controlPlane", "konnectivity", "server"], ["spec", "controlPlane", "konnectivity", "server", "resources"], ["spec", "controlPlane", "konnectivity", "server", "resourcesPreset"], ["spec", "images"], ["spec", "images", "waitForKubeconfig"]] + keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "storageClass"], ["spec", "nodeGroups"], ["spec", "nodeGroups", "md0"], ["spec", "nodeGroups", "md0", "minReplicas"], ["spec", "nodeGroups", "md0", "maxReplicas"], ["spec", "nodeGroups", "md0", "instanceType"], ["spec", "nodeGroups", "md0", "ephemeralStorage"], ["spec", "nodeGroups", "md0", "roles"], ["spec", "nodeGroups", "md0", "resources"], ["spec", "nodeGroups", "md0", "gpus"], ["spec", "version"], ["spec", "host"], ["spec", "addons"], ["spec", "addons", "certManager"], ["spec", "addons", "certManager", "enabled"], ["spec", "addons", "certManager", "valuesOverride"], ["spec", "addons", "cilium"], ["spec", "addons", "cilium", "valuesOverride"], ["spec", "addons", "gatewayAPI"], ["spec", "addons", "gatewayAPI", "enabled"], ["spec", "addons", "ingressNginx"], ["spec", "addons", "ingressNginx", "enabled"], ["spec", "addons", "ingressNginx", "exposeMethod"], ["spec", "addons", "ingressNginx", "hosts"], ["spec", "addons", "ingressNginx", "valuesOverride"], ["spec", "addons", "gpuOperator"], ["spec", "addons", "gpuOperator", "enabled"], ["spec", "addons", "gpuOperator", "valuesOverride"], ["spec", "addons", "fluxcd"], ["spec", "addons", "fluxcd", "enabled"], ["spec", "addons", "fluxcd", "valuesOverride"], ["spec", "addons", "monitoringAgents"], ["spec", "addons", "monitoringAgents", "enabled"], ["spec", "addons", "monitoringAgents", "valuesOverride"], ["spec", "addons", "verticalPodAutoscaler"], ["spec", "addons", "verticalPodAutoscaler", "valuesOverride"], ["spec", "addons", "velero"], ["spec", "addons", "velero", "enabled"], ["spec", "addons", "velero", "valuesOverride"], ["spec", "addons", "coredns"], ["spec", "addons", "coredns", "valuesOverride"], ["spec", "controlPlane"], ["spec", "controlPlane", "replicas"], ["spec", "controlPlane", "apiServer"], ["spec", "controlPlane", "apiServer", "resources"], ["spec", "controlPlane", "apiServer", "resourcesPreset"], ["spec", "controlPlane", "controllerManager"], ["spec", "controlPlane", "controllerManager", "resources"], ["spec", "controlPlane", "controllerManager", "resourcesPreset"], ["spec", "controlPlane", "scheduler"], ["spec", "controlPlane", "scheduler", "resources"], ["spec", "controlPlane", "scheduler", "resourcesPreset"], ["spec", "controlPlane", "konnectivity"], ["spec", "controlPlane", "konnectivity", "server"], ["spec", "controlPlane", "konnectivity", "server", "resources"], ["spec", "controlPlane", "konnectivity", "server", "resourcesPreset"]] secrets: exclude: [] include: diff --git a/packages/system/kubevirt-csi-node/values.yaml b/packages/system/kubevirt-csi-node/values.yaml index 56e28543..ad7e6f01 100644 --- a/packages/system/kubevirt-csi-node/values.yaml +++ b/packages/system/kubevirt-csi-node/values.yaml @@ -1,3 +1,3 @@ storageClass: replicated csiDriver: - image: ghcr.io/cozystack/cozystack/kubevirt-csi-driver:0.0.0@sha256:72154a97054e16cdf3dea6129d962b8d7e86b55cf9386095e8ac2ce7c8b69172 + image: ghcr.io/cozystack/cozystack/kubevirt-csi-driver:0.0.0@sha256:5b209248acba3d8cf0d999edd8a915915a2e565a1c8ac1f5dde0dffec20fa02e diff --git a/packages/system/lineage-controller-webhook/Makefile b/packages/system/lineage-controller-webhook/Makefile index cdc1cf01..d5bada31 100644 --- a/packages/system/lineage-controller-webhook/Makefile +++ b/packages/system/lineage-controller-webhook/Makefile @@ -4,13 +4,8 @@ NAMESPACE=cozy-system include ../../../hack/common-envs.mk include ../../../hack/package.mk -.PHONY: test - image: image-lineage-controller-webhook -test: - helm unittest . - image-lineage-controller-webhook: docker buildx build -f images/lineage-controller-webhook/Dockerfile ../../.. \ --tag $(REGISTRY)/lineage-controller-webhook:$(call settag,$(TAG)) \ diff --git a/packages/system/lineage-controller-webhook/README.md b/packages/system/lineage-controller-webhook/README.md deleted file mode 100644 index 1e43c9c8..00000000 --- a/packages/system/lineage-controller-webhook/README.md +++ /dev/null @@ -1,67 +0,0 @@ -# lineage-controller-webhook - -Cozystack system package for the **lineage controller webhook** — a mutating -admission webhook that stamps "lineage" labels onto tenant workloads, linking -each resource back to the Cozystack `Application` that ultimately owns it. - -The webhook intercepts CREATE and UPDATE on `pods`, `secrets`, `services`, -`persistentvolumeclaims`, `ingresses.networking.k8s.io`, and -`workloadmonitors.cozystack.io` outside system namespaces. For each request it -walks the ownership graph upward (Kubernetes `ownerReferences`, then the -`HelmRelease` Flux installed the resource with, then the Cozystack -`Application`-derived CRD that produced the HelmRelease) and writes the -discovered application's group, kind and name as labels -(`apps.cozystack.io/application.{group,kind,name}`) on the incoming object. -Those labels let the aggregated API server, the Cozystack dashboard, and other -lineage-aware consumers reason about which application a resource belongs to. - -The webhook serves TLS on port 9443 with a cert-manager issued certificate, and -runs with `failurePolicy: Fail` — every CREATE/UPDATE on the resources above -is gated on the webhook being reachable. - -## Topology - -The chart ships a single shape, modelled on `cozystack-api`: - -- **Deployment** with two replicas (override via `replicas`). -- **Soft `nodeAffinity`** preferring `node-role.kubernetes.io/control-plane` - (`Exists`, so it matches both Talos's empty value and k3s/kubeadm's `"true"`). - Soft means: the pod lands on a control-plane node when one is reachable, and - on any worker otherwise — no override needed for managed Kubernetes, - Cozy-in-Cozy tenants, or any other cluster where control-plane nodes aren't - visible. -- **Permissive `tolerations`** (`[{operator: Exists}]`) so the pod can land on - tainted control-plane nodes when the soft affinity is satisfiable. -- **Soft `podAntiAffinity`** on `kubernetes.io/hostname` so replicas spread - across nodes when possible (best-effort). -- **`PodDisruptionBudget`** with `maxUnavailable: 1`, unconditional. At - `replicas: 1` it's a useful no-op (allows full drain); at `replicas >= 2` - it caps disruption to one pod. -- **`Service` with `spec.trafficDistribution: PreferClose`** so the apiserver - prefers a webhook endpoint on its own node when one exists, and transparently - falls over to a remote endpoint otherwise. Requires Kubernetes ≥ 1.31; on - older clusters the field is silently ignored and traffic uses default - cluster-wide distribution. - -## Parameters - -All keys live under the top-level `lineageControllerWebhook:` map. - -| Name | Description | Value | -| ----------------------------- | --------------------------------------------------------- | -------------------------------------------------------------------------------- | -| `image` | Container image (digest-pinned) | `ghcr.io/cozystack/cozystack/lineage-controller-webhook:v1.3.0@sha256:e898…6fb0` | -| `debug` | Enable `--zap-log-level=debug` instead of `info` | `false` | -| `replicas` | Deployment replica count | `2` | -| `localK8sAPIEndpoint.enabled` | **Deprecated.** See note below. | `false` | - -### `localK8sAPIEndpoint.enabled` (deprecated) - -When enabled, this injects `KUBERNETES_SERVICE_HOST=status.hostIP` and -`KUBERNETES_SERVICE_PORT=6443` so the webhook talks to the kube-apiserver on -its own node. It was originally added to avoid latency on the -webhook-to-apiserver path, but it is only valid when the pod is actually -scheduled on an apiserver-bearing node — which the chart's soft control-plane -affinity no longer guarantees. With this flag enabled and the pod scheduled -off a control-plane node, the controller will crash-loop dialing a non- -apiserver hostIP. Slated for removal once the latency motivation is addressed -in the webhook itself; **leave disabled**. diff --git a/packages/system/lineage-controller-webhook/templates/workload.yaml b/packages/system/lineage-controller-webhook/templates/daemonset.yaml similarity index 67% rename from packages/system/lineage-controller-webhook/templates/workload.yaml rename to packages/system/lineage-controller-webhook/templates/daemonset.yaml index 097919c6..860aee6e 100644 --- a/packages/system/lineage-controller-webhook/templates/workload.yaml +++ b/packages/system/lineage-controller-webhook/templates/daemonset.yaml @@ -1,11 +1,10 @@ apiVersion: apps/v1 -kind: Deployment +kind: DaemonSet metadata: name: lineage-controller-webhook labels: app: lineage-controller-webhook spec: - replicas: {{ .Values.lineageControllerWebhook.replicas }} selector: matchLabels: app: lineage-controller-webhook @@ -14,25 +13,13 @@ spec: labels: app: lineage-controller-webhook spec: - serviceAccountName: lineage-controller-webhook + {{- with .Values.lineageControllerWebhook.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} tolerations: - operator: Exists - affinity: - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - preference: - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - topologyKey: kubernetes.io/hostname - labelSelector: - matchLabels: - app: lineage-controller-webhook + serviceAccountName: lineage-controller-webhook containers: - name: lineage-controller-webhook image: "{{ .Values.lineageControllerWebhook.image }}" diff --git a/packages/system/lineage-controller-webhook/templates/pdb.yaml b/packages/system/lineage-controller-webhook/templates/pdb.yaml deleted file mode 100644 index 0786c8ff..00000000 --- a/packages/system/lineage-controller-webhook/templates/pdb.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: policy/v1 -kind: PodDisruptionBudget -metadata: - name: lineage-controller-webhook -spec: - maxUnavailable: 1 - selector: - matchLabels: - app: lineage-controller-webhook diff --git a/packages/system/lineage-controller-webhook/templates/service.yaml b/packages/system/lineage-controller-webhook/templates/service.yaml index fbb3ad54..c5df90fb 100644 --- a/packages/system/lineage-controller-webhook/templates/service.yaml +++ b/packages/system/lineage-controller-webhook/templates/service.yaml @@ -5,7 +5,7 @@ metadata: labels: app: lineage-controller-webhook spec: - trafficDistribution: PreferClose + internalTrafficPolicy: Local type: ClusterIP ports: - port: 443 diff --git a/packages/system/lineage-controller-webhook/tests/service_test.yaml b/packages/system/lineage-controller-webhook/tests/service_test.yaml deleted file mode 100644 index 3cdbddd8..00000000 --- a/packages/system/lineage-controller-webhook/tests/service_test.yaml +++ /dev/null @@ -1,30 +0,0 @@ -suite: lineage-controller-webhook service -templates: - - templates/service.yaml - -release: - name: lineage-controller-webhook - namespace: cozy-system - -tests: - - it: renders a ClusterIP service on 443 -> 9443 with PreferClose traffic distribution - asserts: - - isKind: - of: Service - - equal: - path: spec.type - value: ClusterIP - - equal: - path: spec.trafficDistribution - value: PreferClose - - notExists: - path: spec.internalTrafficPolicy - - equal: - path: spec.ports[0].port - value: 443 - - equal: - path: spec.ports[0].targetPort - value: 9443 - - equal: - path: spec.selector.app - value: lineage-controller-webhook diff --git a/packages/system/lineage-controller-webhook/tests/workload_test.yaml b/packages/system/lineage-controller-webhook/tests/workload_test.yaml deleted file mode 100644 index bcb24014..00000000 --- a/packages/system/lineage-controller-webhook/tests/workload_test.yaml +++ /dev/null @@ -1,99 +0,0 @@ -suite: lineage-controller-webhook workload + PDB -templates: - - templates/workload.yaml - - templates/pdb.yaml - -release: - name: lineage-controller-webhook - namespace: cozy-system - -tests: - # ---- Default Deployment shape ---------------------------------------------- - - - it: defaults render a 2-replica Deployment with soft control-plane nodeAffinity, soft podAntiAffinity, and Exists tolerations - template: templates/workload.yaml - asserts: - - isKind: - of: Deployment - - equal: - path: spec.replicas - value: 2 - - equal: - path: spec.template.spec.tolerations - value: - - operator: Exists - - equal: - path: spec.template.spec.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution - value: - - weight: 100 - preference: - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - - notExists: - path: spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution - - equal: - path: spec.template.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.topologyKey - value: kubernetes.io/hostname - - - it: defaults inject no localK8sAPIEndpoint env vars (knob is opt-in) - template: templates/workload.yaml - asserts: - - notExists: - path: spec.template.spec.containers[0].env - - # ---- PDB -------------------------------------------------------------------- - - - it: PDB renders unconditionally with maxUnavailable=1 at the default replica count - template: templates/pdb.yaml - asserts: - - isKind: - of: PodDisruptionBudget - - equal: - path: spec.maxUnavailable - value: 1 - - notExists: - path: spec.minAvailable - - - it: PDB still renders at replicas=1 (no-op, but consistent shape) - set: - lineageControllerWebhook.replicas: 1 - template: templates/pdb.yaml - asserts: - - isKind: - of: PodDisruptionBudget - - equal: - path: spec.maxUnavailable - value: 1 - - # ---- Replica override ------------------------------------------------------- - - - it: replicas value drives spec.replicas - set: - lineageControllerWebhook.replicas: 5 - template: templates/workload.yaml - asserts: - - equal: - path: spec.replicas - value: 5 - - # ---- Deprecated localK8sAPIEndpoint knob ----------------------------------- - - - it: localK8sAPIEndpoint.enabled=true injects KUBERNETES_SERVICE_HOST and PORT env vars - set: - lineageControllerWebhook.localK8sAPIEndpoint.enabled: true - template: templates/workload.yaml - asserts: - - contains: - path: spec.template.spec.containers[0].env - content: - name: KUBERNETES_SERVICE_HOST - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.hostIP - - contains: - path: spec.template.spec.containers[0].env - content: - name: KUBERNETES_SERVICE_PORT - value: "6443" diff --git a/packages/system/lineage-controller-webhook/values.yaml b/packages/system/lineage-controller-webhook/values.yaml index e56d332d..103dbf85 100644 --- a/packages/system/lineage-controller-webhook/values.yaml +++ b/packages/system/lineage-controller-webhook/values.yaml @@ -1,13 +1,10 @@ lineageControllerWebhook: - image: ghcr.io/cozystack/cozystack/lineage-controller-webhook:v1.3.0@sha256:e8984709686a5eaf19b89da378d7b8c688ea5607e0783a88d9c9e4ccfca96fb0 + image: ghcr.io/cozystack/cozystack/lineage-controller-webhook:v1.2.1@sha256:0fef727af2ea8d87503ba998d2ec03c12eb432d03b6642a555b81b95fbec4cb0 debug: false - replicas: 2 - # DEPRECATED. Injects KUBERNETES_SERVICE_HOST=status.hostIP and - # KUBERNETES_SERVICE_PORT=6443 so the webhook talks to the apiserver on its - # own node — only valid when the pod is actually scheduled on an apiserver- - # bearing node. Now that the chart's nodeAffinity to control-plane is soft - # (preferred, not required), the pod can land off-control-plane and crash- - # loop dialing a non-apiserver hostIP. Slated for removal once webhook - # performance work obviates the locality motivation. Leave disabled. localK8sAPIEndpoint: - enabled: false + enabled: true + # nodeSelector for the DaemonSet + # Talos uses empty value: "node-role.kubernetes.io/control-plane": "" + # Generic k8s (k3s, kubeadm) uses: "node-role.kubernetes.io/control-plane": "true" + nodeSelector: + node-role.kubernetes.io/control-plane: "" diff --git a/packages/system/linstor-gui/Chart.yaml b/packages/system/linstor-gui/Chart.yaml deleted file mode 100644 index e361a819..00000000 --- a/packages/system/linstor-gui/Chart.yaml +++ /dev/null @@ -1,3 +0,0 @@ -apiVersion: v2 -name: cozy-linstor-gui -version: 0.0.0 # Placeholder, the actual version will be automatically set during the build process diff --git a/packages/system/linstor-gui/Makefile b/packages/system/linstor-gui/Makefile deleted file mode 100644 index b408491d..00000000 --- a/packages/system/linstor-gui/Makefile +++ /dev/null @@ -1,29 +0,0 @@ -export NAME=linstor-gui -export NAMESPACE=cozy-linstor - -include ../../../hack/common-envs.mk -include ../../../hack/package.mk - -LINSTOR_GUI_VERSION ?= 2.3.0 - -.PHONY: image image-linstor-gui test - -image: image-linstor-gui - -image-linstor-gui: - docker buildx build images/linstor-gui \ - --build-arg LINSTOR_GUI_VERSION=$(LINSTOR_GUI_VERSION) \ - --tag $(REGISTRY)/linstor-gui:$(call settag,$(LINSTOR_GUI_VERSION)) \ - --tag $(REGISTRY)/linstor-gui:$(call settag,$(LINSTOR_GUI_VERSION)-$(TAG)) \ - --cache-from type=registry,ref=$(REGISTRY)/linstor-gui:latest \ - --cache-to type=inline \ - --metadata-file images/linstor-gui.json \ - $(BUILDX_ARGS) - REPOSITORY="$(REGISTRY)/linstor-gui" \ - yq -i '.image.repository = strenv(REPOSITORY)' values.yaml - TAG="$(call settag,$(LINSTOR_GUI_VERSION))@$$(yq e '."containerimage.digest"' images/linstor-gui.json -o json -r)" \ - yq -i '.image.tag = strenv(TAG)' values.yaml - rm -f images/linstor-gui.json - -test: - helm unittest . diff --git a/packages/system/linstor-gui/README.md b/packages/system/linstor-gui/README.md deleted file mode 100644 index 8f8053ab..00000000 --- a/packages/system/linstor-gui/README.md +++ /dev/null @@ -1,64 +0,0 @@ -# linstor-gui - -Cozystack system package for [LINBIT/linstor-gui](https://github.com/LINBIT/linstor-gui) -— a web UI for managing LINSTOR nodes, resources, volumes and snapshots. - -Installed alongside the `linstor` package in the `cozy-linstor` namespace. The UI -proxies the LINSTOR controller REST API at `https://linstor-controller.cozy-linstor.svc:3371` -using mTLS with the `linstor-client-tls` secret created by the `linstor` package. - -## Exposing the UI - -### Option 1 — Keycloak-protected Ingress (recommended) - -The chart ships an `oauth2-proxy` based gatekeeper plus a `KeycloakClient` CRD -so the UI can be published on `linstor-gui.` behind the cluster -Keycloak realm. Access is restricted to members of the -`cozystack-cluster-admin` Keycloak group — the same group that grants -cluster-admin RBAC on the host cluster. Authenticating against the `cozy` -realm alone is not sufficient; users outside that group receive a 403 from -oauth2-proxy before any request reaches the UI or the LINSTOR controller. - -To turn it on, add `linstor-gui` to `publishing.exposedServices` in the core -`cozystack` values (same list that controls `dashboard`). OIDC must be -enabled (`authentication.oidc.enabled: true`) — if it is not, the Ingress and -gatekeeper Deployment are deliberately **not** rendered, because the LINSTOR -REST API surface must not be exposed unauthenticated. - -Once enabled, the UI is reachable at `https://linstor-gui.` and -authentication is delegated to Keycloak via the `linstor-gui` client -(auto-provisioned through the `KeycloakClient` CRD; the client secret is -persisted in the `linstor-gui-client` Secret in `cozy-linstor`). - -### Option 2 — Port-forward - -If you have not set up Keycloak or want ad-hoc access, use the `ClusterIP` -Service: - -```bash -kubectl -n cozy-linstor port-forward svc/linstor-gui 3373:80 -``` - -then open . - -## Parameters - -### Image - -| Name | Description | Value | -| ------------------ | ---------------------------------------------------------- | ----------------------------------------- | -| `image.repository` | LINSTOR GUI container image repository | `ghcr.io/cozystack/cozystack/linstor-gui` | -| `image.tag` | LINSTOR GUI container image tag (digest recommended) | `2.3.0` | - -### Deployment - -| Name | Description | Value | -| ---------- | ------------------------------- | ----- | -| `replicas` | Number of linstor-gui replicas | `1` | - -### LINSTOR controller connection - -| Name | Description | Value | -| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------- | -| `linstor.endpoint` | In-cluster URL of the LINSTOR controller REST API (HTTPS, mTLS) | `https://linstor-controller.cozy-linstor.svc:3371` | -| `linstor.clientSecret` | Kubernetes Secret with `tls.crt`, `tls.key`, `ca.crt` used as the mTLS client certificate against the LINSTOR controller. Created by the `linstor` package. | `linstor-client-tls` | diff --git a/packages/system/linstor-gui/images/linstor-gui/Dockerfile b/packages/system/linstor-gui/images/linstor-gui/Dockerfile deleted file mode 100644 index 2082ef0d..00000000 --- a/packages/system/linstor-gui/images/linstor-gui/Dockerfile +++ /dev/null @@ -1,19 +0,0 @@ -# Upstream: https://github.com/LINBIT/linstor-gui (GPL-3.0) -# Serves the pre-built LINSTOR GUI tarball published by LINBIT from a hardened -# nginx-unprivileged image. nginx.conf is supplied by the chart via ConfigMap, -# so the upstream docker-entrypoint.sh / nginx.conf.template is not used. -FROM nginxinc/nginx-unprivileged:1.29-alpine - -ARG LINSTOR_GUI_VERSION -USER root -RUN apk add --no-cache curl tar && \ - curl -fsSL "https://pkg.linbit.com/downloads/linstor/linstor-gui-${LINSTOR_GUI_VERSION}.tar.gz" -o /tmp/linstor-gui.tar.gz && \ - mkdir -p /usr/share/nginx/html && \ - tar -xzf /tmp/linstor-gui.tar.gz -C /usr/share/nginx/html --strip-components=2 "linstor-gui-${LINSTOR_GUI_VERSION}/dist" && \ - rm -f /tmp/linstor-gui.tar.gz && \ - apk del curl tar && \ - chown -R 101:101 /usr/share/nginx/html - -USER 101 -EXPOSE 3373 -CMD ["nginx", "-g", "daemon off;"] diff --git a/packages/system/linstor-gui/templates/_helpers.tpl b/packages/system/linstor-gui/templates/_helpers.tpl deleted file mode 100644 index 6cad99ee..00000000 --- a/packages/system/linstor-gui/templates/_helpers.tpl +++ /dev/null @@ -1,17 +0,0 @@ -{{/* -Common labels -*/}} -{{- define "linstor-gui.labels" -}} -app.kubernetes.io/name: linstor-gui -app.kubernetes.io/instance: {{ .Release.Name }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -app.kubernetes.io/part-of: cozystack -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "linstor-gui.selectorLabels" -}} -app.kubernetes.io/name: linstor-gui -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} diff --git a/packages/system/linstor-gui/templates/configmap-nginx.yaml b/packages/system/linstor-gui/templates/configmap-nginx.yaml deleted file mode 100644 index 86acd0f7..00000000 --- a/packages/system/linstor-gui/templates/configmap-nginx.yaml +++ /dev/null @@ -1,113 +0,0 @@ -{{/* -Parse the LINSTOR endpoint so nginx can set `proxy_ssl_name` correctly. -The endpoint must be an https:// URL — http:// would downgrade the -mTLS-protected controller path. -*/}} -{{- if not (hasPrefix "https://" .Values.linstor.endpoint) -}} -{{- fail "linstor.endpoint must start with https:// to enforce mTLS to the LINSTOR controller" -}} -{{- end -}} -{{- $endpoint := trimPrefix "https://" .Values.linstor.endpoint -}} -{{- $endpointHost := (splitList ":" $endpoint) | first -}} -apiVersion: v1 -kind: ConfigMap -metadata: - name: linstor-gui-nginx - labels: - {{- include "linstor-gui.labels" . | nindent 4 }} -data: - nginx.conf: | - worker_processes auto; - pid /tmp/nginx.pid; - - events { - worker_connections 1024; - } - - http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - server_tokens off; - - client_body_temp_path /tmp/client_body; - proxy_temp_path /tmp/proxy; - fastcgi_temp_path /tmp/fastcgi; - uwsgi_temp_path /tmp/uwsgi; - scgi_temp_path /tmp/scgi; - - sendfile on; - keepalive_timeout 65; - - server { - listen 3373; - server_name _; - - root /usr/share/nginx/html; - index index.html; - - # Shared proxy + mTLS settings for the LINSTOR controller upstream - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_ssl_certificate /etc/linstor/client/tls.crt; - proxy_ssl_certificate_key /etc/linstor/client/tls.key; - proxy_ssl_trusted_certificate /etc/linstor/client/ca.crt; - proxy_ssl_verify on; - proxy_ssl_verify_depth 2; - proxy_ssl_server_name on; - proxy_ssl_name {{ $endpointHost }}; - - # Static UI assets - location / { - try_files $uri $uri/ /index.html; - } - - # Lock the linstor-gui in-app authentication panel. - # - # Unlike most LINSTOR features, the GUI's "authenticationEnabled" - # toggle and user list are NOT controller-side properties — they - # are persisted as ordinary KeyValueStore instances named - # `__gui__settings` and `__gui__users`. The SPA reads them on - # every page load and, if `authenticationEnabled=true`, gates the - # whole UI behind its own login screen against a self-managed - # admin user. - # - # Cozystack already enforces authentication at the Ingress - # (oauth2-proxy + Keycloak), and the in-app login is a foot-gun: - # if a user enables it and forgets/misconfigures the password, the - # only recovery is shelling into the linstor-controller pod and - # running `linstor key-value-store modify __gui__settings - # authenticationEnabled false`. - # - # We allow GETs (so the SPA can read state and render normally - # with auth=off) but reject every mutating method on those two - # KeyValueStore instances. Any other key-value-store entry the - # GUI uses (e.g. __gui__mode) keeps working as expected. - location ~ ^/v1/key-value-store/__gui__(settings|users)(/|$) { - default_type application/json; - # `if ... return` is the one documented-safe use of `if` in - # nginx (see "If is Evil"), so it's fine here. We allow GET/HEAD - # for the SPA's read path and reject every mutating method. - if ($request_method !~ ^(GET|HEAD)$) { - return 403 '{"ret_code":-1,"message":"linstor-gui in-app authentication is disabled by cozystack: authentication is enforced at the Ingress via Keycloak."}'; - } - proxy_pass {{ .Values.linstor.endpoint }}; - } - - # Proxy LINSTOR REST API over mTLS to the controller - location /v1 { - proxy_pass {{ .Values.linstor.endpoint }}; - } - - location /metrics { - proxy_pass {{ .Values.linstor.endpoint }}; - } - - location = /healthz { - access_log off; - # default_type must come BEFORE `return` — `add_header` after a - # `return` is silently dropped by nginx because the response is - # already finalized. - default_type text/plain; - return 200 'ok'; - } - } - } diff --git a/packages/system/linstor-gui/templates/deployment.yaml b/packages/system/linstor-gui/templates/deployment.yaml deleted file mode 100644 index 92e2f7a6..00000000 --- a/packages/system/linstor-gui/templates/deployment.yaml +++ /dev/null @@ -1,100 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: linstor-gui - labels: - {{- include "linstor-gui.labels" . | nindent 4 }} - annotations: - reloader.stakater.com/auto: "true" -spec: - replicas: {{ .Values.replicas }} - selector: - matchLabels: - {{- include "linstor-gui.selectorLabels" . | nindent 6 }} - strategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 0 - maxSurge: 1 - template: - metadata: - labels: - {{- include "linstor-gui.labels" . | nindent 8 }} - spec: - serviceAccountName: linstor-gui - automountServiceAccountToken: false - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchLabels: - {{- include "linstor-gui.selectorLabels" . | nindent 20 }} - topologyKey: kubernetes.io/hostname - containers: - - name: linstor-gui - image: {{ .Values.image.repository }}:{{ .Values.image.tag }} - imagePullPolicy: IfNotPresent - ports: - - name: http - containerPort: 3373 - protocol: TCP - resources: - requests: - cpu: 50m - memory: 64Mi - limits: - cpu: 200m - memory: 128Mi - livenessProbe: - httpGet: - path: /healthz - port: http - initialDelaySeconds: 5 - periodSeconds: 20 - readinessProbe: - httpGet: - path: /healthz - port: http - initialDelaySeconds: 2 - periodSeconds: 10 - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - runAsNonRoot: true - runAsUser: 101 - runAsGroup: 101 - capabilities: - drop: - - ALL - volumeMounts: - - name: nginx-config - mountPath: /etc/nginx/nginx.conf - subPath: nginx.conf - readOnly: true - - name: linstor-client-tls - mountPath: /etc/linstor/client - readOnly: true - - name: tmp - mountPath: /tmp - - name: nginx-cache - mountPath: /var/cache/nginx - volumes: - - name: nginx-config - configMap: - name: linstor-gui-nginx - - name: linstor-client-tls - secret: - secretName: {{ .Values.linstor.clientSecret }} - # 0444 so nginx (UID 101) can read the mTLS client keypair; - # the secret volume is pod-local, not a broader disclosure. - defaultMode: 0444 - - name: tmp - emptyDir: {} - - name: nginx-cache - emptyDir: {} diff --git a/packages/system/linstor-gui/templates/gatekeeper-sa.yaml b/packages/system/linstor-gui/templates/gatekeeper-sa.yaml deleted file mode 100644 index d89f68dc..00000000 --- a/packages/system/linstor-gui/templates/gatekeeper-sa.yaml +++ /dev/null @@ -1,13 +0,0 @@ -{{- $oidcEnabled := index .Values._cluster "oidc-enabled" }} -{{- if eq $oidcEnabled "true" }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: linstor-gui-gatekeeper - labels: - app.kubernetes.io/instance: linstor-gui - app.kubernetes.io/name: gatekeeper - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/part-of: cozystack -automountServiceAccountToken: false -{{- end }} diff --git a/packages/system/linstor-gui/templates/gatekeeper-svc.yaml b/packages/system/linstor-gui/templates/gatekeeper-svc.yaml deleted file mode 100644 index 7d50219e..00000000 --- a/packages/system/linstor-gui/templates/gatekeeper-svc.yaml +++ /dev/null @@ -1,22 +0,0 @@ -{{- $oidcEnabled := index .Values._cluster "oidc-enabled" }} -{{- if eq $oidcEnabled "true" }} -apiVersion: v1 -kind: Service -metadata: - name: linstor-gui-gatekeeper - labels: - app.kubernetes.io/instance: linstor-gui - app.kubernetes.io/name: gatekeeper - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/part-of: cozystack -spec: - type: ClusterIP - ports: - - name: ingress - port: 8000 - protocol: TCP - targetPort: 8000 - selector: - app.kubernetes.io/instance: linstor-gui - app.kubernetes.io/name: gatekeeper -{{- end }} diff --git a/packages/system/linstor-gui/templates/gatekeeper.yaml b/packages/system/linstor-gui/templates/gatekeeper.yaml deleted file mode 100644 index 81451f41..00000000 --- a/packages/system/linstor-gui/templates/gatekeeper.yaml +++ /dev/null @@ -1,148 +0,0 @@ -{{- $host := index .Values._cluster "root-host" }} -{{- $oidcEnabled := index .Values._cluster "oidc-enabled" }} -{{- $oidcInsecureSkipVerify := index .Values._cluster "oidc-insecure-skip-verify" }} -{{- $keycloakInternalUrl := index .Values._cluster "keycloak-internal-url" | default "" }} - -{{- /* -Gatekeeper (oauth2-proxy) fronts the linstor-gui Service and handles the -OIDC flow against the cluster Keycloak realm. The Ingress below points at -this Service — not at linstor-gui directly — so every external request -authenticates before hitting nginx/mTLS→controller. - -Only rendered when oidc-enabled=true. If the cluster has OIDC disabled we -deliberately do not ship an unauthenticated token-proxy fallback (unlike -dashboard): linstor-gui surfaces raw storage state and should not be -reachable via Ingress without Keycloak login. - -Access is further restricted to the cozystack-cluster-admin group -(--allowed-group below). The proxy authenticates with a static mTLS client -cert and LINSTOR itself has no per-user RBAC, so group membership is the -only boundary between a realm user and raw storage state. -*/ -}} -{{- if eq $oidcEnabled "true" }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: linstor-gui-gatekeeper - labels: - app.kubernetes.io/instance: linstor-gui - app.kubernetes.io/name: gatekeeper - app.kubernetes.io/managed-by: {{ .Release.Service }} - app.kubernetes.io/part-of: cozystack - annotations: - reloader.stakater.com/auto: "true" -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/instance: linstor-gui - app.kubernetes.io/name: gatekeeper - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - template: - metadata: - labels: - app.kubernetes.io/instance: linstor-gui - app.kubernetes.io/name: gatekeeper - spec: - serviceAccountName: linstor-gui-gatekeeper - automountServiceAccountToken: false - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchLabels: - app.kubernetes.io/instance: linstor-gui - app.kubernetes.io/name: gatekeeper - topologyKey: kubernetes.io/hostname - containers: - - name: auth-proxy - image: quay.io/oauth2-proxy/oauth2-proxy:v7.12.0 - imagePullPolicy: IfNotPresent - args: - - --provider=oidc - - --upstream=http://linstor-gui.{{ .Release.Namespace }}.svc:80 - - --http-address=0.0.0.0:8000 - - --redirect-url=https://linstor-gui.{{ $host }}/oauth2/callback - - --oidc-issuer-url=https://keycloak.{{ $host }}/realms/cozy - {{- if $keycloakInternalUrl }} - - --skip-oidc-discovery - - --login-url=https://keycloak.{{ $host }}/realms/cozy/protocol/openid-connect/auth - - --redeem-url={{ $keycloakInternalUrl }}/protocol/openid-connect/token - - --oidc-jwks-url={{ $keycloakInternalUrl }}/protocol/openid-connect/certs - - --validate-url={{ $keycloakInternalUrl }}/protocol/openid-connect/userinfo - - --backend-logout-url={{ $keycloakInternalUrl }}/protocol/openid-connect/logout?id_token_hint={id_token} - {{- else }} - - --backend-logout-url=https://keycloak.{{ $host }}/realms/cozy/protocol/openid-connect/logout?id_token_hint={id_token} - {{- end }} - - --whitelist-domain=keycloak.{{ $host }} - - --email-domain=* - - --pass-access-token=true - - --pass-authorization-header=true - - --cookie-refresh=3m - - --cookie-name=kc-access - - --cookie-secure=true - - --cookie-secret=$(OAUTH2_PROXY_COOKIE_SECRET) - - --skip-provider-button - - --scope=openid email profile groups offline_access - - --allowed-group=cozystack-cluster-admin - {{- if eq $oidcInsecureSkipVerify "true" }} - - --ssl-insecure-skip-verify=true - {{- end }} - env: - - name: OAUTH2_PROXY_CLIENT_ID - value: linstor-gui - - name: OAUTH2_PROXY_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: linstor-gui-client - key: client-secret-key - - name: OAUTH2_PROXY_COOKIE_SECRET - valueFrom: - secretKeyRef: - name: linstor-gui-auth-config - key: cookieSecret - ports: - - name: proxy - containerPort: 8000 - protocol: TCP - livenessProbe: - httpGet: - path: /ping - port: proxy - initialDelaySeconds: 10 - periodSeconds: 15 - timeoutSeconds: 2 - failureThreshold: 3 - readinessProbe: - httpGet: - path: /ping - port: proxy - initialDelaySeconds: 5 - periodSeconds: 10 - timeoutSeconds: 2 - failureThreshold: 3 - resources: - requests: - cpu: 50m - memory: 64Mi - limits: - cpu: 200m - memory: 256Mi - securityContext: - runAsNonRoot: true - runAsUser: 1001 - runAsGroup: 1001 - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - capabilities: - drop: - - ALL - seccompProfile: - type: RuntimeDefault -{{- end }} diff --git a/packages/system/linstor-gui/templates/ingress.yaml b/packages/system/linstor-gui/templates/ingress.yaml deleted file mode 100644 index c50b0f86..00000000 --- a/packages/system/linstor-gui/templates/ingress.yaml +++ /dev/null @@ -1,54 +0,0 @@ -{{- $solver := (index .Values._cluster "solver") | default "http01" }} -{{- $clusterIssuer := (index .Values._cluster "issuer-name") | default "letsencrypt-prod" }} -{{- $host := index .Values._cluster "root-host" }} -{{- $oidcEnabled := index .Values._cluster "oidc-enabled" }} -{{- $exposeServices := splitList "," ((index .Values._cluster "expose-services") | default "") }} -{{- $exposeIngress := (index .Values._cluster "expose-ingress") | default "tenant-root" }} - -{{- /* -Only publish an Ingress when the operator has opted this service in AND -OIDC is enabled cluster-wide. If OIDC is off we deliberately do not -expose linstor-gui: without Keycloak there is no authentication layer in -front of the LINSTOR REST API proxy. Users can still reach the UI via -`kubectl port-forward` to the ClusterIP service. -*/ -}} -{{- if and (has "linstor-gui" $exposeServices) (eq $oidcEnabled "true") }} -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: linstor-gui-ingress - labels: - {{- include "linstor-gui.labels" . | nindent 4 }} - annotations: - cert-manager.io/cluster-issuer: {{ $clusterIssuer }} - {{- if eq $solver "http01" }} - acme.cert-manager.io/http01-ingress-ingressclassname: {{ $exposeIngress }} - {{- end }} - nginx.ingress.kubernetes.io/proxy-body-size: 100m - # Keycloak access+refresh+id tokens make the oauth2-proxy session - # cookie ~8–10 KB (split across multiple Set-Cookie headers), which - # overflows the default proxy_buffer_size and causes "upstream sent - # too big header" -> HTTP 502 on /oauth2/callback. Same numbers as - # dashboard ingress. - nginx.ingress.kubernetes.io/proxy-buffer-size: 100m - nginx.ingress.kubernetes.io/proxy-buffers-number: "4" - nginx.ingress.kubernetes.io/proxy-read-timeout: "3600" - nginx.ingress.kubernetes.io/proxy-send-timeout: "3600" -spec: - ingressClassName: {{ $exposeIngress }} - rules: - - host: linstor-gui.{{ $host }} - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: linstor-gui-gatekeeper - port: - number: 8000 - tls: - - hosts: - - linstor-gui.{{ $host }} - secretName: linstor-gui-ingress-tls -{{- end }} diff --git a/packages/system/linstor-gui/templates/keycloakclient.yaml b/packages/system/linstor-gui/templates/keycloakclient.yaml deleted file mode 100644 index e5dab87b..00000000 --- a/packages/system/linstor-gui/templates/keycloakclient.yaml +++ /dev/null @@ -1,72 +0,0 @@ -{{- $host := index .Values._cluster "root-host" }} - -{{- /* -Persist client + cookie secrets across upgrades: if the Secrets already -exist, reuse their values; otherwise generate new random ones. The -KeycloakClient CRD references the same client secret so Keycloak stays -in sync with what oauth2-proxy reads. -*/ -}} -{{- $existingClientSecret := lookup "v1" "Secret" .Release.Namespace "linstor-gui-client" }} -{{- $clientSecret := "" }} -{{- if $existingClientSecret }} - {{- $clientSecret = index $existingClientSecret.data "client-secret-key" | b64dec }} -{{- else }} - {{- $clientSecret = randAlphaNum 32 }} -{{- end }} - -{{- $existingAuthConfig := lookup "v1" "Secret" .Release.Namespace "linstor-gui-auth-config" }} -{{- $cookieSecret := "" }} -{{- if $existingAuthConfig }} - {{- $cookieSecret = index $existingAuthConfig.data "cookieSecret" | b64dec }} -{{- else }} - {{- $cookieSecret = randAlphaNum 16 }} -{{- end }} ---- -apiVersion: v1 -kind: Secret -metadata: - name: linstor-gui-client - labels: - {{- include "linstor-gui.labels" . | nindent 4 }} -type: Opaque -data: - client-secret-key: {{ $clientSecret | b64enc }} ---- -apiVersion: v1 -kind: Secret -metadata: - name: linstor-gui-auth-config - labels: - {{- include "linstor-gui.labels" . | nindent 4 }} -type: Opaque -data: - cookieSecret: {{ $cookieSecret | b64enc }} -{{- if .Capabilities.APIVersions.Has "v1.edp.epam.com/v1" }} ---- -apiVersion: v1.edp.epam.com/v1 -kind: KeycloakClient -metadata: - name: linstor-gui-client - annotations: - # Re-reconcile the Keycloak client if the random secret regenerates. - secret-hash: {{ $clientSecret | sha256sum }} -spec: - realmRef: - name: keycloakrealm-cozy - kind: ClusterKeycloakRealm - secret: $linstor-gui-client:client-secret-key - advancedProtocolMappers: true - name: linstor-gui - clientId: linstor-gui - directAccess: true - public: false - webUrl: "https://linstor-gui.{{ $host }}" - defaultClientScopes: - - groups - optionalClientScopes: - - offline_access - attributes: - post.logout.redirect.uris: "+" - redirectUris: - - "https://linstor-gui.{{ $host }}/oauth2/callback/*" -{{- end }} diff --git a/packages/system/linstor-gui/templates/service.yaml b/packages/system/linstor-gui/templates/service.yaml deleted file mode 100644 index 66ae2b2d..00000000 --- a/packages/system/linstor-gui/templates/service.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: linstor-gui - labels: - {{- include "linstor-gui.labels" . | nindent 4 }} -spec: - type: ClusterIP - ports: - - name: http - port: 80 - targetPort: http - protocol: TCP - selector: - {{- include "linstor-gui.selectorLabels" . | nindent 4 }} diff --git a/packages/system/linstor-gui/templates/serviceaccount.yaml b/packages/system/linstor-gui/templates/serviceaccount.yaml deleted file mode 100644 index 8e472671..00000000 --- a/packages/system/linstor-gui/templates/serviceaccount.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: linstor-gui - labels: - {{- include "linstor-gui.labels" . | nindent 4 }} -automountServiceAccountToken: false diff --git a/packages/system/linstor-gui/tests/deployment_test.yaml b/packages/system/linstor-gui/tests/deployment_test.yaml deleted file mode 100644 index 50711eb3..00000000 --- a/packages/system/linstor-gui/tests/deployment_test.yaml +++ /dev/null @@ -1,95 +0,0 @@ -suite: linstor-gui deployment -templates: - - templates/deployment.yaml - - templates/service.yaml - - templates/configmap-nginx.yaml - -tests: - - it: renders a ClusterIP service on port 80 -> http - template: templates/service.yaml - release: - name: linstor-gui - namespace: cozy-linstor - asserts: - - isKind: - of: Service - - equal: - path: spec.type - value: ClusterIP - - equal: - path: spec.ports[0].port - value: 80 - - equal: - path: spec.ports[0].targetPort - value: http - - - it: mounts the LINSTOR client TLS secret into the pod - template: templates/deployment.yaml - release: - name: linstor-gui - namespace: cozy-linstor - asserts: - - isKind: - of: Deployment - - contains: - path: spec.template.spec.volumes - content: - name: linstor-client-tls - secret: - secretName: linstor-client-tls - defaultMode: 0444 - - contains: - path: spec.template.spec.containers[0].volumeMounts - content: - name: linstor-client-tls - mountPath: /etc/linstor/client - readOnly: true - - - it: runs as a non-root user with a read-only root filesystem - template: templates/deployment.yaml - release: - name: linstor-gui - namespace: cozy-linstor - asserts: - - equal: - path: spec.template.spec.containers[0].securityContext.runAsNonRoot - value: true - - equal: - path: spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem - value: true - - equal: - path: spec.template.spec.containers[0].securityContext.allowPrivilegeEscalation - value: false - - - it: proxies /v1 to the configured LINSTOR controller with mTLS - template: templates/configmap-nginx.yaml - release: - name: linstor-gui - namespace: cozy-linstor - asserts: - - isKind: - of: ConfigMap - - matchRegex: - path: data["nginx.conf"] - pattern: "location /v1" - - matchRegex: - path: data["nginx.conf"] - pattern: "proxy_pass https://linstor-controller.cozy-linstor.svc:3371" - - matchRegex: - path: data["nginx.conf"] - pattern: "proxy_ssl_certificate\\s+/etc/linstor/client/tls.crt" - - - it: overrides linstor endpoint and client secret - template: templates/configmap-nginx.yaml - release: - name: linstor-gui - namespace: cozy-linstor - set: - linstor.endpoint: https://my-controller.example.svc:3371 - asserts: - - matchRegex: - path: data["nginx.conf"] - pattern: "proxy_pass https://my-controller.example.svc:3371" - - matchRegex: - path: data["nginx.conf"] - pattern: "proxy_ssl_name my-controller.example.svc" diff --git a/packages/system/linstor-gui/tests/ingress_auth_test.yaml b/packages/system/linstor-gui/tests/ingress_auth_test.yaml deleted file mode 100644 index 2f0b1d73..00000000 --- a/packages/system/linstor-gui/tests/ingress_auth_test.yaml +++ /dev/null @@ -1,177 +0,0 @@ -suite: linstor-gui ingress + keycloak auth -templates: - - templates/ingress.yaml - - templates/gatekeeper.yaml - - templates/gatekeeper-svc.yaml - - templates/gatekeeper-sa.yaml - - templates/keycloakclient.yaml - -release: - name: linstor-gui - namespace: cozy-linstor - -tests: - - it: does not render an Ingress when OIDC is disabled cluster-wide - template: templates/ingress.yaml - set: - _cluster: - root-host: example.org - oidc-enabled: "false" - expose-services: "linstor-gui" - expose-ingress: "tenant-root" - issuer-name: "letsencrypt-prod" - solver: "http01" - asserts: - - hasDocuments: - count: 0 - - - it: does not render an Ingress when linstor-gui is not in expose-services - template: templates/ingress.yaml - set: - _cluster: - root-host: example.org - oidc-enabled: "true" - expose-services: "dashboard,api" - expose-ingress: "tenant-root" - issuer-name: "letsencrypt-prod" - solver: "http01" - asserts: - - hasDocuments: - count: 0 - - - it: renders an Ingress to the gatekeeper service when exposed and OIDC enabled - template: templates/ingress.yaml - set: - _cluster: - root-host: dev10.example.org - oidc-enabled: "true" - expose-services: "dashboard,linstor-gui" - expose-ingress: "tenant-root" - issuer-name: "letsencrypt-prod" - solver: "http01" - asserts: - - isKind: - of: Ingress - - equal: - path: spec.ingressClassName - value: tenant-root - - equal: - path: spec.rules[0].host - value: linstor-gui.dev10.example.org - - equal: - path: spec.rules[0].http.paths[0].backend.service.name - value: linstor-gui-gatekeeper - - equal: - path: spec.rules[0].http.paths[0].backend.service.port.number - value: 8000 - - equal: - path: metadata.annotations["cert-manager.io/cluster-issuer"] - value: letsencrypt-prod - - - it: does not render the gatekeeper Deployment when OIDC is disabled - template: templates/gatekeeper.yaml - set: - _cluster: - root-host: example.org - oidc-enabled: "false" - expose-services: "linstor-gui" - asserts: - - hasDocuments: - count: 0 - - - it: renders an oauth2-proxy Deployment pointing at the cluster Keycloak realm - template: templates/gatekeeper.yaml - set: - _cluster: - root-host: dev10.example.org - oidc-enabled: "true" - oidc-insecure-skip-verify: "false" - keycloak-internal-url: "" - expose-services: "linstor-gui" - asserts: - - isKind: - of: Deployment - - equal: - path: metadata.name - value: linstor-gui-gatekeeper - - matchRegex: - path: spec.template.spec.containers[0].image - pattern: "^quay\\.io/oauth2-proxy/oauth2-proxy:" - - contains: - path: spec.template.spec.containers[0].args - content: --provider=oidc - - contains: - path: spec.template.spec.containers[0].args - content: --upstream=http://linstor-gui.cozy-linstor.svc:80 - - contains: - path: spec.template.spec.containers[0].args - content: --redirect-url=https://linstor-gui.dev10.example.org/oauth2/callback - - contains: - path: spec.template.spec.containers[0].args - content: --oidc-issuer-url=https://keycloak.dev10.example.org/realms/cozy - - contains: - path: spec.template.spec.containers[0].args - content: --scope=openid email profile groups offline_access - - contains: - path: spec.template.spec.containers[0].args - content: --allowed-group=cozystack-cluster-admin - - contains: - path: spec.template.spec.containers[0].env - content: - name: OAUTH2_PROXY_CLIENT_ID - value: linstor-gui - - equal: - path: spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem - value: true - - - it: creates client + cookie Secrets and a KeycloakClient CRD - template: templates/keycloakclient.yaml - capabilities: - apiVersions: - - v1.edp.epam.com/v1 - set: - _cluster: - root-host: dev10.example.org - asserts: - - hasDocuments: - count: 3 - - documentIndex: 0 - isKind: - of: Secret - - documentIndex: 0 - equal: - path: metadata.name - value: linstor-gui-client - - documentIndex: 1 - isKind: - of: Secret - - documentIndex: 1 - equal: - path: metadata.name - value: linstor-gui-auth-config - - documentIndex: 2 - isKind: - of: KeycloakClient - - documentIndex: 2 - equal: - path: spec.clientId - value: linstor-gui - - documentIndex: 2 - contains: - path: spec.redirectUris - content: "https://linstor-gui.dev10.example.org/oauth2/callback/*" - - - it: skips the KeycloakClient CRD when the API is absent, still creates Secrets - template: templates/keycloakclient.yaml - set: - _cluster: - root-host: example.org - asserts: - - hasDocuments: - count: 2 - - documentIndex: 0 - isKind: - of: Secret - - documentIndex: 1 - isKind: - of: Secret diff --git a/packages/system/linstor-gui/values.yaml b/packages/system/linstor-gui/values.yaml deleted file mode 100644 index e3c49aea..00000000 --- a/packages/system/linstor-gui/values.yaml +++ /dev/null @@ -1,17 +0,0 @@ -## @section Image -## @param image.repository LINSTOR GUI container image repository -## @param image.tag LINSTOR GUI container image tag (digest recommended) -image: - repository: ghcr.io/cozystack/cozystack/linstor-gui - tag: 2.3.0 - -## @section Deployment -## @param replicas Number of linstor-gui replicas -replicas: 1 - -## @section LINSTOR controller connection -## @param linstor.endpoint In-cluster URL of the LINSTOR controller REST API (HTTPS, mTLS) -## @param linstor.clientSecret Kubernetes Secret with `tls.crt`, `tls.key`, `ca.crt` used as the mTLS client certificate against the LINSTOR controller. Created by the `linstor` package. -linstor: - endpoint: "https://linstor-controller.cozy-linstor.svc:3371" - clientSecret: "linstor-client-tls" diff --git a/packages/system/linstor-scheduler/Makefile b/packages/system/linstor-scheduler/Makefile index d83a2929..f9986c9a 100644 --- a/packages/system/linstor-scheduler/Makefile +++ b/packages/system/linstor-scheduler/Makefile @@ -3,13 +3,9 @@ export NAMESPACE=cozy-linstor include ../../../hack/package.mk -test: - helm unittest . - update: rm -rf charts helm repo add piraeus-charts https://piraeus.io/helm-charts/ helm repo update piraeus-charts helm pull piraeus-charts/linstor-scheduler --untar --untardir charts patch --no-backup-if-mismatch -p4 < patches/disable-ca-key-rotation.patch - patch --no-backup-if-mismatch -p4 < patches/add-scheduler-component-label.patch diff --git a/packages/system/linstor-scheduler/charts/linstor-scheduler/templates/deployment.yaml b/packages/system/linstor-scheduler/charts/linstor-scheduler/templates/deployment.yaml index 289021fd..ea43cb83 100644 --- a/packages/system/linstor-scheduler/charts/linstor-scheduler/templates/deployment.yaml +++ b/packages/system/linstor-scheduler/charts/linstor-scheduler/templates/deployment.yaml @@ -20,7 +20,6 @@ spec: {{- end }} labels: {{- include "linstor-scheduler.selectorLabels" . | nindent 8 }} - app.kubernetes.io/component: scheduler spec: {{- with .Values.imagePullSecrets }} imagePullSecrets: diff --git a/packages/system/linstor-scheduler/patches/add-scheduler-component-label.patch b/packages/system/linstor-scheduler/patches/add-scheduler-component-label.patch deleted file mode 100644 index 01eed9a8..00000000 --- a/packages/system/linstor-scheduler/patches/add-scheduler-component-label.patch +++ /dev/null @@ -1,9 +0,0 @@ -diff --git a/packages/system/linstor-scheduler/charts/linstor-scheduler/templates/deployment.yaml b/packages/system/linstor-scheduler/charts/linstor-scheduler/templates/deployment.yaml ---- a/packages/system/linstor-scheduler/charts/linstor-scheduler/templates/deployment.yaml -+++ b/packages/system/linstor-scheduler/charts/linstor-scheduler/templates/deployment.yaml -@@ -21,4 +21,5 @@ - labels: - {{- include "linstor-scheduler.selectorLabels" . | nindent 8 }} -+ app.kubernetes.io/component: scheduler - spec: - {{- with .Values.imagePullSecrets }} diff --git a/packages/system/linstor-scheduler/templates/extender-service.yaml b/packages/system/linstor-scheduler/templates/extender-service.yaml deleted file mode 100644 index e8148b43..00000000 --- a/packages/system/linstor-scheduler/templates/extender-service.yaml +++ /dev/null @@ -1,26 +0,0 @@ -{{/* - Exposes the linstor-scheduler-extender sidecar (port 8099) as a - cluster-internal Service so that other schedulers (e.g. cozystack-scheduler) - can use it as a scheduler extender. - - The selector labels match the subchart's deployment pods. They are - hardcoded here because the subchart helpers expect a subchart rendering - context that is unavailable in the wrapper chart. -*/}} -apiVersion: v1 -kind: Service -metadata: - name: linstor-scheduler-extender - labels: - app.kubernetes.io/component: extender -spec: - type: ClusterIP - ports: - - port: 8099 - targetPort: 8099 - protocol: TCP - name: http - selector: - app.kubernetes.io/name: linstor-scheduler - app.kubernetes.io/instance: linstor-scheduler - app.kubernetes.io/component: scheduler diff --git a/packages/system/linstor-scheduler/tests/extender-service_test.yaml b/packages/system/linstor-scheduler/tests/extender-service_test.yaml deleted file mode 100644 index 60e272c2..00000000 --- a/packages/system/linstor-scheduler/tests/extender-service_test.yaml +++ /dev/null @@ -1,35 +0,0 @@ -suite: extender service targets deployment pods - -templates: - - templates/extender-service.yaml - - charts/linstor-scheduler/templates/deployment.yaml - -release: - name: linstor-scheduler - -tests: - - it: service selector matches deployment pod labels - template: charts/linstor-scheduler/templates/deployment.yaml - asserts: - - equal: - path: spec.template.metadata.labels["app.kubernetes.io/name"] - value: linstor-scheduler - - equal: - path: spec.template.metadata.labels["app.kubernetes.io/instance"] - value: linstor-scheduler - - equal: - path: spec.template.metadata.labels["app.kubernetes.io/component"] - value: scheduler - - - it: service selector uses the same values - template: templates/extender-service.yaml - asserts: - - equal: - path: spec.selector["app.kubernetes.io/name"] - value: linstor-scheduler - - equal: - path: spec.selector["app.kubernetes.io/instance"] - value: linstor-scheduler - - equal: - path: spec.selector["app.kubernetes.io/component"] - value: scheduler diff --git a/packages/system/linstor/Makefile b/packages/system/linstor/Makefile index 8d566678..b211baec 100644 --- a/packages/system/linstor/Makefile +++ b/packages/system/linstor/Makefile @@ -4,8 +4,8 @@ export NAMESPACE=cozy-$(NAME) include ../../../hack/common-envs.mk include ../../../hack/package.mk -LINSTOR_VERSION ?= 1.33.2 -LINSTOR_CSI_VERSION ?= v1.10.6 +LINSTOR_VERSION ?= 1.32.3 +LINSTOR_CSI_VERSION ?= v1.10.5 image: image-piraeus-server image-linstor-csi diff --git a/packages/system/linstor/images/linstor-csi/Dockerfile b/packages/system/linstor/images/linstor-csi/Dockerfile index df5d179d..6cfd44aa 100644 --- a/packages/system/linstor/images/linstor-csi/Dockerfile +++ b/packages/system/linstor/images/linstor-csi/Dockerfile @@ -1,6 +1,6 @@ FROM golang:1.25 AS builder -ARG VERSION=v1.10.6 +ARG VERSION=v1.10.5 ARG LINSTOR_WAIT_UNTIL_VERSION=v0.3.1 ARG TARGETARCH ARG TARGETOS diff --git a/packages/system/linstor/images/linstor-csi/patches/001-relocate-after-clone-restore.diff b/packages/system/linstor/images/linstor-csi/patches/001-relocate-after-clone-restore.diff index 0e272670..ff0aec0c 100644 --- a/packages/system/linstor/images/linstor-csi/patches/001-relocate-after-clone-restore.diff +++ b/packages/system/linstor/images/linstor-csi/patches/001-relocate-after-clone-restore.diff @@ -1,7 +1,25 @@ diff --git a/pkg/client/linstor.go b/pkg/client/linstor.go -index ac4651d..46d565d 100644 +index f544493..98e7fde 100644 --- a/pkg/client/linstor.go +++ b/pkg/client/linstor.go +@@ -181,7 +181,7 @@ func LogLevel(s string) func(*Linstor) error { + func (s *Linstor) ListAllWithStatus(ctx context.Context) ([]volume.VolumeStatus, error) { + var vols []volume.VolumeStatus + +- resourcesByName := make(map[string][]lapi.Resource) ++ resourcesByName := make(map[string][]lapi.ResourceWithVolumes) + + resDefs, err := s.client.ResourceDefinitions.GetAll(ctx, lapi.RDGetAllRequest{WithVolumeDefinitions: true}) + if err != nil { +@@ -194,7 +194,7 @@ func (s *Linstor) ListAllWithStatus(ctx context.Context) ([]volume.VolumeStatus, + } + + for i := range allResources { +- resourcesByName[allResources[i].Name] = append(resourcesByName[allResources[i].Name], allResources[i].Resource) ++ resourcesByName[allResources[i].Name] = append(resourcesByName[allResources[i].Name], allResources[i]) + } + + for _, rd := range resDefs { @@ -462,6 +462,14 @@ func (s *Linstor) Clone(ctx context.Context, vol, src *volume.Info, params *volu return err } @@ -17,7 +35,7 @@ index ac4651d..46d565d 100644 logger.Debug("reconcile extra properties") err = s.client.ResourceDefinitions.Modify(ctx, vol.ID, lapi.GenericPropsModify{OverrideProps: vol.Properties}) -@@ -1297,6 +1305,14 @@ func (s *Linstor) VolFromSnap(ctx context.Context, snap *volume.Snapshot, vol *v +@@ -1280,6 +1288,14 @@ func (s *Linstor) VolFromSnap(ctx context.Context, snap *volume.Snapshot, vol *v return err } @@ -32,7 +50,7 @@ index ac4651d..46d565d 100644 logger.Debug("reconcile extra properties") err = s.client.ResourceDefinitions.Modify(ctx, vol.ID, lapi.GenericPropsModify{OverrideProps: vol.Properties}) -@@ -1487,9 +1503,8 @@ func (s *Linstor) reconcileSnapshotResources(ctx context.Context, snapshot *volu +@@ -1470,9 +1486,8 @@ func (s *Linstor) reconcileSnapshotResources(ctx context.Context, snapshot *volu return fmt.Errorf("snapshot '%s' not deployed on any node", snap.Name) } @@ -44,7 +62,7 @@ index ac4651d..46d565d 100644 for _, snapNode := range snap.Nodes { if err := s.NodeAvailable(ctx, snapNode); err != nil { logger.WithField("selected node candidate", snapNode).WithError(err).Debug("node is not available") -@@ -1497,13 +1512,23 @@ func (s *Linstor) reconcileSnapshotResources(ctx context.Context, snapshot *volu +@@ -1480,13 +1495,23 @@ func (s *Linstor) reconcileSnapshotResources(ctx context.Context, snapshot *volu } if slices.Contains(preferredNodes, snapNode) { @@ -74,7 +92,7 @@ index ac4651d..46d565d 100644 } if selectedNode == "" { -@@ -1696,6 +1721,114 @@ func (s *Linstor) reconcileResourcePlacement(ctx context.Context, vol *volume.In +@@ -1679,6 +1704,114 @@ func (s *Linstor) reconcileResourcePlacement(ctx context.Context, vol *volume.In return nil } @@ -189,8 +207,40 @@ index ac4651d..46d565d 100644 // FindSnapsByID searches the snapshot in the backend func (s *Linstor) FindSnapsByID(ctx context.Context, id string) ([]*volume.Snapshot, error) { snapshotId, err := volume.ParseSnapshotId(id) +@@ -2173,7 +2306,7 @@ func (s *Linstor) Status(ctx context.Context, volId string) ([]string, *csi.Volu + "volume": volId, + }).Debug("getting assignments") + +- ress, err := s.client.Resources.GetAll(ctx, volId) ++ ress, err := s.client.Resources.GetResourceView(ctx, &lapi.ListOpts{Resource: []string{volId}}) + if err != nil { + return nil, nil, fmt.Errorf("failed to list resources for '%s': %w", volId, err) + } +@@ -2261,13 +2394,20 @@ func GetSnapshotRemoteAndReadiness(snap *lapi.Snapshot) (string, bool, error) { + return "", slices.Contains(snap.Flags, lapiconsts.FlagSuccessful), nil + } + +-func NodesAndConditionFromResources(ress []lapi.Resource) ([]string, *csi.VolumeCondition) { ++func NodesAndConditionFromResources(ress []lapi.ResourceWithVolumes) ([]string, *csi.VolumeCondition) { + var allNodes, abnormalNodes []string + + for i := range ress { + res := &ress[i] + +- allNodes = append(allNodes, res.NodeName) ++ // A resource is a CSI publish target if any of its volumes were created ++ // by ControllerPublishVolume, identified by the temporary-diskless-attach property. ++ if slices.ContainsFunc(res.Volumes, func(v lapi.Volume) bool { ++ createdFor, ok := v.Props[linstor.PropertyCreatedFor] ++ return ok && createdFor == linstor.CreatedForTemporaryDisklessAttach ++ }) { ++ allNodes = append(allNodes, res.NodeName) ++ } + + if res.State == nil { + abnormalNodes = append(abnormalNodes, res.NodeName) diff --git a/pkg/volume/parameter.go b/pkg/volume/parameter.go -index 39acd95..54d1dfb 100644 +index 39acd95..aed18ab 100644 --- a/pkg/volume/parameter.go +++ b/pkg/volume/parameter.go @@ -50,6 +50,7 @@ const ( diff --git a/packages/system/linstor/images/linstor-csi/patches/002-protocol-c-override-for-dual-attach.diff b/packages/system/linstor/images/linstor-csi/patches/002-protocol-c-override-for-dual-attach.diff deleted file mode 100644 index 279c636b..00000000 --- a/packages/system/linstor/images/linstor-csi/patches/002-protocol-c-override-for-dual-attach.diff +++ /dev/null @@ -1,132 +0,0 @@ -diff --git a/pkg/client/linstor.go b/pkg/client/linstor.go -index ac4651d..9d75c79 100644 ---- a/pkg/client/linstor.go -+++ b/pkg/client/linstor.go -@@ -653,11 +653,35 @@ func (s *Linstor) Attach(ctx context.Context, volId, node string, rwxBlock bool) - } - - if otherResInUse > 0 && rwxBlock { -- rdPropsModify := lapi.GenericPropsModify{OverrideProps: map[string]string{ -+ rdProps := map[string]string{ - linstor.PropertyAllowTwoPrimaries: "yes", -- }} -+ } - -- err = s.client.ResourceDefinitions.Modify(ctx, volId, rdPropsModify) -+ // DRBD requires Protocol C whenever allow-two-primaries is enabled. -+ // If the resource is configured with Protocol A or B (typically -+ // through its resource-group), drbdadm adjust on the satellites -+ // fails with "Protocol C required" (errno 139) and live migration -+ // cannot proceed. Override Protocol to C on the resource-definition -+ // itself so it applies to every connection (including diskless -+ // peers, e.g. a TieBreaker). The marker lets Detach revert exactly -+ // the override we installed without disturbing operator-set props. -+ proto, protoErr := s.getEffectiveDrbdProtocol(ctx, volId) -+ if protoErr != nil { -+ s.log.WithError(protoErr).WithField("volume", volId). -+ Warn("failed to determine effective DRBD protocol; skipping Protocol=C override for dual-attach") -+ } else if proto == "A" || proto == "B" { -+ s.log.WithFields(logrus.Fields{ -+ "volume": volId, -+ "protocol": proto, -+ }).Info("installing Protocol=C override on resource-definition for dual-attach") -+ -+ rdProps[linstor.PropertyDrbdNetProtocol] = "C" -+ rdProps[linstor.PropertyCsiProtocolOverride] = "yes" -+ } -+ -+ err = s.client.ResourceDefinitions.Modify(ctx, volId, lapi.GenericPropsModify{ -+ OverrideProps: rdProps, -+ }) - if err != nil { - return "", err - } -@@ -774,11 +798,26 @@ func (s *Linstor) Detach(ctx context.Context, volId, node string) error { - } - - if resInUse == 1 { -- rdPropsModify := lapi.GenericPropsModify{DeleteProps: []string{ -- linstor.PropertyAllowTwoPrimaries, -- }} -+ deleteProps := []string{linstor.PropertyAllowTwoPrimaries} -+ -+ // Drop the Protocol=C override only if Attach installed it (gated by -+ // our marker), so an operator-set Protocol property on the -+ // resource-definition is preserved. -+ rd, getErr := s.client.ResourceDefinitions.Get(ctx, volId) -+ if getErr != nil { -+ log.WithError(getErr).Warn("failed to fetch resource-definition props; skipping Protocol=C override removal") -+ } else if rd.Props[linstor.PropertyCsiProtocolOverride] != "" { -+ log.Info("removing Protocol=C override from resource-definition") - -- err = s.client.ResourceDefinitions.Modify(ctx, volId, rdPropsModify) -+ deleteProps = append(deleteProps, -+ linstor.PropertyDrbdNetProtocol, -+ linstor.PropertyCsiProtocolOverride, -+ ) -+ } -+ -+ err = s.client.ResourceDefinitions.Modify(ctx, volId, lapi.GenericPropsModify{ -+ DeleteProps: deleteProps, -+ }) - if err != nil { - return nil404(err) - } -@@ -805,6 +844,33 @@ func (s *Linstor) Detach(ctx context.Context, volId, node string) error { - return nil404(s.client.Resources.Delete(ctx, volId, node)) - } - -+// getEffectiveDrbdProtocol returns the DRBD network protocol ("A", "B" or "C") -+// that LINSTOR will hand to drbdadm for the given resource. Resource-definition -+// properties take precedence; otherwise the value is inherited from the -+// resource-group. An empty string is returned when neither level sets the -+// property (LINSTOR's compiled-in default is "C"). -+func (s *Linstor) getEffectiveDrbdProtocol(ctx context.Context, volId string) (string, error) { -+ rd, err := s.client.ResourceDefinitions.Get(ctx, volId) -+ if err != nil { -+ return "", err -+ } -+ -+ if v, ok := rd.Props[linstor.PropertyDrbdNetProtocol]; ok { -+ return v, nil -+ } -+ -+ if rd.ResourceGroupName == "" { -+ return "", nil -+ } -+ -+ rg, err := s.client.ResourceGroups.Get(ctx, rd.ResourceGroupName) -+ if err != nil { -+ return "", err -+ } -+ -+ return rg.Props[linstor.PropertyDrbdNetProtocol], nil -+} -+ - // CapacityBytes returns the amount of free space in the storage pool specified by the params and topology. - func (s *Linstor) CapacityBytes(ctx context.Context, storagePools []string, overProvision *float64, segments map[string]string) (int64, error) { - log := s.log.WithField("storage-pools", storagePools).WithField("segments", segments) -diff --git a/pkg/linstor/const.go b/pkg/linstor/const.go -index 9a5f79c..c8bc9c3 100644 ---- a/pkg/linstor/const.go -+++ b/pkg/linstor/const.go -@@ -44,6 +44,19 @@ const ( - // PropertyAllowTwoPrimaries is DRBD option to allow second primary. Mainly used for live-migration. - PropertyAllowTwoPrimaries = lc.NamespcDrbdNetOptions + "/allow-two-primaries" - -+ // PropertyDrbdNetProtocol is the DRBD network replication protocol option -+ // (A = async, B = semi-sync, C = sync). DRBD requires Protocol C whenever -+ // allow-two-primaries is enabled, otherwise drbdadm adjust fails with -+ // "Protocol C required" (errno 139). -+ PropertyDrbdNetProtocol = lc.NamespcDrbdNetOptions + "/protocol" -+ -+ // PropertyCsiProtocolOverride marks a resource-connection where this driver -+ // has installed a temporary Protocol=C override to make a Protocol-A/B -+ // resource compatible with allow-two-primaries during live migration. The -+ // marker lets us safely remove only the overrides we set, without touching -+ // connection-level overrides installed by the operator. -+ PropertyCsiProtocolOverride = lc.NamespcAuxiliary + "/csi-protocol-override" -+ - // CreatedForTemporaryDisklessAttach marks a resource as temporary, i.e. it should be removed after it is no longer - // needed. - CreatedForTemporaryDisklessAttach = "temporary-diskless-attach" diff --git a/packages/system/linstor/images/piraeus-server/patches/README.md b/packages/system/linstor/images/piraeus-server/patches/README.md index dfe75c2e..558fdfeb 100644 --- a/packages/system/linstor/images/piraeus-server/patches/README.md +++ b/packages/system/linstor/images/piraeus-server/patches/README.md @@ -1,17 +1,14 @@ # LINSTOR Server Patches -Custom patches for piraeus-server (linstor-server) v1.33.2. +Custom patches for piraeus-server (linstor-server) v1.32.3. -- **allow-toggle-disk-retry.diff** — Backport maintainer implementation of toggle-disk retry/abort - - Source PR/comment: [#475](https://github.com/LINBIT/linstor-server/pull/475), [maintainer note](https://github.com/LINBIT/linstor-server/pull/475#issuecomment-3949630419) - - Backported from upstream commit: [`3d97f71c9`](https://github.com/LINBIT/linstor-server/commit/3d97f71c95a493588d3d521c63eac4d846935fb3) -- **fix-duplicate-tcp-ports.diff** — Preserve DRBD TCP ports during toggle-disk and avoid redundant `ensureStackDataExists()` - - Source PR/review: [#476](https://github.com/LINBIT/linstor-server/pull/476), [review suggestion](https://github.com/LINBIT/linstor-server/pull/476#discussion_r3007725079) - - Backported from commits: [`79d6375c5`](https://github.com/kvaps/linstor-server/commit/79d6375c55d6181b35a7b7f0fe8dbdfb86e126cd), [`bcc89902f`](https://github.com/kvaps/linstor-server/commit/bcc89902f4f61ac1589dd07ebb7f5aae1935370d) -- **fix-luks-header-size.diff** — Account for LUKS2 `--offset`, metadata/keyslots sizing, and device `optimal_io_size` - - Source PR/comment: [#472](https://github.com/LINBIT/linstor-server/pull/472), [maintainer note](https://github.com/LINBIT/linstor-server/pull/472#issuecomment-3949687603) - - Backported from commits: [`ccc85fbd2`](https://github.com/LINBIT/linstor-server/commit/ccc85fbd2c65f0b97c52403fa80f1efdb886ec4e), [`71b601554`](https://github.com/LINBIT/linstor-server/commit/71b601554f41bcb50cd5bd06989c5b0d3a814acd) - - Note: upstream commit [`3d0402a0c`](https://github.com/LINBIT/linstor-server/commit/3d0402a0c25f0a4b57b380321f10e89982f26e7a) is already included in `v1.33.1` -- **retry-adjust-after-stale-bitmap.diff** — Retry `drbdadm adjust` after detaching a stale local bitmap state - - Source PR: [#491](https://github.com/LINBIT/linstor-server/pull/491) - - Backported from commit: [`51ae50a84`](https://github.com/kvaps/linstor-server/commit/51ae50a84dcb98093f543b819652c750a94d96c9) +- **adjust-on-resfile-change.diff** — Use actual device path in res file during toggle-disk; fix LUKS data offset + - Upstream: [#473](https://github.com/LINBIT/linstor-server/pull/473), [#472](https://github.com/LINBIT/linstor-server/pull/472) +- **allow-toggle-disk-retry.diff** — Allow retry and cancellation of failed toggle-disk operations + - Upstream: [#475](https://github.com/LINBIT/linstor-server/pull/475) +- **force-metadata-check-on-disk-add.diff** — Create metadata during toggle-disk from diskless to diskful + - Upstream: [#474](https://github.com/LINBIT/linstor-server/pull/474) +- **fix-duplicate-tcp-ports.diff** — Preserve TCP ports during toggle-disk to prevent port mismatch between controller and satellites + - Upstream: [#476](https://github.com/LINBIT/linstor-server/pull/476) (superseded by this expanded fix) +- **skip-adjust-when-device-inaccessible.diff** — Fix resources stuck in StandAlone after reboot, Unknown state race condition, and encrypted resource deletion + - Upstream: [#477](https://github.com/LINBIT/linstor-server/pull/477) diff --git a/packages/system/linstor/images/piraeus-server/patches/adjust-on-resfile-change.diff b/packages/system/linstor/images/piraeus-server/patches/adjust-on-resfile-change.diff new file mode 100644 index 00000000..6788efaf --- /dev/null +++ b/packages/system/linstor/images/piraeus-server/patches/adjust-on-resfile-change.diff @@ -0,0 +1,48 @@ +diff --git a/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/ConfFileBuilder.java b/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/ConfFileBuilder.java +index 36c52ccf8..c0bb7b967 100644 +--- a/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/ConfFileBuilder.java ++++ b/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/ConfFileBuilder.java +@@ -894,12 +894,16 @@ public class ConfFileBuilder + if (((Volume) vlmData.getVolume()).getFlags().isUnset(localAccCtx, Volume.Flags.DELETE)) + { + final String disk; ++ // Check if we're in toggle-disk operation (adding disk to diskless resource) ++ boolean isDiskAdding = vlmData.getVolume().getAbsResource().getStateFlags().isSomeSet( ++ localAccCtx, Resource.Flags.DISK_ADD_REQUESTED, Resource.Flags.DISK_ADDING); + if ((!isPeerRsc && vlmData.getDataDevice() == null) || + (isPeerRsc && + // FIXME: vlmData.getRscLayerObject().getFlags should be used here + vlmData.getVolume().getAbsResource().disklessForDrbdPeers(accCtx) + ) || +- (!isPeerRsc && ++ // For toggle-disk: if dataDevice is set and we're adding disk, use the actual device ++ (!isPeerRsc && !isDiskAdding && + // FIXME: vlmData.getRscLayerObject().getFlags should be used here + vlmData.getVolume().getAbsResource().isDrbdDiskless(accCtx) + ) +diff --git a/satellite/src/main/java/com/linbit/linstor/layer/luks/CryptSetupCommands.java b/satellite/src/main/java/com/linbit/linstor/layer/luks/CryptSetupCommands.java +index 54dd5c19f..018de58cf 100644 +--- a/satellite/src/main/java/com/linbit/linstor/layer/luks/CryptSetupCommands.java ++++ b/satellite/src/main/java/com/linbit/linstor/layer/luks/CryptSetupCommands.java +@@ -34,6 +34,9 @@ public class CryptSetupCommands implements Luks + private static final Version V2_1_0 = new Version(2, 1, 0); + private static final Version V2_0_0 = new Version(2, 0, 0); + private static final String PBDKF_MAX_MEMORY_KIB = "262144"; // 256 MiB ++ // Fixed LUKS2 data offset in 512-byte sectors (16 MiB = 32768 sectors) ++ // This ensures consistent LUKS header size across all nodes regardless of system defaults ++ private static final String LUKS2_DATA_OFFSET_SECTORS = "32768"; + + @SuppressWarnings("unused") + private final ErrorReporter errorReporter; +@@ -78,6 +81,11 @@ public class CryptSetupCommands implements Luks + command.add(CRYPTSETUP); + command.add("-q"); + command.add("luksFormat"); ++ // Always specify explicit offset to ensure consistent LUKS header size across all nodes ++ // Without this, different systems may create LUKS with different header sizes (16MiB vs 32MiB) ++ // which causes "Low.dev. smaller than requested DRBD-dev. size" errors during toggle-disk ++ command.add("--offset"); ++ command.add(LUKS2_DATA_OFFSET_SECTORS); + if (version.greaterOrEqual(V2_0_0)) + { + command.add("--pbkdf-memory"); diff --git a/packages/system/linstor/images/piraeus-server/patches/allow-toggle-disk-retry.diff b/packages/system/linstor/images/piraeus-server/patches/allow-toggle-disk-retry.diff index 952ba16c..264e0221 100644 --- a/packages/system/linstor/images/piraeus-server/patches/allow-toggle-disk-retry.diff +++ b/packages/system/linstor/images/piraeus-server/patches/allow-toggle-disk-retry.diff @@ -1,232 +1,168 @@ diff --git a/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscToggleDiskApiCallHandler.java b/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscToggleDiskApiCallHandler.java -index d93a18014..a944cb809 100644 +index d93a18014..cc8ce4f04 100644 --- a/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscToggleDiskApiCallHandler.java +++ b/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscToggleDiskApiCallHandler.java -@@ -111,6 +111,14 @@ import reactor.util.function.Tuple2; - @Singleton - public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionListener - { -+ private enum ToggleDiskAction -+ { -+ NOOP, -+ NORMAL, -+ ABORT, -+ RETRY -+ } -+ - private final AccessContext apiCtx; - private final ScopeRunner scopeRunner; - private final BackgroundRunner backgroundRunner; -@@ -386,69 +394,33 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL - "Toggle Disk on %s/%s %s", nodeNameStr, rscNameStr, removeDisk ? "removing disk" : "adding disk"); +@@ -57,7 +57,9 @@ import com.linbit.linstor.stateflags.StateFlags; + import com.linbit.linstor.storage.StorageException; + import com.linbit.linstor.storage.data.adapter.drbd.DrbdRscData; + import com.linbit.linstor.storage.interfaces.categories.resource.AbsRscLayerObject; ++import com.linbit.linstor.storage.interfaces.categories.resource.VlmProviderObject; + import com.linbit.linstor.storage.kinds.DeviceLayerKind; ++import com.linbit.linstor.storage.kinds.DeviceProviderKind; + import com.linbit.linstor.storage.utils.LayerUtils; + import com.linbit.linstor.tasks.AutoDiskfulTask; + import com.linbit.linstor.utils.layer.LayerRscUtils; +@@ -387,21 +389,84 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL Resource rsc = ctrlApiDataLoader.loadRsc(nodeName, rscName, true); -+ ResourceDefinition rscDfn = rsc.getResourceDefinition(); -- if (hasDiskAddRequested(rsc)) -- { ++ // Allow retry of the same operation if the previous attempt failed ++ // (the requested flag remains set for retry on reconnection, but we should also allow manual retry) ++ // Also allow cancellation of a failed operation by requesting the opposite operation + if (hasDiskAddRequested(rsc)) + { - throw new ApiRcException(ApiCallRcImpl.simpleEntry( - ApiConsts.FAIL_RSC_BUSY, - "Addition of disk to resource already requested", - true - )); -- } -- if (hasDiskRemoveRequested(rsc)) -- { ++ if (removeDisk) ++ { ++ // User wants to cancel the failed add-disk operation and go back to diskless ++ // Use the existing disk removal flow to properly cleanup storage on satellite ++ errorReporter.logInfo( ++ "Toggle Disk cancel on %s/%s - cancelling failed DISK_ADD_REQUESTED, reverting to diskless", ++ nodeNameStr, rscNameStr); ++ unmarkDiskAddRequested(rsc); ++ // Also clear DISK_ADDING if it was set ++ unmarkDiskAdding(rsc); ++ ++ // Set storage pool to diskless pool (overwrite the diskful pool that was set) ++ Props rscProps = ctrlPropsHelper.getProps(rsc); ++ rscProps.map().put(ApiConsts.KEY_STOR_POOL_NAME, LinStor.DISKLESS_STOR_POOL_NAME); ++ ++ // Set DISK_REMOVE_REQUESTED to use the existing disk removal flow ++ // This will: ++ // 1. updateAndAdjustDisk sets DISK_REMOVING flag ++ // 2. Satellite sees DISK_REMOVING and deletes LUKS/storage devices ++ // 3. finishOperation rebuilds layer stack as diskless ++ // We keep the existing layer data so satellite can properly cleanup ++ markDiskRemoveRequested(rsc); ++ ++ ctrlTransactionHelper.commit(); ++ ++ // Use existing disk removal flow - this will properly cleanup storage on satellite ++ return Flux ++ .just(ApiCallRcImpl.singleApiCallRc( ++ ApiConsts.MODIFIED, ++ "Cancelling disk addition, reverting to diskless" ++ )) ++ .concatWith(updateAndAdjustDisk(nodeName, rscName, true, toggleIntoTiebreakerRef, context)) ++ .concatWith(ctrlRscDfnApiCallHandler.get().updateProps(rsc.getResourceDefinition())); ++ } ++ // If adding disk and DISK_ADD_REQUESTED is already set, treat as retry ++ // Simply retry the operation with existing layer data - satellite will handle it idempotently ++ // NOTE: We don't remove/recreate layer data here because removeLayerData() only deletes ++ // from controller DB without calling drbdadm down on satellite, leaving orphaned DRBD devices ++ errorReporter.logInfo( ++ "Toggle Disk retry on %s/%s - DISK_ADD_REQUESTED already set, retrying operation", ++ nodeNameStr, rscNameStr); ++ ctrlTransactionHelper.commit(); ++ return Flux ++ .just(new ApiCallRcImpl()) ++ .concatWith(updateAndAdjustDisk(nodeName, rscName, false, toggleIntoTiebreakerRef, context)) ++ .concatWith(ctrlRscDfnApiCallHandler.get().updateProps(rsc.getResourceDefinition())); + } + if (hasDiskRemoveRequested(rsc)) + { - throw new ApiRcException(ApiCallRcImpl.simpleEntry( - ApiConsts.FAIL_RSC_BUSY, - "Removal of disk from resource already requested", - true - )); -- } -+ ToggleDiskAction action = determineToggleDiskAction(rsc, removeDisk); -+ errorReporter.logDebug("Toggle Disk action: %s", action); - -- if (!removeDisk && !ctrlVlmCrtApiHelper.isDiskless(rsc)) -+ switch (action) - { -- throw new ApiRcException(ApiCallRcImpl.simpleEntry( -- ApiConsts.WARN_RSC_ALREADY_HAS_DISK, -- "Resource already has disk", -- true -- )); -- } -- if (removeDisk && ctrlVlmCrtApiHelper.isDiskless(rsc)) -- { -- throw new ApiRcException(ApiCallRcImpl.simpleEntry( -- ApiConsts.WARN_RSC_ALREADY_DISKLESS, -- "Resource already diskless", -- true -- )); -+ case NOOP: -+ return handleNoopAction(rsc, removeDisk); -+ case RETRY: -+ return handleRetryAction(rsc, removeDisk, toggleIntoTiebreakerRef, context); -+ case ABORT: -+ clearToggleDiskFlags(rsc); ++ if (!removeDisk) ++ { ++ // User wants to cancel the failed remove-disk operation + errorReporter.logInfo( -+ "Aborting previous toggle disk transition, starting new transition to %s", -+ removeDisk ? "diskless" : "diskful" ++ "Toggle Disk cancel on %s/%s - cancelling failed DISK_REMOVE_REQUESTED", ++ nodeNameStr, rscNameStr); ++ unmarkDiskRemoveRequested(rsc); ++ ctrlTransactionHelper.commit(); ++ return Flux.just( ++ ApiCallRcImpl.singleApiCallRc( ++ ApiConsts.MODIFIED, ++ "Cancelled disk removal request" ++ ) + ); -+ break; -+ case NORMAL: -+ break; -+ default: -+ throw new ImplementationError("Unhandled case: " + action); ++ } ++ // If removing disk and DISK_REMOVE_REQUESTED is already set, treat as retry ++ errorReporter.logInfo( ++ "Toggle Disk retry on %s/%s - DISK_REMOVE_REQUESTED already set, continuing operation", ++ nodeNameStr, rscNameStr); ++ ctrlTransactionHelper.commit(); ++ return Flux ++ .just(new ApiCallRcImpl()) ++ .concatWith(updateAndAdjustDisk(nodeName, rscName, true, toggleIntoTiebreakerRef, context)) ++ .concatWith(ctrlRscDfnApiCallHandler.get().updateProps(rsc.getResourceDefinition())); } + if (!removeDisk && !ctrlVlmCrtApiHelper.isDiskless(rsc)) +@@ -412,17 +477,43 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL + true + )); + } ++ ResourceDefinition rscDfn = rsc.getResourceDefinition(); ++ AccessContext peerCtx = peerAccCtx.get(); ++ + if (removeDisk && ctrlVlmCrtApiHelper.isDiskless(rsc)) + { ++ // Resource is marked as diskless - check if it has orphaned storage layers that need cleanup ++ AbsRscLayerObject layerData = getLayerData(peerCtx, rsc); ++ if (layerData != null && (LayerUtils.hasLayer(layerData, DeviceLayerKind.LUKS) || ++ hasNonDisklessStorageLayer(layerData))) ++ { ++ // Resource is marked as diskless but has orphaned storage layers - need cleanup ++ // Use the existing disk removal flow to properly cleanup storage on satellite ++ errorReporter.logInfo( ++ "Toggle Disk cleanup on %s/%s - resource is diskless but has orphaned storage layers, cleaning up", ++ nodeNameStr, rscNameStr); ++ ++ // Set DISK_REMOVE_REQUESTED to use the existing disk removal flow ++ // This will trigger proper satellite cleanup via DISK_REMOVING flag ++ markDiskRemoveRequested(rsc); ++ ++ ctrlTransactionHelper.commit(); ++ ++ // Use existing disk removal flow - this will properly cleanup storage on satellite ++ return Flux ++ .just(ApiCallRcImpl.singleApiCallRc( ++ ApiConsts.MODIFIED, ++ "Cleaning up orphaned storage layers" ++ )) ++ .concatWith(updateAndAdjustDisk(nodeName, rscName, true, toggleIntoTiebreakerRef, context)) ++ .concatWith(ctrlRscDfnApiCallHandler.get().updateProps(rsc.getResourceDefinition())); ++ } + throw new ApiRcException(ApiCallRcImpl.simpleEntry( + ApiConsts.WARN_RSC_ALREADY_DISKLESS, + "Resource already diskless", + true + )); + } +- - ResourceDefinition rscDfn = rsc.getResourceDefinition(); - AccessContext peerCtx = peerAccCtx.get(); -- if (removeDisk) -- { -- // Prevent removal of the last disk -- int haveDiskCount = countDisksAndIsOnline(rscDfn); -- if (haveDiskCount <= 1) -- { -- throw new ApiRcException(ApiCallRcImpl.simpleEntry( -- ApiConsts.FAIL_INSUFFICIENT_REPLICA_COUNT, -- "Cannot remove the disk from the only online resource with a disk", -- true -- )); -- } -+ validateToggleDiskPreconditions(rsc, removeDisk); - -- if (!LayerUtils.hasLayer(getLayerData(peerCtx, rsc), DeviceLayerKind.DRBD)) -- { -- throw new ApiRcException(ApiCallRcImpl.simpleEntry( -- ApiConsts.FAIL_INVLD_LAYER_STACK, -- "Toggle disk is only supported in combination with DRBD", -- true -- )); -- } -- } -- else -- { -- ensureAllPeersHavePeerSlotLeft(rscDfn); -- } -+ AccessContext peerCtx = peerAccCtx.get(); - - // Save the requested storage pool in the resource properties. - // This does not cause the storage pool to be used automatically. -@@ -628,10 +600,10 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL - - ctrlTransactionHelper.commit(); - -- String action = removeDisk ? "Removal of disk from" : "Addition of disk to"; -+ String actionStr = removeDisk ? "Removal of disk from" : "Addition of disk to"; - responses.addEntry(ApiCallRcImpl.simpleEntry( - ApiConsts.MODIFIED, -- action + " resource '" + rscDfn.getName().displayValue + "' " + -+ actionStr + " resource '" + rscDfn.getName().displayValue + "' " + - "on node '" + rsc.getNode().getName().displayValue + "' registered" - )); - -@@ -651,6 +623,40 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL - ); - } - -+ private Flux handleNoopAction(Resource rscRef, boolean removeDiskRef) -+ { -+ String state = removeDiskRef ? "diskless" : "diskful"; -+ ApiCallRcImpl responses = new ApiCallRcImpl(); -+ responses.addEntry(ApiCallRcImpl.simpleEntry( -+ ApiConsts.INFO_NOOP, -+ "Resource '" + rscRef.getResourceDefinition().getName().displayValue + "' on node '" + -+ rscRef.getNode().getName().displayValue + "' is already " + state -+ )); -+ return Flux.just(responses); -+ } -+ -+ private Flux handleRetryAction( -+ Resource rscRef, -+ boolean removeDisk, -+ boolean toggleIntoTiebreakerRef, -+ ResponseContext context -+ ) -+ { -+ String direction = removeDisk ? "diskless" : "diskful"; -+ ApiCallRcImpl responses = new ApiCallRcImpl(); -+ NodeName nodeName = rscRef.getNode().getName(); -+ ResourceName rscName = rscRef.getResourceDefinition().getName(); -+ responses.addEntry(ApiCallRcImpl.simpleEntry( -+ ApiConsts.INFO_NOOP, -+ "Retrying toggle disk to " + direction + " for resource '" + rscName.displayValue + -+ "' on node '" + nodeName.displayValue + "'" -+ )); -+ -+ return Flux -+ .just(responses) -+ .concatWith(updateAndAdjustDisk(nodeName, rscName, removeDisk, toggleIntoTiebreakerRef, context)); -+ } -+ - private long getVlmDfnSizePrivileged(VolumeDefinition vlmDfnRef) - { - try -@@ -781,6 +787,96 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL + if (removeDisk) + { + // Prevent removal of the last disk +@@ -1446,6 +1537,30 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL } } -+ private ToggleDiskAction determineToggleDiskAction(Resource rsc, boolean removeDisk) -+ { -+ boolean isDiskless = ctrlVlmCrtApiHelper.isDiskless(rsc); -+ boolean diskAddRequested = hasDiskAddRequested(rsc); -+ boolean diskRemoveRequested = hasDiskRemoveRequested(rsc); -+ -+ ToggleDiskAction action; -+ if (isDiskless) -+ { -+ if (diskAddRequested) -+ { -+ action = removeDisk ? ToggleDiskAction.ABORT : ToggleDiskAction.RETRY; -+ } -+ else if (diskRemoveRequested) -+ { -+ action = removeDisk ? ToggleDiskAction.RETRY : ToggleDiskAction.ABORT; -+ } -+ else -+ { -+ action = removeDisk ? ToggleDiskAction.NOOP : ToggleDiskAction.NORMAL; -+ } -+ } -+ else -+ { -+ if (diskRemoveRequested) -+ { -+ action = removeDisk ? ToggleDiskAction.RETRY : ToggleDiskAction.ABORT; -+ } -+ else -+ { -+ action = removeDisk ? ToggleDiskAction.NORMAL : ToggleDiskAction.NOOP; -+ } -+ } -+ -+ return action; -+ } -+ -+ private void validateToggleDiskPreconditions(Resource rsc, boolean removeDisk) -+ { -+ ResourceDefinition rscDfn = rsc.getResourceDefinition(); -+ if (removeDisk) -+ { -+ validateDiskRemovalAllowed(rsc, rscDfn); -+ } -+ else -+ { -+ ensureAllPeersHavePeerSlotLeft(rscDfn); -+ } -+ } -+ -+ private void clearToggleDiskFlags(Resource rsc) ++ private void unmarkDiskAddRequested(Resource rsc) + { + try + { -+ rsc.getStateFlags().disableFlags( -+ apiCtx, -+ Resource.Flags.DISK_ADD_REQUESTED, -+ Resource.Flags.DISK_ADDING, -+ Resource.Flags.DISK_REMOVE_REQUESTED, -+ Resource.Flags.DISK_REMOVING -+ ); ++ rsc.getStateFlags().disableFlags(apiCtx, Resource.Flags.DISK_ADD_REQUESTED); + } + catch (AccessDeniedException | DatabaseException exc) + { @@ -234,28 +170,60 @@ index d93a18014..a944cb809 100644 + } + } + -+ private void validateDiskRemovalAllowed(Resource rsc, ResourceDefinition rscDfn) ++ private void unmarkDiskRemoveRequested(Resource rsc) + { -+ int haveDiskCount = countDisksAndIsOnline(rscDfn); -+ if (haveDiskCount <= 1) ++ try + { -+ throw new ApiRcException(ApiCallRcImpl.simpleEntry( -+ ApiConsts.FAIL_INSUFFICIENT_REPLICA_COUNT, -+ "Cannot remove the disk from the only online resource with a disk", -+ true -+ )); ++ rsc.getStateFlags().disableFlags(apiCtx, Resource.Flags.DISK_REMOVE_REQUESTED); + } -+ -+ if (!LayerUtils.hasLayer(getLayerData(peerAccCtx.get(), rsc), DeviceLayerKind.DRBD)) ++ catch (AccessDeniedException | DatabaseException exc) + { -+ throw new ApiRcException(ApiCallRcImpl.simpleEntry( -+ ApiConsts.FAIL_INVLD_LAYER_STACK, -+ "Toggle disk is only supported in combination with DRBD", -+ true -+ )); ++ throw new ImplementationError(exc); + } + } + - /** - * Although we need to rebuild the layerData as the layerList might have changed, if we do not - * deactivate (i.e. down) the current resource, we need to make sure that deleting DrbdRscData + private void markDiskAdded(Resource rscData) + { + try +@@ -1511,6 +1626,41 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL + return layerData; + } + ++ /** ++ * Check if the layer stack has a non-diskless STORAGE layer. ++ * This is used to detect orphaned storage layers that need cleanup. ++ */ ++ private boolean hasNonDisklessStorageLayer(AbsRscLayerObject layerDataRef) ++ { ++ boolean hasNonDiskless = false; ++ if (layerDataRef != null) ++ { ++ if (layerDataRef.getLayerKind() == DeviceLayerKind.STORAGE) ++ { ++ for (VlmProviderObject vlmData : layerDataRef.getVlmLayerObjects().values()) ++ { ++ if (vlmData.getProviderKind() != DeviceProviderKind.DISKLESS) ++ { ++ hasNonDiskless = true; ++ break; ++ } ++ } ++ } ++ if (!hasNonDiskless) ++ { ++ for (AbsRscLayerObject child : layerDataRef.getChildren()) ++ { ++ if (hasNonDisklessStorageLayer(child)) ++ { ++ hasNonDiskless = true; ++ break; ++ } ++ } ++ } ++ } ++ return hasNonDiskless; ++ } ++ + private LockGuard createLockGuard() + { + return lockGuardFactory.buildDeferred(LockType.WRITE, LockObj.NODES_MAP, LockObj.RSC_DFN_MAP); diff --git a/packages/system/linstor/images/piraeus-server/patches/fix-duplicate-tcp-ports.diff b/packages/system/linstor/images/piraeus-server/patches/fix-duplicate-tcp-ports.diff index ed106f98..b34a2500 100644 --- a/packages/system/linstor/images/piraeus-server/patches/fix-duplicate-tcp-ports.diff +++ b/packages/system/linstor/images/piraeus-server/patches/fix-duplicate-tcp-ports.diff @@ -1,25 +1,69 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Andrei Kvapil +Date: Fri, 28 Mar 2026 13:00:00 +0100 +Subject: [PATCH] fix(drbd): preserve TCP ports during toggle-disk operations + +Prevent TCP port mismatches after toggle-disk operations by preserving +existing TCP ports when rebuilding DrbdRscData. + +Root Cause: +----------- +During toggle-disk operations, removeLayerData() deletes DrbdRscData +(freeing its TCP ports from the number pool), then ensureStackDataExists() +creates new DrbdRscData. Since the payload has no explicit tcpPorts, +the controller allocates new ports from the pool -- which may differ from +the old ports if other resources claimed them in the meantime. + +The controller correctly avoids collisions in its own number pool, but +the satellite may miss the update (e.g. during controller restart or +network issues). When this happens, the satellite keeps the old ports +while peers receive the new ones, causing DRBD connection failures +(StandAlone/Connecting state). + +Additionally, remove the redundant ensureStackDataExists() call from +resetStoragePools() -- the caller already invokes it with the correct +payload. + +Solution: +--------- +1. Add copyDrbdTcpPortsIfExists() to save existing TCP ports into the + LayerPayload before removeLayerData() deletes them. +2. Call it from copyDrbdNodeIdIfExists() (covers both toggle-disk paths) + and from the needsDeactivate path (shared storage pool case). +3. Remove the redundant ensureStackDataExists() from resetStoragePools(). + +This ensures the same TCP ports are reused when DrbdRscData is recreated, +eliminating the window for port mismatch between controller and satellites. + +Co-Authored-By: Claude +Signed-off-by: Andrei Kvapil +--- + .../controller/CtrlRscToggleDiskApiCallHandler.java | 40 +++++++++++++++++++-- + .../linstor/layer/resource/CtrlRscLayerDataFactory.java | 2 -- + 2 files changed, 38 insertions(+), 4 deletions(-) + diff --git a/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscToggleDiskApiCallHandler.java b/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscToggleDiskApiCallHandler.java -index d93a18014..01cfbbacf 100644 +index ccdb0cee5..b0554c2ec 100644 --- a/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscToggleDiskApiCallHandler.java +++ b/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscToggleDiskApiCallHandler.java -@@ -37,6 +37,7 @@ import com.linbit.linstor.core.objects.StorPool; - import com.linbit.linstor.core.objects.Volume; - import com.linbit.linstor.core.objects.VolumeDefinition; - import com.linbit.linstor.core.objects.utils.MixedStorPoolHelper; +@@ -58,6 +58,7 @@ import com.linbit.linstor.stateflags.StateFlags; + import com.linbit.linstor.storage.StorageException; + import com.linbit.linstor.storage.data.adapter.drbd.DrbdRscData; + import com.linbit.linstor.storage.interfaces.categories.resource.AbsRscLayerObject; +import com.linbit.linstor.core.types.TcpPortNumber; - import com.linbit.linstor.dbdrivers.DatabaseException; - import com.linbit.linstor.event.EventWaiter; - import com.linbit.linstor.event.ObjectIdentifier; -@@ -85,6 +86,7 @@ import java.util.List; - import java.util.Map; + import com.linbit.linstor.storage.interfaces.categories.resource.VlmProviderObject; + import com.linbit.linstor.storage.kinds.DeviceLayerKind; + import com.linbit.linstor.storage.kinds.DeviceProviderKind; +@@ -88,6 +89,7 @@ import java.util.LinkedHashMap; + import java.util.List; import java.util.Map.Entry; import java.util.Set; +import java.util.TreeSet; - + import org.reactivestreams.Publisher; import reactor.core.publisher.Flux; -@@ -575,12 +577,13 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL - +@@ -587,8 +589,9 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL + /* * We also have to remove the currently diskless DrbdRscData and free up the node-id as now we must - * use the shared resource's node-id @@ -29,39 +73,30 @@ index d93a18014..01cfbbacf 100644 } else { -- copyDrbdNodeIdIfExists(rsc, payload); -+ copyDrbdSettings(rsc, payload); - } - /* - * rebuilds the layerdata in case we just removed it.. -@@ -782,10 +785,20 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL - } - +@@ -726,7 +729,7 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL /** -- * Although we need to rebuild the layerData as the layerList might have changed, if we do not -- * deactivate (i.e. down) the current resource, we need to make sure that deleting DrbdRscData + * Although we need to rebuild the layerData as the layerList might have changed, if we do not + * deactivate (i.e. down) the current resource, we need to make sure that deleting DrbdRscData - * and recreating a new DrbdRscData ends up with the same node-id as before. -+ * Copies DRBD settings (node-id and TCP ports) from the existing DrbdRscData into the payload -+ * before removeLayerData() deletes them. This ensures that recreated DrbdRscData ends up with -+ * the same node-id and TCP ports as before. -+ * -+ * TCP ports must be preserved because if the satellite misses the update (e.g. due to controller -+ * restart or connectivity issues), it will keep the old ports while peers receive the new ones, -+ * causing DRBD connections to fail with StandAlone state. ++ * and recreating a new DrbdRscData ends up with the same node-id and TCP ports as before. */ -+ private void copyDrbdSettings(Resource rsc, LayerPayload payload) throws ImplementationError -+ { -+ copyDrbdNodeIdIfExists(rsc, payload); + private void copyDrbdNodeIdIfExists(Resource rsc, LayerPayload payload) throws ImplementationError + { +@@ -743,6 +746,37 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL + DrbdRscData drbdRscData = (DrbdRscData) drbdRscDataSet.iterator().next(); + payload.drbdRsc.nodeId = drbdRscData.getNodeId().value; + } + copyDrbdTcpPortsIfExists(rsc, payload); + } + - private void copyDrbdNodeIdIfExists(Resource rsc, LayerPayload payload) throws ImplementationError - { - Set> drbdRscDataSet = LayerRscUtils.getRscDataByLayer( -@@ -804,6 +817,28 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL - } - } - ++ /** ++ * Preserves existing TCP ports during toggle-disk operations. ++ * ++ * When removeLayerData() deletes DrbdRscData, the TCP ports are freed from the number pool. ++ * If ensureStackDataExists() then allocates different ports, and the satellite misses the update ++ * (e.g. due to controller restart or connectivity issues), the satellite keeps the old ports ++ * while peers get the new ones, causing DRBD connections to fail with StandAlone state. ++ */ + private void copyDrbdTcpPortsIfExists(Resource rsc, LayerPayload payload) throws ImplementationError + { + Set> drbdRscDataSet = LayerRscUtils.getRscDataByLayer( @@ -82,56 +117,22 @@ index d93a18014..01cfbbacf 100644 + payload.drbdRsc.tcpPorts = portInts; + } + } -+ } -+ + } + private List removeLayerData(Resource rscRef) - { - List layerList; -@@ -1058,15 +1093,15 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL - /* - * We also have to remove the possible meta-children of previous StorageRscData. - * LayerData will be recreated with ensureStackDataExists. -- * However, we still need to remember our node-id if we had / have DRBD in the list -+ * However, we still need to remember our DRBD settings if we had / have DRBD in the list - */ -- copyDrbdNodeIdIfExists(rsc, payload); -+ copyDrbdSettings(rsc, payload); - layerList = removeLayerData(rsc); - } - else - { - markDiskAdded(rsc); -- ctrlLayerStackHelper.resetStoragePools(rsc); -+ ctrlLayerStackHelper.resetStoragePools(rsc, false); - } - ctrlLayerStackHelper.ensureStackDataExists(rsc, layerList, payload); - diff --git a/controller/src/main/java/com/linbit/linstor/layer/resource/CtrlRscLayerDataFactory.java b/controller/src/main/java/com/linbit/linstor/layer/resource/CtrlRscLayerDataFactory.java -index 3538b380c..f9733b6f1 100644 +index 3538b380c..4f589145e 100644 --- a/controller/src/main/java/com/linbit/linstor/layer/resource/CtrlRscLayerDataFactory.java +++ b/controller/src/main/java/com/linbit/linstor/layer/resource/CtrlRscLayerDataFactory.java -@@ -263,6 +263,11 @@ public class CtrlRscLayerDataFactory - } - - public void resetStoragePools(Resource rscRef) -+ { -+ resetStoragePools(rscRef, true); -+ } -+ -+ public void resetStoragePools(Resource rscRef, boolean callEnsureStackDataExistsRef) - { - try - { -@@ -276,8 +281,10 @@ public class CtrlRscLayerDataFactory - +@@ -276,8 +276,6 @@ public class CtrlRscLayerDataFactory + rscDataToProcess.addAll(rscData.getChildren()); } - - ensureStackDataExists(rscRef, null, new LayerPayload()); -+ if (callEnsureStackDataExistsRef) -+ { -+ ensureStackDataExists(rscRef, null, new LayerPayload()); -+ } } catch (AccessDeniedException exc) { +-- +2.39.5 (Apple Git-154) + diff --git a/packages/system/linstor/images/piraeus-server/patches/fix-luks-header-size.diff b/packages/system/linstor/images/piraeus-server/patches/fix-luks-header-size.diff deleted file mode 100644 index 8611825b..00000000 --- a/packages/system/linstor/images/piraeus-server/patches/fix-luks-header-size.diff +++ /dev/null @@ -1,570 +0,0 @@ -diff --git a/satellite/src/main/java/com/linbit/linstor/layer/storage/AbsStorageProvider.java b/satellite/src/main/java/com/linbit/linstor/layer/storage/AbsStorageProvider.java ---- a/satellite/src/main/java/com/linbit/linstor/layer/storage/AbsStorageProvider.java -+++ b/satellite/src/main/java/com/linbit/linstor/layer/storage/AbsStorageProvider.java -@@ -2081,7 +2081,6 @@ public abstract class AbsStorageProvider< - ) - throws IOException, AccessDeniedException - { -- final StorPoolName storPoolObjName = storPoolObj.getName(); - if (storDevicePath != null) - { - errorReporter.logDebug("updateMinIoSize: Have storDevicePath \"%s\"", storDevicePath); -@@ -2094,35 +2093,18 @@ public abstract class AbsStorageProvider< - "updateMinIoSize: Block device path is \"%s\"", - blockDevicePath.toString() - ); -- final long minIoSize = BlockSizeInfo.getBlockSize(blockDevicePath); -- -- boolean updateValue = true; -- -- final String propKey = StorageConstants.NAMESPACE_INTERNAL + '/' + -- StorageConstants.BLK_DEV_MIN_IO_SIZE; -- final Props storPoolProps = storPoolObj.getProps(storDriverAccCtx); -- final @Nullable String currentPropValue = storPoolProps.getProp(propKey); -- if (currentPropValue != null) -- { -- try -- { -- final long currentPropMinIoSize = Long.parseLong(currentPropValue); -- updateValue = currentPropMinIoSize != minIoSize; -- } -- catch (NumberFormatException ignored) -- { -- } -- } -- -- if (updateValue) -- { -- final String propValue = Long.toString(minIoSize); -- errorReporter.logDebug( -- "Storage pool \"%s\": Set property \"%s\" = \"%s\"", -- storPoolObjName.displayValue, propKey, propValue -- ); -- propsChange.changeStorPoolProp(storPoolObj, propKey, propValue); -- } -+ updatePropIfNeeded( -+ propsChange, -+ storPoolObj, -+ StorageConstants.NAMESPACE_INTERNAL + '/' + StorageConstants.BLK_DEV_MIN_IO_SIZE, -+ Long.toString(BlockSizeInfo.getBlockSize(blockDevicePath)) -+ ); -+ updatePropIfNeeded( -+ propsChange, -+ storPoolObj, -+ StorageConstants.NAMESPACE_INTERNAL + '/' + StorageConstants.BLK_DEV_OPT_IO_SIZE, -+ Long.toString(BlockSizeInfo.getOptimalIoSize(blockDevicePath)) -+ ); - } - else - { -@@ -2130,6 +2112,28 @@ public abstract class AbsStorageProvider< - } - } - -+ private void updatePropIfNeeded( -+ LocalPropsChangePojo propsChangeRef, -+ StorPool storPoolObjRef, -+ String propKeyRef, -+ String propValueRef -+ ) -+ throws AccessDeniedException -+ { -+ final Props storPoolProps = storPoolObjRef.getProps(storDriverAccCtx); -+ final @Nullable String currentPropValue = storPoolProps.getProp(propKeyRef); -+ if (currentPropValue == null || !currentPropValue.equals(propValueRef)) -+ { -+ errorReporter.logDebug( -+ "Storage pool \"%s\": Set property \"%s\" = \"%s\"", -+ storPoolObjRef.getName().displayValue, -+ propKeyRef, -+ propValueRef -+ ); -+ propsChangeRef.changeStorPoolProp(storPoolObjRef, propKeyRef, propValueRef); -+ } -+ } -+ - @SuppressWarnings("unchecked") - @Override - public void updateAllocatedSize(VlmProviderObject vlmDataRef) -diff --git a/satellite/src/main/java/com/linbit/linstor/layer/storage/utils/BlockSizeInfo.java b/satellite/src/main/java/com/linbit/linstor/layer/storage/utils/BlockSizeInfo.java ---- a/satellite/src/main/java/com/linbit/linstor/layer/storage/utils/BlockSizeInfo.java -+++ b/satellite/src/main/java/com/linbit/linstor/layer/storage/utils/BlockSizeInfo.java -@@ -3,8 +3,11 @@ package com.linbit.linstor.layer.storage.utils; - import com.linbit.utils.MathUtils; - import com.linbit.utils.SymbolicLinkResolver; - -+import static com.linbit.linstor.layer.storage.BlockSizeConsts.DFLT_OPT_IO_SIZE; - import static com.linbit.linstor.layer.storage.BlockSizeConsts.DFLT_IO_SIZE; -+import static com.linbit.linstor.layer.storage.BlockSizeConsts.MAX_OPT_IO_SIZE; - import static com.linbit.linstor.layer.storage.BlockSizeConsts.MAX_IO_SIZE; -+import static com.linbit.linstor.layer.storage.BlockSizeConsts.MIN_OPT_IO_SIZE; - import static com.linbit.linstor.layer.storage.BlockSizeConsts.MIN_IO_SIZE; - - import java.io.FileInputStream; -@@ -13,6 +16,8 @@ import java.nio.file.Path; - - public class BlockSizeInfo - { -+ private static final int NUMBER_BUFFER_SIZE = 32; -+ - /** - * Determines the blocksize, aka minimum I/O size, for the specified backing storage path. - * -@@ -51,7 +56,7 @@ public class BlockSizeInfo - final Path infoSourceName = blockDevice.getFileName(); - final Path infoSource = Path.of("/sys/block", infoSourceName.toString(), "queue/physical_block_size"); - -- final byte[] data = new byte[32]; -+ final byte[] data = new byte[NUMBER_BUFFER_SIZE]; - try (final FileInputStream fileIn = new FileInputStream(infoSource.toString())) - { - final int readCount = fileIn.read(data); -@@ -75,4 +80,38 @@ public class BlockSizeInfo - } - return blockSize; - } -+ -+ public static long getOptimalIoSize(final Path storageObj) -+ { -+ long optIoSize = DFLT_OPT_IO_SIZE; -+ try -+ { -+ final Path blockDevice = SymbolicLinkResolver.resolveSymLink(storageObj); -+ final Path infoSourceName = blockDevice.getFileName(); -+ final Path infoSource = Path.of("/sys/block", infoSourceName.toString(), "queue/optimal_io_size"); -+ -+ final byte[] data = new byte[NUMBER_BUFFER_SIZE]; -+ try (final FileInputStream fileIn = new FileInputStream(infoSource.toString())) -+ { -+ final int readCount = fileIn.read(data); -+ if (readCount > 0) -+ { -+ String numberStr = new String(data, 0, readCount); -+ numberStr = numberStr.trim(); -+ try -+ { -+ final long unboundedOptIoSize = Long.parseLong(numberStr); -+ optIoSize = MathUtils.bounds(MIN_OPT_IO_SIZE, unboundedOptIoSize, MAX_OPT_IO_SIZE); -+ } -+ catch (NumberFormatException ignored) -+ { -+ } -+ } -+ } -+ } -+ catch (IOException ignored) -+ { -+ } -+ return optIoSize; -+ } - } -diff --git a/server/src/main/java/com/linbit/SizeConv.java b/server/src/main/java/com/linbit/SizeConv.java ---- a/server/src/main/java/com/linbit/SizeConv.java -+++ b/server/src/main/java/com/linbit/SizeConv.java -@@ -16,6 +16,7 @@ public class SizeConv - public enum SizeUnit - { - UNIT_B, -+ UNIT_SECTORS, - UNIT_KiB, - UNIT_MiB, - UNIT_GiB, -@@ -41,6 +42,9 @@ public class SizeConv - case UNIT_B: - factor = FACTOR_B; - break; -+ case UNIT_SECTORS: -+ factor = FACTOR_SECTORS; -+ break; - case UNIT_KiB: - factor = FACTOR_KiB; - break; -@@ -111,6 +115,9 @@ public class SizeConv - case "b": - unit = SizeUnit.UNIT_B; - break; -+ case "s": -+ unit = SizeUnit.UNIT_SECTORS; -+ break; - case "k": - // fall-through - case "kb": -@@ -217,6 +224,9 @@ public class SizeConv - // Factor 1 - public static final BigInteger FACTOR_B = BigInteger.valueOf(1L); - -+ // Factor 512 -+ public static final BigInteger FACTOR_SECTORS = BigInteger.valueOf(512L); -+ - // Factor 1,024 - // Naming convention exception: SI unit capitalization rules - @SuppressWarnings("checkstyle:constantname") -diff --git a/server/src/main/java/com/linbit/linstor/layer/luks/LuksLayerSizeCalculator.java b/server/src/main/java/com/linbit/linstor/layer/luks/LuksLayerSizeCalculator.java ---- a/server/src/main/java/com/linbit/linstor/layer/luks/LuksLayerSizeCalculator.java -+++ b/server/src/main/java/com/linbit/linstor/layer/luks/LuksLayerSizeCalculator.java -@@ -1,27 +1,62 @@ - package com.linbit.linstor.layer.luks; - -+import com.linbit.SizeConv; -+import com.linbit.SizeConv.SizeUnit; - import com.linbit.exceptions.InvalidSizeException; -+import com.linbit.linstor.PriorityProps; -+import com.linbit.linstor.annotation.Nullable; -+import com.linbit.linstor.api.ApiConsts; -+import com.linbit.linstor.core.objects.AbsResource; -+import com.linbit.linstor.core.objects.AbsVolume; -+import com.linbit.linstor.core.objects.Resource; -+import com.linbit.linstor.core.objects.ResourceDefinition; -+import com.linbit.linstor.core.objects.ResourceGroup; -+import com.linbit.linstor.core.objects.Snapshot; -+import com.linbit.linstor.core.objects.SnapshotVolume; -+import com.linbit.linstor.core.objects.StorPool; -+import com.linbit.linstor.core.objects.Volume; -+import com.linbit.linstor.core.objects.VolumeDefinition; - import com.linbit.linstor.dbdrivers.DatabaseException; - import com.linbit.linstor.layer.AbsLayerSizeCalculator; -+import com.linbit.linstor.netcom.Peer; -+import com.linbit.linstor.propscon.InvalidKeyException; -+import com.linbit.linstor.propscon.ReadOnlyProps; - import com.linbit.linstor.security.AccessDeniedException; -+import com.linbit.linstor.storage.StorageConstants; - import com.linbit.linstor.storage.data.adapter.luks.LuksVlmData; - import com.linbit.linstor.storage.interfaces.categories.resource.VlmProviderObject; - import com.linbit.linstor.storage.kinds.DeviceLayerKind; - import com.linbit.linstor.storage.kinds.ExtTools; - import com.linbit.linstor.storage.kinds.ExtToolsInfo; - import com.linbit.linstor.storage.kinds.ExtToolsInfo.Version; -+import com.linbit.linstor.utils.layer.LayerVlmUtils; -+import com.linbit.utils.ShellUtils; -+import com.linbit.utils.SignedAlign; - - import javax.inject.Inject; - import javax.inject.Singleton; - -+import java.util.Iterator; -+import java.util.LinkedList; -+import java.util.List; -+import java.util.Set; -+import java.util.regex.Matcher; -+import java.util.regex.Pattern; -+ - @Singleton - public class LuksLayerSizeCalculator extends AbsLayerSizeCalculator> - { -+ public static final String LUKS2_OPT_METADATA_SIZE = "--luks2-metadata-size"; -+ public static final String LUKS2_OPT_KEYSLOTS_SIZE = "--luks2-keyslots-size"; -+ public static final String LUKS2_OPT_OFFSET = "--offset"; -+ private static final String LUKS2_OPT_ALIGN_PAYLOAD = "--align-payload"; -+ private static final long DFLT_LUKS2_METADATA_SIZE_IN_BYTES = 16L << 10; -+ private static final long DFLT_ALIGNMENT_1MIB_IN_BYTES = 1L << 20; -+ private static final long LUKS1_HEADER_SIZE_IN_KIB = 2L << 10; -+ private static final long LUKS2_HEADER_SIZE_IN_KIB = 16L << 10; -+ private static final long LUKS2_HEADER_SIZE_IN_BYTES = LUKS2_HEADER_SIZE_IN_KIB << 10; - -- // linstor calculates in KiB -- private static final int MIB = 1024; -- private static final int LUKS1_HEADER_SIZE = 2 * MIB; -- private static final int LUKS2_HEADER_SIZE = 16 * MIB; -+ private static final Pattern PATTERN_SIZE = Pattern.compile("(\\d+)([kKmMgGtT]i?[bB]?|[sS]|)"); - - @Inject - public LuksLayerSizeCalculator(AbsLayerSizeCalculatorInit initRef) -@@ -74,30 +109,266 @@ public class LuksLayerSizeCalculator extends AbsLayerSizeCalculator vlmDataRef) -- throws AccessDeniedException -+ throws AccessDeniedException, InvalidSizeException - { -- ExtToolsInfo cryptSetupInfo = vlmDataRef.getRscLayerObject() -+ @Nullable Peer peer = vlmDataRef.getRscLayerObject() - .getAbsResource() - .getNode() -- .getPeer(sysCtx) -- .getExtToolsManager() -+ .getPeer(sysCtx); -+ if (peer == null) -+ { -+ throw new InvalidSizeException( -+ "Could not calculate size of LUKS volume, since cryptsetup's version could not be determined", -+ null -+ ); -+ } -+ ExtToolsInfo cryptSetupInfo = peer.getExtToolsManager() - .getExtToolInfo(ExtTools.CRYPT_SETUP); -- long luksHeaderSize; -+ -+ final long luksHeaderSize; - if (cryptSetupInfo != null && cryptSetupInfo.isSupported()) - { - if (cryptSetupInfo.hasVersionOrHigher(new Version(2, 1))) - { -- luksHeaderSize = LUKS2_HEADER_SIZE; -+ luksHeaderSize = calcLuks2HeaderSize(vlmDataRef); - } - else - { -- luksHeaderSize = LUKS1_HEADER_SIZE; -+ luksHeaderSize = LUKS1_HEADER_SIZE_IN_KIB; - } - } - else - { -- luksHeaderSize = -1; -+ throw new InvalidSizeException( -+ "Could not calculate size of LUKS volume, since cryptsetup's version could not be determined", -+ null -+ ); - } - return luksHeaderSize; - } -+ -+ private long calcLuks2HeaderSize(VlmProviderObject vlmDataRef) throws AccessDeniedException -+ { -+ PriorityProps prioProps = getPrioProps(vlmDataRef); -+ -+ final @Nullable String userOptProp = prioProps.getProp( -+ ApiConsts.KEY_STOR_DRIVER_LUKS_FORMAT_OPTIONS, -+ ApiConsts.NAMESPC_STORAGE_DRIVER -+ ); -+ List userOptions = userOptProp != null ? -+ ShellUtils.shellSplit(userOptProp) : -+ new LinkedList<>(); -+ -+ final long alignedHeaderSizeInBytes; -+ final long unalignedHeaderSizeInBytes; -+ @Nullable Long cryptsetupOffsetInBytes = getLongOptionValue( -+ userOptions, -+ LUKS2_OPT_OFFSET, -+ SizeUnit.UNIT_SECTORS -+ ); -+ if (cryptsetupOffsetInBytes != null) -+ { -+ alignedHeaderSizeInBytes = cryptsetupOffsetInBytes; -+ } -+ else -+ { -+ @Nullable Long cryptsetupLuks2KeyslotsSize = getLongOptionValue( -+ userOptions, -+ LUKS2_OPT_KEYSLOTS_SIZE -+ ); -+ if (cryptsetupLuks2KeyslotsSize == null) -+ { -+ unalignedHeaderSizeInBytes = LUKS2_HEADER_SIZE_IN_BYTES; -+ } -+ else -+ { -+ @Nullable Long cryptsetupLuks2MetadataSize = getLongOptionValue( -+ userOptions, -+ LUKS2_OPT_METADATA_SIZE -+ ); -+ cryptsetupLuks2MetadataSize = cryptsetupLuks2MetadataSize == null ? -+ DFLT_LUKS2_METADATA_SIZE_IN_BYTES : -+ cryptsetupLuks2MetadataSize; -+ -+ unalignedHeaderSizeInBytes = 2 * cryptsetupLuks2MetadataSize + cryptsetupLuks2KeyslotsSize; -+ } -+ -+ long alignment = getAlignment(vlmDataRef, userOptions); -+ alignedHeaderSizeInBytes = new SignedAlign(alignment).ceiling(unalignedHeaderSizeInBytes); -+ } -+ return SizeConv.convert( -+ alignedHeaderSizeInBytes, -+ SizeUnit.UNIT_B, -+ SizeUnit.UNIT_KiB -+ ); -+ } -+ -+ private long getAlignment(VlmProviderObject vlmDataRef, List userOptions) throws AccessDeniedException -+ { -+ @Nullable Long cryptsetupAlignPayloadInBytes = getLongOptionValue( -+ userOptions, -+ LUKS2_OPT_ALIGN_PAYLOAD, -+ SizeUnit.UNIT_SECTORS -+ ); -+ -+ long alignment = DFLT_ALIGNMENT_1MIB_IN_BYTES; -+ if (cryptsetupAlignPayloadInBytes != null) -+ { -+ alignment = cryptsetupAlignPayloadInBytes; -+ } -+ else -+ { -+ final long maxOptIoSize = getMaxOptIoSize(vlmDataRef); -+ alignment = Math.max(alignment, maxOptIoSize); -+ } -+ return alignment; -+ } -+ -+ private long getMaxOptIoSize(VlmProviderObject vlmDataRef) throws InvalidKeyException, AccessDeniedException -+ { -+ long ret = 0; -+ Set storPoolSet = LayerVlmUtils.getStorPoolSet(vlmDataRef, sysCtx); -+ for (StorPool sp : storPoolSet) -+ { -+ @Nullable String strValue = sp.getProps(sysCtx) -+ .getProp( -+ StorageConstants.BLK_DEV_OPT_IO_SIZE, -+ StorageConstants.NAMESPACE_INTERNAL -+ ); -+ if (strValue != null) -+ { -+ try -+ { -+ long parsed = Long.parseLong(strValue); -+ ret = Math.max(parsed, ret); -+ } -+ catch (NumberFormatException ignored) -+ { -+ errorReporter.logWarning( -+ "LuksHeaderSize: Failed to parse '%s' from prop %s. Defaulting to 0 opt_io_size " + -+ "(no recommendation/hint)", -+ strValue, -+ StorageConstants.NAMESPACE_INTERNAL + "/" + StorageConstants.BLK_DEV_OPT_IO_SIZE -+ ); -+ } -+ } -+ } -+ return ret; -+ } -+ -+ private @Nullable Long getLongOptionValue(List userOptionsRef, String optRef) -+ { -+ return getLongOptionValue(userOptionsRef, optRef, SizeUnit.UNIT_B); -+ } -+ -+ @SuppressWarnings("checkstyle:magicnumber") -+ private @Nullable Long getLongOptionValue(List userOptionsRef, String optRef, SizeUnit dfltSizeUnit) -+ { -+ @Nullable Long ret = null; -+ @Nullable String val = findLastValue(userOptionsRef, optRef); -+ if (val != null && !val.isBlank()) -+ { -+ Matcher matcher = PATTERN_SIZE.matcher(val); -+ if (matcher.matches()) -+ { -+ try -+ { -+ ret = Long.parseLong(matcher.group(1)); -+ String unit = matcher.group(2); -+ SizeUnit sizeUnit; -+ if (unit.isBlank()) -+ { -+ sizeUnit = dfltSizeUnit; -+ } -+ else -+ { -+ boolean forcePowerOfTwo = unit.length() == 1 || unit.length() == 3; -+ sizeUnit = SizeUnit.parse(unit, forcePowerOfTwo); -+ } -+ -+ ret = SizeConv.convert(ret, sizeUnit, SizeUnit.UNIT_B); -+ } -+ catch (NumberFormatException ignored) -+ { -+ errorReporter.logWarning( -+ "LuksHeaderSize: Failed to parse '%s' from option '%s %s'.", -+ matcher.group(1), -+ optRef, -+ val -+ ); -+ } -+ } -+ } -+ return ret; -+ } -+ -+ private @Nullable String findLastValue(List userOptionsRef, String optRef) -+ { -+ Iterator it = userOptionsRef.iterator(); -+ @Nullable String val = null; -+ while (it.hasNext()) -+ { -+ String opt = it.next(); -+ if (opt.equals(optRef)) -+ { -+ if (it.hasNext()) -+ { -+ val = it.next(); -+ } -+ else -+ { -+ val = null; -+ } -+ } -+ else if (opt.startsWith(optRef + "=")) -+ { -+ val = opt.substring(optRef.length() + 1); -+ if (val.isBlank()) -+ { -+ val = null; -+ } -+ } -+ } -+ return val; -+ } -+ -+ private PriorityProps getPrioProps(VlmProviderObject vlmDataRef) throws AccessDeniedException -+ { -+ final AbsVolume vlm = vlmDataRef.getVolume(); -+ final AbsResource rsc = vlm.getAbsResource(); -+ final ResourceDefinition rscDfn = vlm.getResourceDefinition(); -+ final VolumeDefinition vlmDfn = vlm.getVolumeDefinition(); -+ final ResourceGroup rscGrp = rscDfn.getResourceGroup(); -+ -+ final ReadOnlyProps vlmProps; -+ final ReadOnlyProps rscProps; -+ if (vlm instanceof Volume) -+ { -+ vlmProps = ((Volume) vlm).getProps(sysCtx); -+ rscProps = ((Resource) rsc).getProps(sysCtx); -+ } -+ else -+ { -+ vlmProps = ((SnapshotVolume) vlm).getVlmProps(sysCtx); -+ rscProps = ((Snapshot) rsc).getRscProps(sysCtx); -+ } -+ -+ final PriorityProps prioProps = new PriorityProps( -+ vlmProps, -+ rscProps -+ ); -+ for (StorPool storPool : LayerVlmUtils.getStorPoolSet(vlmDataRef, sysCtx)) -+ { -+ prioProps.addProps(storPool.getProps(sysCtx)); -+ } -+ prioProps.addProps( -+ rsc.getNode().getProps(sysCtx), -+ vlmDfn.getProps(sysCtx), -+ rscDfn.getProps(sysCtx), -+ rscGrp.getVolumeGroupProps(sysCtx, vlmDfn.getVolumeNumber()), -+ rscGrp.getProps(sysCtx), -+ stltProps -+ ); -+ return prioProps; -+ } - } -diff --git a/server/src/main/java/com/linbit/linstor/layer/storage/BlockSizeConsts.java b/server/src/main/java/com/linbit/linstor/layer/storage/BlockSizeConsts.java ---- a/server/src/main/java/com/linbit/linstor/layer/storage/BlockSizeConsts.java -+++ b/server/src/main/java/com/linbit/linstor/layer/storage/BlockSizeConsts.java -@@ -13,4 +13,9 @@ public class BlockSizeConsts - - // Default value for the minimum_io_size value of non-storage layers - public static final long DFLT_SPECIAL_IO_SIZE = (1L << 12); -+ -+ // optimal_io_size may be 0 to indicate no recommendation. -+ public static final long MIN_OPT_IO_SIZE = 0; -+ public static final long DFLT_OPT_IO_SIZE = 0; -+ public static final long MAX_OPT_IO_SIZE = Long.MAX_VALUE; - } -diff --git a/server/src/main/java/com/linbit/linstor/storage/StorageConstants.java b/server/src/main/java/com/linbit/linstor/storage/StorageConstants.java ---- a/server/src/main/java/com/linbit/linstor/storage/StorageConstants.java -+++ b/server/src/main/java/com/linbit/linstor/storage/StorageConstants.java -@@ -11,6 +11,7 @@ public class StorageConstants - public static final String NAMESPACE_NVME = ApiConsts.NAMESPC_STORAGE_DRIVER + "/NVME"; - public static final String NAMESPACE_INTERNAL = NAMESPACE_STOR_DRIVER + "/internal/"; - -+ public static final String BLK_DEV_OPT_IO_SIZE = "optIoSize"; - public static final String BLK_DEV_MIN_IO_SIZE = "minIoSize"; - public static final String BLK_DEV_MIN_IO_SIZE_AUTO = "minIoSizeAuto"; - public static final String BLK_DEV_MAX_BIO_SIZE = "maxBioSize"; diff --git a/packages/system/linstor/images/piraeus-server/patches/force-metadata-check-on-disk-add.diff b/packages/system/linstor/images/piraeus-server/patches/force-metadata-check-on-disk-add.diff new file mode 100644 index 00000000..0c413005 --- /dev/null +++ b/packages/system/linstor/images/piraeus-server/patches/force-metadata-check-on-disk-add.diff @@ -0,0 +1,63 @@ +diff --git a/satellite/src/main/java/com/linbit/linstor/layer/drbd/DrbdLayer.java b/satellite/src/main/java/com/linbit/linstor/layer/drbd/DrbdLayer.java +index a302ee835..01967a31f 100644 +--- a/satellite/src/main/java/com/linbit/linstor/layer/drbd/DrbdLayer.java ++++ b/satellite/src/main/java/com/linbit/linstor/layer/drbd/DrbdLayer.java +@@ -371,10 +371,13 @@ public class DrbdLayer implements DeviceLayer + boolean isDiskless = drbdRscData.getAbsResource().isDrbdDiskless(workerCtx); + StateFlags rscFlags = drbdRscData.getAbsResource().getStateFlags(); + boolean isDiskRemoving = rscFlags.isSet(workerCtx, Resource.Flags.DISK_REMOVING); ++ // Check if we're in toggle-disk operation (adding disk to diskless resource) ++ boolean isDiskAdding = rscFlags.isSomeSet(workerCtx, Resource.Flags.DISK_ADD_REQUESTED, Resource.Flags.DISK_ADDING); + + boolean contProcess = isDiskless; + +- boolean processChildren = !isDiskless || isDiskRemoving; ++ // Process children when: has disk, removing disk, OR adding disk (toggle-disk) ++ boolean processChildren = !isDiskless || isDiskRemoving || isDiskAdding; + // do not process children when ONLY DRBD_DELETE flag is set (DELETE flag is still unset) + processChildren &= (!rscFlags.isSet(workerCtx, Resource.Flags.DRBD_DELETE) || + rscFlags.isSet(workerCtx, Resource.Flags.DELETE)); +@@ -570,7 +573,11 @@ public class DrbdLayer implements DeviceLayer + { + // hasMetaData needs to be run after child-resource processed + List> createMetaData = new ArrayList<>(); +- if (!drbdRscData.getAbsResource().isDrbdDiskless(workerCtx) && !skipDisk) ++ // Check if we're in toggle-disk operation (adding disk to diskless resource) ++ boolean isDiskAddingForMd = drbdRscData.getAbsResource().getStateFlags() ++ .isSomeSet(workerCtx, Resource.Flags.DISK_ADD_REQUESTED, Resource.Flags.DISK_ADDING); ++ // Create metadata when: has disk OR adding disk (toggle-disk), and skipDisk is disabled ++ if ((!drbdRscData.getAbsResource().isDrbdDiskless(workerCtx) || isDiskAddingForMd) && !skipDisk) + { + // do not try to create meta data while the resource is diskless or skipDisk is enabled + for (DrbdVlmData drbdVlmData : checkMetaData) +@@ -988,8 +995,10 @@ public class DrbdLayer implements DeviceLayer + { + List> checkMetaData = new ArrayList<>(); + Resource rsc = drbdRscData.getAbsResource(); ++ // Include DISK_ADD_REQUESTED/DISK_ADDING for toggle-disk scenario where we need to check/create metadata + if (!rsc.isDrbdDiskless(workerCtx) || +- rsc.getStateFlags().isSet(workerCtx, Resource.Flags.DISK_REMOVING) ++ rsc.getStateFlags().isSet(workerCtx, Resource.Flags.DISK_REMOVING) || ++ rsc.getStateFlags().isSomeSet(workerCtx, Resource.Flags.DISK_ADD_REQUESTED, Resource.Flags.DISK_ADDING) + ) + { + // using a dedicated list to prevent concurrentModificationException +@@ -1177,9 +1186,16 @@ public class DrbdLayer implements DeviceLayer + + boolean hasMetaData; + ++ // Check if we need to verify/create metadata ++ // Force metadata check when: ++ // 1. checkMetaData is enabled ++ // 2. volume doesn't have disk yet (diskless -> diskful transition) ++ // 3. DISK_ADD_REQUESTED/DISK_ADDING flag is set (retry scenario where storage exists but no metadata) ++ boolean isDiskAddingState = drbdVlmData.getRscLayerObject().getAbsResource().getStateFlags() ++ .isSomeSet(workerCtx, Resource.Flags.DISK_ADD_REQUESTED, Resource.Flags.DISK_ADDING); + if (drbdVlmData.checkMetaData() || +- // when adding a disk, DRBD believes that it is diskless but we still need to create metadata +- !drbdVlmData.hasDisk()) ++ !drbdVlmData.hasDisk() || ++ isDiskAddingState) + { + if (mdUtils.hasMetaData()) + { diff --git a/packages/system/linstor/images/piraeus-server/patches/retry-adjust-after-stale-bitmap.diff b/packages/system/linstor/images/piraeus-server/patches/retry-adjust-after-stale-bitmap.diff deleted file mode 100644 index 00959167..00000000 --- a/packages/system/linstor/images/piraeus-server/patches/retry-adjust-after-stale-bitmap.diff +++ /dev/null @@ -1,141 +0,0 @@ -diff --git a/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/DrbdAdm.java b/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/DrbdAdm.java -index 5627d1be8..ece191292 100644 ---- a/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/DrbdAdm.java -+++ b/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/DrbdAdm.java -@@ -42,6 +42,9 @@ import java.util.Arrays; - import java.util.List; - import java.util.concurrent.ArrayBlockingQueue; - import java.util.concurrent.TimeUnit; -+import java.nio.charset.StandardCharsets; -+import java.util.regex.Matcher; -+import java.util.regex.Pattern; - import java.util.stream.Collectors; - - @Singleton -@@ -56,6 +58,9 @@ public class DrbdAdm - - public static final int WAIT_CONNECT_RES_TIME = 10; - private static final long DOWN_WAIT_TIMEOUT_SEC = 5; -+ private static final long FORCE_DETACH_RETRY_WAIT_MS = 250; -+ private static final String BITMAP_LEAK_ERR_MSG = "already has a bitmap, this should not happen"; -+ private static final Pattern BITMAP_LEAK_MINOR_PATTERN = Pattern.compile("\\bminor\\s+(\\d+)\\b"); - - private final ExtCmdFactory extCmdFactory; - private final AccessContext sysCtx; -@@ -131,8 +136,38 @@ public class DrbdAdm - // command.add(resName); - command.add(drbdRscData.getSuffixedResourceName()); - // execute(Arrays.asList("drbdsetup", "show", drbdRscData.getSuffixedResourceName())); -- execute(command); -- // execute(Arrays.asList("drbdsetup", "show", drbdRscData.getSuffixedResourceName())); -+ String[] commandArr = command.toArray(new String[0]); -+ try -+ { -+ File nullDevice = new File(Platform.nullDevice()); -+ ExtCmd extCmd = extCmdFactory.create(); -+ if (Platform.isWindows()) -+ { -+ extCmd.setTimeout(TimeoutType.WAIT, 5 * 60 * 1000); -+ } -+ -+ OutputData outputData = extCmd.pipeExec(ProcessBuilder.Redirect.from(nullDevice), commandArr); -+ if ( -+ outputData.exitCode != 0 && -+ isBitmapLeakOnAttach(outputData) && -+ cleanupStaleBitmapAndRetry(extCmd, nullDevice, outputData) -+ ) -+ { -+ outputData = extCmd.pipeExec(ProcessBuilder.Redirect.from(nullDevice), commandArr); -+ } -+ if (outputData.exitCode != 0) -+ { -+ throw new ExtCmdFailedException(commandArr, outputData); -+ } -+ } -+ catch (ChildProcessTimeoutException timeoutExc) -+ { -+ throw new ExtCmdFailedException(commandArr, timeoutExc); -+ } -+ catch (IOException ioExc) -+ { -+ throw new ExtCmdFailedException(commandArr, ioExc); -+ } - - drbdRscData.setAdjustRequired(false); - } -@@ -805,6 +840,75 @@ public class DrbdAdm - } - } - -+ static boolean isBitmapLeakOnAttach(OutputData outputData) -+ { -+ return extractBitmapLeakMinor(outputData) != null; -+ } -+ -+ static @Nullable Integer extractBitmapLeakMinor(OutputData outputData) -+ { -+ String stderr = new String(outputData.stderrData, StandardCharsets.UTF_8); -+ if (!stderr.contains(BITMAP_LEAK_ERR_MSG)) -+ { -+ return null; -+ } -+ -+ Matcher matcher = BITMAP_LEAK_MINOR_PATTERN.matcher(stderr); -+ if (!matcher.find()) -+ { -+ return null; -+ } -+ -+ return Integer.parseInt(matcher.group(1)); -+ } -+ -+ private boolean cleanupStaleBitmapAndRetry( -+ ExtCmd extCmd, -+ File nullDevice, -+ OutputData outputData -+ ) -+ throws IOException, ChildProcessTimeoutException, ExtCmdFailedException -+ { -+ @Nullable Integer minor = extractBitmapLeakMinor(outputData); -+ if (minor == null) -+ { -+ return false; -+ } -+ -+ OutputData detachOut = extCmd.pipeExec( -+ ProcessBuilder.Redirect.from(nullDevice), -+ DRBDSETUP_UTIL, -+ "detach", -+ Integer.toString(minor) -+ ); -+ if (detachOut.exitCode == 0) -+ { -+ return true; -+ } -+ -+ OutputData forceDetachOut = extCmd.pipeExec( -+ ProcessBuilder.Redirect.from(nullDevice), -+ DRBDSETUP_UTIL, -+ "detach", -+ Integer.toString(minor), -+ "--force" -+ ); -+ if (forceDetachOut.exitCode != 0) -+ { -+ throw new ExtCmdFailedException(forceDetachOut.executedCommand, forceDetachOut); -+ } -+ -+ try -+ { -+ Thread.sleep(FORCE_DETACH_RETRY_WAIT_MS); -+ } -+ catch (InterruptedException ignored) -+ { -+ Thread.currentThread().interrupt(); -+ } -+ return true; -+ } -+ - public static class DrbdPrimary implements AutoCloseable - { - private final DrbdAdm drbdAdm; diff --git a/packages/system/linstor/images/piraeus-server/patches/skip-adjust-when-device-inaccessible.diff b/packages/system/linstor/images/piraeus-server/patches/skip-adjust-when-device-inaccessible.diff new file mode 100644 index 00000000..a93d7811 --- /dev/null +++ b/packages/system/linstor/images/piraeus-server/patches/skip-adjust-when-device-inaccessible.diff @@ -0,0 +1,155 @@ +diff --git a/satellite/src/main/java/com/linbit/linstor/core/devmgr/DeviceHandlerImpl.java b/satellite/src/main/java/com/linbit/linstor/core/devmgr/DeviceHandlerImpl.java +index 49138a8fd..2f768ca0d 100644 +--- a/satellite/src/main/java/com/linbit/linstor/core/devmgr/DeviceHandlerImpl.java ++++ b/satellite/src/main/java/com/linbit/linstor/core/devmgr/DeviceHandlerImpl.java +@@ -83,6 +83,8 @@ import java.util.TreeMap; + import java.util.TreeSet; + import java.util.concurrent.atomic.AtomicBoolean; + import java.util.function.Function; ++import java.nio.file.Files; ++import java.nio.file.Paths; + + @Singleton + public class DeviceHandlerImpl implements DeviceHandler +@@ -1646,7 +1648,10 @@ public class DeviceHandlerImpl implements DeviceHandler + private void updateDiscGran(VlmProviderObject vlmData) throws DatabaseException, StorageException + { + String devicePath = vlmData.getDevicePath(); +- if (devicePath != null && vlmData.exists()) ++ // Check if device path physically exists before calling lsblk ++ // This is important for DRBD devices which might be temporarily unavailable during adjust ++ // (drbdadm adjust brings devices down/up, and kernel might not have created the device node yet) ++ if (devicePath != null && vlmData.exists() && Files.exists(Paths.get(devicePath))) + { + if (vlmData.getDiscGran() == VlmProviderObject.UNINITIALIZED_SIZE) + { +diff --git a/satellite/src/main/java/com/linbit/linstor/layer/drbd/DrbdLayer.java b/satellite/src/main/java/com/linbit/linstor/layer/drbd/DrbdLayer.java +index 01967a31f..78b8195a4 100644 +--- a/satellite/src/main/java/com/linbit/linstor/layer/drbd/DrbdLayer.java ++++ b/satellite/src/main/java/com/linbit/linstor/layer/drbd/DrbdLayer.java +@@ -592,7 +592,29 @@ public class DrbdLayer implements DeviceLayer + // The .res file might not have been generated in the prepare method since it was + // missing information from the child-layers. Now that we have processed them, we + // need to make sure the .res file exists in all circumstances. +- regenerateResFile(drbdRscData); ++ // However, if the underlying devices are not accessible (e.g., LUKS device is closed ++ // during resource deletion), we skip regenerating the res file to avoid errors ++ boolean canRegenerateResFile = true; ++ if (!skipDisk && !drbdRscData.getAbsResource().isDrbdDiskless(workerCtx)) ++ { ++ AbsRscLayerObject dataChild = drbdRscData.getChildBySuffix(RscLayerSuffixes.SUFFIX_DATA); ++ if (dataChild != null) ++ { ++ for (DrbdVlmData drbdVlmData : drbdRscData.getVlmLayerObjects().values()) ++ { ++ VlmProviderObject childVlm = dataChild.getVlmProviderObject(drbdVlmData.getVlmNr()); ++ if (childVlm == null || !childVlm.exists() || childVlm.getDevicePath() == null) ++ { ++ canRegenerateResFile = false; ++ break; ++ } ++ } ++ } ++ } ++ if (canRegenerateResFile) ++ { ++ regenerateResFile(drbdRscData); ++ } + + // createMetaData needs rendered resFile + for (DrbdVlmData drbdVlmData : createMetaData) +@@ -766,19 +788,72 @@ public class DrbdLayer implements DeviceLayer + + if (drbdRscData.isAdjustRequired()) + { +- try ++ // Check if underlying devices are accessible before adjusting ++ // This is important for encrypted resources (LUKS) where the device ++ // might be closed during deletion ++ boolean canAdjust = true; ++ ++ // IMPORTANT: Check child volumes only when disk access is actually needed. ++ // For network reconnect (StandAlone -> Connected), disk access is not required. ++ boolean needsDiskAccess = false; ++ ++ // Check if there are pending operations that require disk access ++ for (DrbdVlmData drbdVlmData : drbdRscData.getVlmLayerObjects().values()) + { +- drbdUtils.adjust( +- drbdRscData, +- false, +- skipDisk, +- false +- ); ++ Volume vlm = (Volume) drbdVlmData.getVolume(); ++ StateFlags vlmFlags = vlm.getFlags(); ++ ++ // Disk access is needed if: ++ // - creating a new volume ++ // - resizing ++ // - checking/creating metadata ++ if (!drbdVlmData.exists() || ++ drbdVlmData.checkMetaData() || ++ vlmFlags.isSomeSet(workerCtx, Volume.Flags.RESIZE, Volume.Flags.DRBD_RESIZE)) ++ { ++ needsDiskAccess = true; ++ break; ++ } ++ } ++ ++ // Check child volumes only if disk access is actually needed ++ if (needsDiskAccess && !skipDisk && !drbdRscData.getAbsResource().isDrbdDiskless(workerCtx)) ++ { ++ AbsRscLayerObject dataChild = drbdRscData.getChildBySuffix(RscLayerSuffixes.SUFFIX_DATA); ++ if (dataChild != null) ++ { ++ for (DrbdVlmData drbdVlmData : drbdRscData.getVlmLayerObjects().values()) ++ { ++ VlmProviderObject childVlm = dataChild.getVlmProviderObject(drbdVlmData.getVlmNr()); ++ if (childVlm == null || !childVlm.exists() || childVlm.getDevicePath() == null) ++ { ++ canAdjust = false; ++ break; ++ } ++ } ++ } + } +- catch (ExtCmdFailedException extCmdExc) ++ ++ if (canAdjust) ++ { ++ try ++ { ++ drbdUtils.adjust( ++ drbdRscData, ++ false, ++ skipDisk, ++ false ++ ); ++ } ++ catch (ExtCmdFailedException extCmdExc) ++ { ++ restoreBackupResFile(drbdRscData); ++ throw extCmdExc; ++ } ++ } ++ else + { +- restoreBackupResFile(drbdRscData); +- throw extCmdExc; ++ drbdRscData.setAdjustRequired(false); + } + } + +diff --git a/satellite/src/main/java/com/linbit/linstor/layer/luks/LuksLayer.java b/satellite/src/main/java/com/linbit/linstor/layer/luks/LuksLayer.java +index cdca0b6d2..89c8be9da 100644 +--- a/satellite/src/main/java/com/linbit/linstor/layer/luks/LuksLayer.java ++++ b/satellite/src/main/java/com/linbit/linstor/layer/luks/LuksLayer.java +@@ -383,6 +383,7 @@ public class LuksLayer implements DeviceLayer + vlmData.setSizeState(Size.AS_EXPECTED); + + vlmData.setOpened(true); ++ vlmData.setExists(true); + vlmData.setFailed(false); + } + } diff --git a/packages/system/linstor/templates/satellites-cozy.yaml b/packages/system/linstor/templates/satellites-cozy.yaml index e6f877a0..c621126d 100644 --- a/packages/system/linstor/templates/satellites-cozy.yaml +++ b/packages/system/linstor/templates/satellites-cozy.yaml @@ -14,11 +14,6 @@ spec: containers: - name: linstor-satellite image: {{ .Values.piraeusServer.image.repository }}:{{ .Values.piraeusServer.image.tag }} - startupProbe: - tcpSocket: - port: linstor - periodSeconds: 10 - failureThreshold: 30 securityContext: # real-world installations need some debugging from time to time readOnlyRootFilesystem: false diff --git a/packages/system/linstor/values.yaml b/packages/system/linstor/values.yaml index 62dd2cac..b3f13443 100644 --- a/packages/system/linstor/values.yaml +++ b/packages/system/linstor/values.yaml @@ -1,7 +1,7 @@ piraeusServer: image: repository: ghcr.io/cozystack/cozystack/piraeus-server - tag: 1.33.2@sha256:553f313ab35dc2e345ef3683156d29e75c23177e2750e9af3a83aa9e23941cbb + tag: 1.32.3@sha256:d78071fdc33220168e3f2ab112a86208be95898b75268a0c62763b8a04e145ad # Talos-specific workarounds (disable for generic Linux like Ubuntu/Debian) talos: enabled: true @@ -13,4 +13,4 @@ linstor: linstorCSI: image: repository: ghcr.io/cozystack/cozystack/linstor-csi - tag: v1.10.5@sha256:b8f59b5659fb1791cb764d3f37df4cf29920aadcc10637231ba7d857233f377d + tag: v1.10.5@sha256:e80ae94d8acd8774cb35715b1f103071ae6cd3160f1ebea915e5e6ee49234448 diff --git a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml b/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml deleted file mode 100644 index 5e0f9ea0..00000000 --- a/packages/system/monitoring-agents/alerts/gpu-recording.rules.yaml +++ /dev/null @@ -1,172 +0,0 @@ -apiVersion: operator.victoriametrics.com/v1beta1 -kind: VMRule -metadata: - name: alerts-gpu-recording.rules -spec: - groups: - - name: gpu.recording.cluster.1m - interval: 1m - params: {} - rules: - - record: cluster:gpu_count:total - expr: count(group by (UUID) (DCGM_FI_DEV_GPU_UTIL)) - # Kube-allocated GPU count: GPUs requested by *active* pods — Pending - # and Running only. Source of truth for "what tenants asked for" — used - # for capacity planning and billing. Includes system pods so it stays - # consistent with cluster:gpu_count:total when computing :free. - # - # Phase filter via group_left() on kube_pod_status_phase matches the - # canonical pattern from k8s.rules.container_resource.yaml. Without it - # kube-state-metrics keeps series for Failed/Succeeded pods until the - # apiserver garbage-collects them, which inflates billing hours and - # can push :free negative. - - record: cluster:gpu_count:allocated - expr: | - sum( - kube_pod_container_resource_requests{resource="nvidia_com_gpu"} - * on (namespace, pod) group_left() max by (namespace, pod) ( - kube_pod_status_phase{phase=~"Pending|Running"} == 1 - ) - ) - # clamp_min guards against transients at DCGM/kube-state-metrics - # restart where cluster:gpu_count:total dips before :allocated catches - # up, or rare label drift between the two sources. - - record: cluster:gpu_count:free - expr: clamp_min(cluster:gpu_count:total - (cluster:gpu_count:allocated or vector(0)), 0) - # Cluster-layer filter asymmetry — intentional, not a bug: - # - # * cluster:gpu_count:total and cluster:gpu_power_watts:sum do NOT - # filter cozy-*/kube-* namespaces because they describe the - # physical hardware fleet. A GPU draws power and occupies a - # slot regardless of which namespace's pod happens to hold it; - # excluding infra pods would under-report raw capacity and - # break capacity-planning math. - # - # No namespace filter — DCGM exporter metrics carry the exporter's - # own namespace, not the workload namespace. Filtering by namespace - # here would silently drop all series. - - record: cluster:gpu_util:avg - expr: avg(DCGM_FI_DEV_GPU_UTIL) - - record: cluster:gpu_power_watts:sum - expr: sum(DCGM_FI_DEV_POWER_USAGE) - - - name: gpu.recording.node.1m - interval: 1m - params: {} - rules: - # Kube-requested GPU count per namespace — billable/allocation view. - # This is the only namespace-level rule that works correctly because - # kube_pod_container_resource_requests carries the real workload - # namespace, unlike DCGM exporter metrics which carry the exporter's - # own namespace (cozy-gpu-operator). - - record: namespace:gpu_count:allocated - expr: | - sum by (namespace) ( - kube_pod_container_resource_requests{resource="nvidia_com_gpu", namespace!="", namespace!~"cozy-.*|kube-.*"} - * on (namespace, pod) group_left() max by (namespace, pod) ( - kube_pod_status_phase{phase=~"Pending|Running"} == 1 - ) - ) - # Hardware metrics aggregated per node (Hostname). DCGM exporter - # metrics carry a capital-H "Hostname" label identifying the node - # but no usable workload namespace — so hardware telemetry is - # aggregated at node/GPU granularity, not namespace. - - record: node:gpu_util:avg - expr: avg by (Hostname) (DCGM_FI_DEV_GPU_UTIL) - - record: node:tensor_active:avg - expr: avg by (Hostname) (DCGM_FI_PROF_PIPE_TENSOR_ACTIVE) - - record: node:fb_used_bytes:sum - expr: sum by (Hostname) (DCGM_FI_DEV_FB_USED) * 1048576 - - record: node:power_watts:sum - expr: sum by (Hostname) (DCGM_FI_DEV_POWER_USAGE) - - # Known gap: there is no lower-bound sanity alert for under-reporting. - # If DCGM ever switches POWER/THERMAL_VIOLATION counter units the other - # direction — e.g. from the current nanoseconds back to microseconds, - # making rate()/1e9 produce values around 0.001 instead of real - # fractions — the throttle signal would silently collapse to near-zero - # instead of plateauing at 1.0. GPUThrottleFractionOverOne below catches - # the >1 drift; the <<1 drift would require correlation with independent - # signals (power draw vs TDP, clock dips) that we do not yet wire up. - # Documented here so future maintainers know this is a known gap, not - # an oversight. - - name: gpu.recording.efficiency.1m - interval: 1m - params: {} - rules: - # Tensor hardware saturation per GPU — the honest "am I using the GPU" - # metric for AI/LLM workloads. Aggregated at GPU level (Hostname + - # gpu + UUID) because DCGM metrics don't carry workload namespace. - # Stored as a 0..1 ratio; consumers multiply by 100 at display time. - - record: gpu:tensor_saturation:avg5m - expr: | - avg by (Hostname, gpu, UUID) ( - avg_over_time(DCGM_FI_PROF_PIPE_TENSOR_ACTIVE[5m]) - ) - - # Power efficiency per GPU — utilization per watt, reveals - # unoptimized workloads. Explicit on(...) pins the matching set - # to GPU-identifying labels. - - record: gpu:util_per_watt:avg5m - expr: | - max by (Hostname, gpu, UUID) ( - avg_over_time(DCGM_FI_DEV_GPU_UTIL[5m]) - ) - / on (Hostname, gpu, UUID) - clamp_min( - max by (Hostname, gpu, UUID) ( - avg_over_time(DCGM_FI_DEV_POWER_USAGE[5m]) - ), - 1 - ) - - # Fraction of time power-throttled (TDP cap) — 1.0 = fully throttled. - # DCGM_FI_DEV_*_VIOLATION is documented as µs but on A10/DCGM 3.x the - # counter grows in nanoseconds in practice — divide by 1e9 to get a - # 0..1 fraction (verified empirically when /1e6 yielded >100× reality). - # clamp_max protects against rate() artefacts at counter resets. - # - # max by (Hostname, gpu, UUID) collapses duplicate series from - # dcgm-exporter's pod-mapping labels (one physical GPU may appear - # under multiple pod/container labels during restarts or under - # MIG/MPS). Throttling is a physical-GPU property; taking max - # keeps consumer avg(...) honest — a naive avg() would dilute the - # signal by the number of pods sharing the GPU. - - record: gpu:power_throttle_fraction:rate5m - expr: clamp_max(max by (Hostname, gpu, UUID) (rate(DCGM_FI_DEV_POWER_VIOLATION[5m]) / 1e9), 1) - - # Fraction of time thermal-throttled. - - record: gpu:thermal_throttle_fraction:rate5m - expr: clamp_max(max by (Hostname, gpu, UUID) (rate(DCGM_FI_DEV_THERMAL_VIOLATION[5m]) / 1e9), 1) - - # Regression watch — the /1e9 divisor on *_VIOLATION was derived - # empirically against DCGM 3.x on A10. If a future DCGM version - # restores the documented µs units, pre-clamp values would exceed 1.0 - # while clamp_max(..., 1) silently masks the drift — recorded - # throttle fractions would plateau at 1.0 and consumers would think - # every GPU is always throttled. This alert fires on that condition - # so we can rescale to /1e6 before dashboards start lying. - - name: gpu.recording.throttle.validation.5m - interval: 5m - params: {} - rules: - - alert: GPUThrottleFractionOverOne - expr: | - max by (Hostname, gpu, UUID) (rate(DCGM_FI_DEV_POWER_VIOLATION[5m]) / 1e9) > 1 - or - max by (Hostname, gpu, UUID) (rate(DCGM_FI_DEV_THERMAL_VIOLATION[5m]) / 1e9) > 1 - for: 15m - labels: - severity: warning - component: gpu-monitoring - annotations: - summary: "DCGM throttle fraction exceeds 1.0 pre-clamp on {{ $labels.Hostname }}/{{ $labels.gpu }}" - description: | - gpu:{power,thermal}_throttle_fraction:rate5m clamps to 1.0, but the raw - rate(DCGM_FI_DEV_*_VIOLATION)/1e9 value is already >1 on this GPU — the - empirical nanosecond assumption is broken. Likely the DCGM exporter - version changed and the counter now grows in its documented µs units. - Re-verify on the exporter Pod and adjust the divisor (/1e9 → /1e6) in - gpu.recording.efficiency.1m before dashboards plateau at 100% throttle. - verified_on: "NVIDIA A10, DCGM 3.x — other GPU families (H100, L40S) may require a different divisor" - runbook: "If this alert fires, DCGM may have changed POWER_VIOLATION/THERMAL_VIOLATION counter units. Recalibrate by comparing raw counter rate against known throttle events; see gpu-recording.rules.yaml comments." diff --git a/packages/system/monitoring/dashboards-infra.list b/packages/system/monitoring/dashboards-infra.list index 4c7e494d..b5b9f52a 100644 --- a/packages/system/monitoring/dashboards-infra.list +++ b/packages/system/monitoring/dashboards-infra.list @@ -32,8 +32,3 @@ hubble/overview hubble/dns-namespace hubble/l7-http-metrics hubble/network-overview -gpu/gpu-performance -gpu/gpu-efficiency -gpu/gpu-quotas -gpu/gpu-fleet -gpu/gpu-tenants diff --git a/packages/system/monitoring/templates/alerta/alerta-db.yaml b/packages/system/monitoring/templates/alerta/alerta-db.yaml index af9dc72c..71df5428 100644 --- a/packages/system/monitoring/templates/alerta/alerta-db.yaml +++ b/packages/system/monitoring/templates/alerta/alerta-db.yaml @@ -5,9 +5,7 @@ metadata: name: alerta-db spec: instances: 2 - {{- $existingCluster := lookup "postgresql.cnpg.io/v1" "Cluster" .Release.Namespace "alerta-db" }} - {{- $image := dig "spec" "imageName" "ghcr.io/cloudnative-pg/postgresql:17.7-standard-trixie" $existingCluster }} - imageName: {{ if regexMatch ":17\\." $image }}ghcr.io/cloudnative-pg/postgresql:17.7-standard-trixie{{ else if regexMatch ":[0-9]+\\.[0-9]+$" $image }}{{ printf "%s-standard-trixie" $image }}{{ else }}{{ $image }}{{ end }} + imageName: ghcr.io/cloudnative-pg/postgresql:17.7 {{- if .Values._cluster.scheduling }} {{- $rawConstraints := get .Values._cluster.scheduling "globalAppTopologySpreadConstraints" }} {{- if $rawConstraints }} diff --git a/packages/system/monitoring/templates/alerta/alerta.yaml b/packages/system/monitoring/templates/alerta/alerta.yaml index d30e698d..44046d1f 100644 --- a/packages/system/monitoring/templates/alerta/alerta.yaml +++ b/packages/system/monitoring/templates/alerta/alerta.yaml @@ -181,7 +181,7 @@ metadata: app: alerta annotations: {{- if eq $solver "http01" }} - acme.cert-manager.io/http01-ingress-ingressclassname: {{ $ingress }} + acme.cert-manager.io/http01-ingress-class: {{ $ingress }} {{- end }} cert-manager.io/cluster-issuer: {{ $clusterIssuer }} spec: diff --git a/packages/system/monitoring/templates/dashboards.yaml b/packages/system/monitoring/templates/dashboards.yaml index 0427bfa0..01f1c009 100644 --- a/packages/system/monitoring/templates/dashboards.yaml +++ b/packages/system/monitoring/templates/dashboards.yaml @@ -14,7 +14,7 @@ spec: url: http://grafana-dashboards.cozy-grafana-operator.svc/{{ . }}.json {{- end }} {{- end }} -{{- if or (eq .Release.Namespace "tenant-root") (eq .Release.Namespace "cozy-monitoring") }} +{{- if eq .Release.Namespace "tenant-root" }} {{- range (split "\n" (.Files.Get "dashboards-infra.list")) }} {{- $parts := split "/" . }} {{- if eq (len $parts) 2 }} diff --git a/packages/system/monitoring/templates/grafana/db.yaml b/packages/system/monitoring/templates/grafana/db.yaml index afc2d0d7..73d6502e 100644 --- a/packages/system/monitoring/templates/grafana/db.yaml +++ b/packages/system/monitoring/templates/grafana/db.yaml @@ -4,9 +4,7 @@ metadata: name: grafana-db spec: instances: 2 - {{- $existingCluster := lookup "postgresql.cnpg.io/v1" "Cluster" .Release.Namespace "grafana-db" }} - {{- $image := dig "spec" "imageName" "ghcr.io/cloudnative-pg/postgresql:17.7-standard-trixie" $existingCluster }} - imageName: {{ if regexMatch ":17\\." $image }}ghcr.io/cloudnative-pg/postgresql:17.7-standard-trixie{{ else if regexMatch ":[0-9]+\\.[0-9]+$" $image }}{{ printf "%s-standard-trixie" $image }}{{ else }}{{ $image }}{{ end }} + imageName: ghcr.io/cloudnative-pg/postgresql:17.7 storage: size: {{ .Values.grafana.db.size }} {{- if .Values._cluster.scheduling }} diff --git a/packages/system/monitoring/templates/grafana/grafana.yaml b/packages/system/monitoring/templates/grafana/grafana.yaml index d65a7dc4..d7f51e31 100644 --- a/packages/system/monitoring/templates/grafana/grafana.yaml +++ b/packages/system/monitoring/templates/grafana/grafana.yaml @@ -74,7 +74,7 @@ spec: metadata: annotations: {{- if eq $solver "http01" }} - acme.cert-manager.io/http01-ingress-ingressclassname: "{{ $ingress }}" + acme.cert-manager.io/http01-ingress-class: "{{ $ingress }}" {{- end }} cert-manager.io/cluster-issuer: {{ $clusterIssuer }} spec: diff --git a/packages/system/multus/templates/multus-daemonset-thick.yml b/packages/system/multus/templates/multus-daemonset-thick.yml index cad330ea..bedffee3 100644 --- a/packages/system/multus/templates/multus-daemonset-thick.yml +++ b/packages/system/multus/templates/multus-daemonset-thick.yml @@ -155,7 +155,7 @@ spec: serviceAccountName: multus containers: - name: kube-multus - image: ghcr.io/cozystack/cozystack/multus-cni:v1.3.0@sha256:6735ffc12e5e660951f5a42943b38c308f33774a55836e94c191001405b58ec0 + image: ghcr.io/cozystack/cozystack/multus-cni:v1.2.1@sha256:aaf2ed6a5db1ee5acc0cd4f4683ea33570ff922c211bef37d407d6a01427b566 command: [ "/usr/src/multus-cni/bin/multus-daemon" ] resources: requests: @@ -201,7 +201,7 @@ spec: fieldPath: spec.nodeName initContainers: - name: install-multus-binary - image: ghcr.io/cozystack/cozystack/multus-cni:v1.3.0@sha256:6735ffc12e5e660951f5a42943b38c308f33774a55836e94c191001405b58ec0 + image: ghcr.io/cozystack/cozystack/multus-cni:v1.2.1@sha256:aaf2ed6a5db1ee5acc0cd4f4683ea33570ff922c211bef37d407d6a01427b566 command: - "/usr/src/multus-cni/bin/install_multus" - "-d" diff --git a/packages/system/objectstorage-controller/values.yaml b/packages/system/objectstorage-controller/values.yaml index a3859194..1439e583 100644 --- a/packages/system/objectstorage-controller/values.yaml +++ b/packages/system/objectstorage-controller/values.yaml @@ -1,3 +1,3 @@ objectstorage: controller: - image: "ghcr.io/cozystack/cozystack/objectstorage-controller:v1.3.0@sha256:7a9e4bf9c3f95f364756396815bb51b6c0f58f85db653460574d94b96cd3c7d5" + image: "ghcr.io/cozystack/cozystack/objectstorage-controller:v1.2.1@sha256:036be10f26c4871c63fb63c6a294476289aa063610e7fbbc0dbd21d3cd7bfe5e" diff --git a/packages/system/postgres-operator/Makefile b/packages/system/postgres-operator/Makefile index 5279f8a7..f58f235f 100644 --- a/packages/system/postgres-operator/Makefile +++ b/packages/system/postgres-operator/Makefile @@ -3,12 +3,9 @@ export NAMESPACE=cozy-$(NAME) include ../../../hack/package.mk -test: - helm unittest . - update: rm -rf charts helm repo add cnpg https://cloudnative-pg.github.io/charts helm repo update cnpg - helm pull cnpg/cloudnative-pg --untar --untardir charts --version 0.26.1 + helm pull cnpg/cloudnative-pg --untar --untardir charts rm -rf charts/cloudnative-pg/charts diff --git a/packages/system/postgres-operator/charts/cloudnative-pg/Chart.yaml b/packages/system/postgres-operator/charts/cloudnative-pg/Chart.yaml index 31e6afd0..191ae9c9 100644 --- a/packages/system/postgres-operator/charts/cloudnative-pg/Chart.yaml +++ b/packages/system/postgres-operator/charts/cloudnative-pg/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -appVersion: 1.27.1 +appVersion: 1.25.0 dependencies: - alias: monitoring condition: monitoring.grafanaDashboard.create @@ -22,4 +22,4 @@ name: cloudnative-pg sources: - https://github.com/cloudnative-pg/charts type: application -version: 0.26.1 +version: 0.23.0 diff --git a/packages/system/postgres-operator/charts/cloudnative-pg/README.md b/packages/system/postgres-operator/charts/cloudnative-pg/README.md index 10a69657..9ac1378d 100644 --- a/packages/system/postgres-operator/charts/cloudnative-pg/README.md +++ b/packages/system/postgres-operator/charts/cloudnative-pg/README.md @@ -1,6 +1,6 @@ # cloudnative-pg -![Version: 0.26.1](https://img.shields.io/badge/Version-0.26.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.27.1](https://img.shields.io/badge/AppVersion-1.27.1-informational?style=flat-square) +![Version: 0.23.0](https://img.shields.io/badge/Version-0.23.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.25.0](https://img.shields.io/badge/AppVersion-1.25.0-informational?style=flat-square) CloudNativePG Operator Helm Chart @@ -26,7 +26,7 @@ CloudNativePG Operator Helm Chart | Key | Type | Default | Description | |-----|------|---------|-------------| -| additionalArgs | list | `[]` | Additional arguments to be added to the operator's args list. | +| additionalArgs | list | `[]` | Additinal arguments to be added to the operator's args list. | | additionalEnv | list | `[]` | Array containing extra environment variables which can be templated. For example: - name: RELEASE_NAME value: "{{ .Release.Name }}" - name: MY_VAR value: "mySpecialKey" | | affinity | object | `{}` | Affinity for the operator to be installed. | | commonAnnotations | object | `{}` | Annotations to be added to all other resources. | @@ -57,7 +57,7 @@ CloudNativePG Operator Helm Chart | monitoring.podMonitorMetricRelabelings | list | `[]` | Metrics relabel configurations to apply to samples before ingestion. | | monitoring.podMonitorRelabelings | list | `[]` | Relabel configurations to apply to samples before scraping. | | monitoringQueriesConfigMap.name | string | `"cnpg-default-monitoring"` | The name of the default monitoring configmap. | -| monitoringQueriesConfigMap.queries | string | `"backends:\n query: |\n SELECT sa.datname\n , sa.usename\n , sa.application_name\n , states.state\n , COALESCE(sa.count, 0) AS total\n , COALESCE(sa.max_tx_secs, 0) AS max_tx_duration_seconds\n FROM ( VALUES ('active')\n , ('idle')\n , ('idle in transaction')\n , ('idle in transaction (aborted)')\n , ('fastpath function call')\n , ('disabled')\n ) AS states(state)\n LEFT JOIN (\n SELECT datname\n , state\n , usename\n , COALESCE(application_name, '') AS application_name\n , COUNT(*)\n , COALESCE(EXTRACT (EPOCH FROM (max(now() - xact_start))), 0) AS max_tx_secs\n FROM pg_catalog.pg_stat_activity\n GROUP BY datname, state, usename, application_name\n ) sa ON states.state = sa.state\n WHERE sa.usename IS NOT NULL\n metrics:\n - datname:\n usage: \"LABEL\"\n description: \"Name of the database\"\n - usename:\n usage: \"LABEL\"\n description: \"Name of the user\"\n - application_name:\n usage: \"LABEL\"\n description: \"Name of the application\"\n - state:\n usage: \"LABEL\"\n description: \"State of the backend\"\n - total:\n usage: \"GAUGE\"\n description: \"Number of backends\"\n - max_tx_duration_seconds:\n usage: \"GAUGE\"\n description: \"Maximum duration of a transaction in seconds\"\n\nbackends_waiting:\n query: |\n SELECT count(*) AS total\n FROM pg_catalog.pg_locks blocked_locks\n JOIN pg_catalog.pg_locks blocking_locks\n ON blocking_locks.locktype = blocked_locks.locktype\n AND blocking_locks.database IS NOT DISTINCT FROM blocked_locks.database\n AND blocking_locks.relation IS NOT DISTINCT FROM blocked_locks.relation\n AND blocking_locks.page IS NOT DISTINCT FROM blocked_locks.page\n AND blocking_locks.tuple IS NOT DISTINCT FROM blocked_locks.tuple\n AND blocking_locks.virtualxid IS NOT DISTINCT FROM blocked_locks.virtualxid\n AND blocking_locks.transactionid IS NOT DISTINCT FROM blocked_locks.transactionid\n AND blocking_locks.classid IS NOT DISTINCT FROM blocked_locks.classid\n AND blocking_locks.objid IS NOT DISTINCT FROM blocked_locks.objid\n AND blocking_locks.objsubid IS NOT DISTINCT FROM blocked_locks.objsubid\n AND blocking_locks.pid != blocked_locks.pid\n JOIN pg_catalog.pg_stat_activity blocking_activity ON blocking_activity.pid = blocking_locks.pid\n WHERE NOT blocked_locks.granted\n metrics:\n - total:\n usage: \"GAUGE\"\n description: \"Total number of backends that are currently waiting on other queries\"\n\npg_database:\n query: |\n SELECT datname\n , pg_catalog.pg_database_size(datname) AS size_bytes\n , pg_catalog.age(datfrozenxid) AS xid_age\n , pg_catalog.mxid_age(datminmxid) AS mxid_age\n FROM pg_catalog.pg_database\n WHERE datallowconn\n metrics:\n - datname:\n usage: \"LABEL\"\n description: \"Name of the database\"\n - size_bytes:\n usage: \"GAUGE\"\n description: \"Disk space used by the database\"\n - xid_age:\n usage: \"GAUGE\"\n description: \"Number of transactions from the frozen XID to the current one\"\n - mxid_age:\n usage: \"GAUGE\"\n description: \"Number of multiple transactions (Multixact) from the frozen XID to the current one\"\n\npg_postmaster:\n query: |\n SELECT EXTRACT(EPOCH FROM pg_postmaster_start_time) AS start_time\n FROM pg_catalog.pg_postmaster_start_time()\n metrics:\n - start_time:\n usage: \"GAUGE\"\n description: \"Time at which postgres started (based on epoch)\"\n\npg_replication:\n query: \"SELECT CASE WHEN (\n NOT pg_catalog.pg_is_in_recovery()\n OR pg_catalog.pg_last_wal_receive_lsn() = pg_catalog.pg_last_wal_replay_lsn())\n THEN 0\n ELSE GREATEST (0,\n EXTRACT(EPOCH FROM (now() - pg_catalog.pg_last_xact_replay_timestamp())))\n END AS lag,\n pg_catalog.pg_is_in_recovery() AS in_recovery,\n EXISTS (TABLE pg_stat_wal_receiver) AS is_wal_receiver_up,\n (SELECT count(*) FROM pg_catalog.pg_stat_replication) AS streaming_replicas\"\n metrics:\n - lag:\n usage: \"GAUGE\"\n description: \"Replication lag behind primary in seconds\"\n - in_recovery:\n usage: \"GAUGE\"\n description: \"Whether the instance is in recovery\"\n - is_wal_receiver_up:\n usage: \"GAUGE\"\n description: \"Whether the instance wal_receiver is up\"\n - streaming_replicas:\n usage: \"GAUGE\"\n description: \"Number of streaming replicas connected to the instance\"\n\npg_replication_slots:\n query: |\n SELECT slot_name,\n slot_type,\n database,\n active,\n (CASE pg_catalog.pg_is_in_recovery()\n WHEN TRUE THEN pg_catalog.pg_wal_lsn_diff(pg_catalog.pg_last_wal_receive_lsn(), restart_lsn)\n ELSE pg_catalog.pg_wal_lsn_diff(pg_catalog.pg_current_wal_lsn(), restart_lsn)\n END) as pg_wal_lsn_diff\n FROM pg_catalog.pg_replication_slots\n WHERE NOT temporary\n metrics:\n - slot_name:\n usage: \"LABEL\"\n description: \"Name of the replication slot\"\n - slot_type:\n usage: \"LABEL\"\n description: \"Type of the replication slot\"\n - database:\n usage: \"LABEL\"\n description: \"Name of the database\"\n - active:\n usage: \"GAUGE\"\n description: \"Flag indicating whether the slot is active\"\n - pg_wal_lsn_diff:\n usage: \"GAUGE\"\n description: \"Replication lag in bytes\"\n\npg_stat_archiver:\n query: |\n SELECT archived_count\n , failed_count\n , COALESCE(EXTRACT(EPOCH FROM (now() - last_archived_time)), -1) AS seconds_since_last_archival\n , COALESCE(EXTRACT(EPOCH FROM (now() - last_failed_time)), -1) AS seconds_since_last_failure\n , COALESCE(EXTRACT(EPOCH FROM last_archived_time), -1) AS last_archived_time\n , COALESCE(EXTRACT(EPOCH FROM last_failed_time), -1) AS last_failed_time\n , COALESCE(CAST(CAST('x'||pg_catalog.right(pg_catalog.split_part(last_archived_wal, '.', 1), 16) AS pg_catalog.bit(64)) AS pg_catalog.int8), -1) AS last_archived_wal_start_lsn\n , COALESCE(CAST(CAST('x'||pg_catalog.right(pg_catalog.split_part(last_failed_wal, '.', 1), 16) AS pg_catalog.bit(64)) AS pg_catalog.int8), -1) AS last_failed_wal_start_lsn\n , EXTRACT(EPOCH FROM stats_reset) AS stats_reset_time\n FROM pg_catalog.pg_stat_archiver\n metrics:\n - archived_count:\n usage: \"COUNTER\"\n description: \"Number of WAL files that have been successfully archived\"\n - failed_count:\n usage: \"COUNTER\"\n description: \"Number of failed attempts for archiving WAL files\"\n - seconds_since_last_archival:\n usage: \"GAUGE\"\n description: \"Seconds since the last successful archival operation\"\n - seconds_since_last_failure:\n usage: \"GAUGE\"\n description: \"Seconds since the last failed archival operation\"\n - last_archived_time:\n usage: \"GAUGE\"\n description: \"Epoch of the last time WAL archiving succeeded\"\n - last_failed_time:\n usage: \"GAUGE\"\n description: \"Epoch of the last time WAL archiving failed\"\n - last_archived_wal_start_lsn:\n usage: \"GAUGE\"\n description: \"Archived WAL start LSN\"\n - last_failed_wal_start_lsn:\n usage: \"GAUGE\"\n description: \"Last failed WAL LSN\"\n - stats_reset_time:\n usage: \"GAUGE\"\n description: \"Time at which these statistics were last reset\"\n\npg_stat_bgwriter:\n runonserver: \"<17.0.0\"\n query: |\n SELECT checkpoints_timed\n , checkpoints_req\n , checkpoint_write_time\n , checkpoint_sync_time\n , buffers_checkpoint\n , buffers_clean\n , maxwritten_clean\n , buffers_backend\n , buffers_backend_fsync\n , buffers_alloc\n FROM pg_catalog.pg_stat_bgwriter\n metrics:\n - checkpoints_timed:\n usage: \"COUNTER\"\n description: \"Number of scheduled checkpoints that have been performed\"\n - checkpoints_req:\n usage: \"COUNTER\"\n description: \"Number of requested checkpoints that have been performed\"\n - checkpoint_write_time:\n usage: \"COUNTER\"\n description: \"Total amount of time that has been spent in the portion of checkpoint processing where files are written to disk, in milliseconds\"\n - checkpoint_sync_time:\n usage: \"COUNTER\"\n description: \"Total amount of time that has been spent in the portion of checkpoint processing where files are synchronized to disk, in milliseconds\"\n - buffers_checkpoint:\n usage: \"COUNTER\"\n description: \"Number of buffers written during checkpoints\"\n - buffers_clean:\n usage: \"COUNTER\"\n description: \"Number of buffers written by the background writer\"\n - maxwritten_clean:\n usage: \"COUNTER\"\n description: \"Number of times the background writer stopped a cleaning scan because it had written too many buffers\"\n - buffers_backend:\n usage: \"COUNTER\"\n description: \"Number of buffers written directly by a backend\"\n - buffers_backend_fsync:\n usage: \"COUNTER\"\n description: \"Number of times a backend had to execute its own fsync call (normally the background writer handles those even when the backend does its own write)\"\n - buffers_alloc:\n usage: \"COUNTER\"\n description: \"Number of buffers allocated\"\n\npg_stat_bgwriter_17:\n runonserver: \">=17.0.0\"\n name: pg_stat_bgwriter\n query: |\n SELECT buffers_clean\n , maxwritten_clean\n , buffers_alloc\n , EXTRACT(EPOCH FROM stats_reset) AS stats_reset_time\n FROM pg_catalog.pg_stat_bgwriter\n metrics:\n - buffers_clean:\n usage: \"COUNTER\"\n description: \"Number of buffers written by the background writer\"\n - maxwritten_clean:\n usage: \"COUNTER\"\n description: \"Number of times the background writer stopped a cleaning scan because it had written too many buffers\"\n - buffers_alloc:\n usage: \"COUNTER\"\n description: \"Number of buffers allocated\"\n - stats_reset_time:\n usage: \"GAUGE\"\n description: \"Time at which these statistics were last reset\"\n\npg_stat_checkpointer:\n runonserver: \">=17.0.0\"\n query: |\n SELECT num_timed AS checkpoints_timed\n , num_requested AS checkpoints_req\n , restartpoints_timed\n , restartpoints_req\n , restartpoints_done\n , write_time\n , sync_time\n , buffers_written\n , EXTRACT(EPOCH FROM stats_reset) AS stats_reset_time\n FROM pg_catalog.pg_stat_checkpointer\n metrics:\n - checkpoints_timed:\n usage: \"COUNTER\"\n description: \"Number of scheduled checkpoints that have been performed\"\n - checkpoints_req:\n usage: \"COUNTER\"\n description: \"Number of requested checkpoints that have been performed\"\n - restartpoints_timed:\n usage: \"COUNTER\"\n description: \"Number of scheduled restartpoints due to timeout or after a failed attempt to perform it\"\n - restartpoints_req:\n usage: \"COUNTER\"\n description: \"Number of requested restartpoints that have been performed\"\n - restartpoints_done:\n usage: \"COUNTER\"\n description: \"Number of restartpoints that have been performed\"\n - write_time:\n usage: \"COUNTER\"\n description: \"Total amount of time that has been spent in the portion of processing checkpoints and restartpoints where files are written to disk, in milliseconds\"\n - sync_time:\n usage: \"COUNTER\"\n description: \"Total amount of time that has been spent in the portion of processing checkpoints and restartpoints where files are synchronized to disk, in milliseconds\"\n - buffers_written:\n usage: \"COUNTER\"\n description: \"Number of buffers written during checkpoints and restartpoints\"\n - stats_reset_time:\n usage: \"GAUGE\"\n description: \"Time at which these statistics were last reset\"\n\npg_stat_database:\n query: |\n SELECT datname\n , xact_commit\n , xact_rollback\n , blks_read\n , blks_hit\n , tup_returned\n , tup_fetched\n , tup_inserted\n , tup_updated\n , tup_deleted\n , conflicts\n , temp_files\n , temp_bytes\n , deadlocks\n , blk_read_time\n , blk_write_time\n FROM pg_catalog.pg_stat_database\n metrics:\n - datname:\n usage: \"LABEL\"\n description: \"Name of this database\"\n - xact_commit:\n usage: \"COUNTER\"\n description: \"Number of transactions in this database that have been committed\"\n - xact_rollback:\n usage: \"COUNTER\"\n description: \"Number of transactions in this database that have been rolled back\"\n - blks_read:\n usage: \"COUNTER\"\n description: \"Number of disk blocks read in this database\"\n - blks_hit:\n usage: \"COUNTER\"\n description: \"Number of times disk blocks were found already in the buffer cache, so that a read was not necessary (this only includes hits in the PostgreSQL buffer cache, not the operating system's file system cache)\"\n - tup_returned:\n usage: \"COUNTER\"\n description: \"Number of rows returned by queries in this database\"\n - tup_fetched:\n usage: \"COUNTER\"\n description: \"Number of rows fetched by queries in this database\"\n - tup_inserted:\n usage: \"COUNTER\"\n description: \"Number of rows inserted by queries in this database\"\n - tup_updated:\n usage: \"COUNTER\"\n description: \"Number of rows updated by queries in this database\"\n - tup_deleted:\n usage: \"COUNTER\"\n description: \"Number of rows deleted by queries in this database\"\n - conflicts:\n usage: \"COUNTER\"\n description: \"Number of queries canceled due to conflicts with recovery in this database\"\n - temp_files:\n usage: \"COUNTER\"\n description: \"Number of temporary files created by queries in this database\"\n - temp_bytes:\n usage: \"COUNTER\"\n description: \"Total amount of data written to temporary files by queries in this database\"\n - deadlocks:\n usage: \"COUNTER\"\n description: \"Number of deadlocks detected in this database\"\n - blk_read_time:\n usage: \"COUNTER\"\n description: \"Time spent reading data file blocks by backends in this database, in milliseconds\"\n - blk_write_time:\n usage: \"COUNTER\"\n description: \"Time spent writing data file blocks by backends in this database, in milliseconds\"\n\npg_stat_replication:\n primary: true\n query: |\n SELECT usename\n , COALESCE(application_name, '') AS application_name\n , COALESCE(client_addr::text, '') AS client_addr\n , COALESCE(client_port::text, '') AS client_port\n , EXTRACT(EPOCH FROM backend_start) AS backend_start\n , COALESCE(pg_catalog.age(backend_xmin), 0) AS backend_xmin_age\n , pg_catalog.pg_wal_lsn_diff(pg_catalog.pg_current_wal_lsn(), sent_lsn) AS sent_diff_bytes\n , pg_catalog.pg_wal_lsn_diff(pg_catalog.pg_current_wal_lsn(), write_lsn) AS write_diff_bytes\n , pg_catalog.pg_wal_lsn_diff(pg_catalog.pg_current_wal_lsn(), flush_lsn) AS flush_diff_bytes\n , COALESCE(pg_catalog.pg_wal_lsn_diff(pg_catalog.pg_current_wal_lsn(), replay_lsn),0) AS replay_diff_bytes\n , COALESCE((EXTRACT(EPOCH FROM write_lag)),0)::float AS write_lag_seconds\n , COALESCE((EXTRACT(EPOCH FROM flush_lag)),0)::float AS flush_lag_seconds\n , COALESCE((EXTRACT(EPOCH FROM replay_lag)),0)::float AS replay_lag_seconds\n FROM pg_catalog.pg_stat_replication\n metrics:\n - usename:\n usage: \"LABEL\"\n description: \"Name of the replication user\"\n - application_name:\n usage: \"LABEL\"\n description: \"Name of the application\"\n - client_addr:\n usage: \"LABEL\"\n description: \"Client IP address\"\n - client_port:\n usage: \"LABEL\"\n description: \"Client TCP port\"\n - backend_start:\n usage: \"COUNTER\"\n description: \"Time when this process was started\"\n - backend_xmin_age:\n usage: \"COUNTER\"\n description: \"The age of this standby's xmin horizon\"\n - sent_diff_bytes:\n usage: \"GAUGE\"\n description: \"Difference in bytes from the last write-ahead log location sent on this connection\"\n - write_diff_bytes:\n usage: \"GAUGE\"\n description: \"Difference in bytes from the last write-ahead log location written to disk by this standby server\"\n - flush_diff_bytes:\n usage: \"GAUGE\"\n description: \"Difference in bytes from the last write-ahead log location flushed to disk by this standby server\"\n - replay_diff_bytes:\n usage: \"GAUGE\"\n description: \"Difference in bytes from the last write-ahead log location replayed into the database on this standby server\"\n - write_lag_seconds:\n usage: \"GAUGE\"\n description: \"Time elapsed between flushing recent WAL locally and receiving notification that this standby server has written it\"\n - flush_lag_seconds:\n usage: \"GAUGE\"\n description: \"Time elapsed between flushing recent WAL locally and receiving notification that this standby server has written and flushed it\"\n - replay_lag_seconds:\n usage: \"GAUGE\"\n description: \"Time elapsed between flushing recent WAL locally and receiving notification that this standby server has written, flushed and applied it\"\n\npg_settings:\n query: |\n SELECT name,\n CASE setting WHEN 'on' THEN '1' WHEN 'off' THEN '0' ELSE setting END AS setting\n FROM pg_catalog.pg_settings\n WHERE vartype IN ('integer', 'real', 'bool')\n ORDER BY 1\n metrics:\n - name:\n usage: \"LABEL\"\n description: \"Name of the setting\"\n - setting:\n usage: \"GAUGE\"\n description: \"Setting value\"\n\npg_extensions:\n query: |\n SELECT\n current_database() as datname,\n name as extname,\n default_version,\n installed_version,\n CASE\n WHEN default_version = installed_version THEN 0\n ELSE 1\n END AS update_available\n FROM pg_catalog.pg_available_extensions\n WHERE installed_version IS NOT NULL\n metrics:\n - datname:\n usage: \"LABEL\"\n description: \"Name of the database\"\n - extname:\n usage: \"LABEL\"\n description: \"Extension name\"\n - default_version:\n usage: \"LABEL\"\n description: \"Default version\"\n - installed_version:\n usage: \"LABEL\"\n description: \"Installed version\"\n - update_available:\n usage: \"GAUGE\"\n description: \"An update is available\"\n target_databases:\n - '*'\n"` | A string representation of a YAML defining monitoring queries. | +| monitoringQueriesConfigMap.queries | string | `"backends:\n query: |\n SELECT sa.datname\n , sa.usename\n , sa.application_name\n , states.state\n , COALESCE(sa.count, 0) AS total\n , COALESCE(sa.max_tx_secs, 0) AS max_tx_duration_seconds\n FROM ( VALUES ('active')\n , ('idle')\n , ('idle in transaction')\n , ('idle in transaction (aborted)')\n , ('fastpath function call')\n , ('disabled')\n ) AS states(state)\n LEFT JOIN (\n SELECT datname\n , state\n , usename\n , COALESCE(application_name, '') AS application_name\n , COUNT(*)\n , COALESCE(EXTRACT (EPOCH FROM (max(now() - xact_start))), 0) AS max_tx_secs\n FROM pg_catalog.pg_stat_activity\n GROUP BY datname, state, usename, application_name\n ) sa ON states.state = sa.state\n WHERE sa.usename IS NOT NULL\n metrics:\n - datname:\n usage: \"LABEL\"\n description: \"Name of the database\"\n - usename:\n usage: \"LABEL\"\n description: \"Name of the user\"\n - application_name:\n usage: \"LABEL\"\n description: \"Name of the application\"\n - state:\n usage: \"LABEL\"\n description: \"State of the backend\"\n - total:\n usage: \"GAUGE\"\n description: \"Number of backends\"\n - max_tx_duration_seconds:\n usage: \"GAUGE\"\n description: \"Maximum duration of a transaction in seconds\"\n\nbackends_waiting:\n query: |\n SELECT count(*) AS total\n FROM pg_catalog.pg_locks blocked_locks\n JOIN pg_catalog.pg_locks blocking_locks\n ON blocking_locks.locktype = blocked_locks.locktype\n AND blocking_locks.database IS NOT DISTINCT FROM blocked_locks.database\n AND blocking_locks.relation IS NOT DISTINCT FROM blocked_locks.relation\n AND blocking_locks.page IS NOT DISTINCT FROM blocked_locks.page\n AND blocking_locks.tuple IS NOT DISTINCT FROM blocked_locks.tuple\n AND blocking_locks.virtualxid IS NOT DISTINCT FROM blocked_locks.virtualxid\n AND blocking_locks.transactionid IS NOT DISTINCT FROM blocked_locks.transactionid\n AND blocking_locks.classid IS NOT DISTINCT FROM blocked_locks.classid\n AND blocking_locks.objid IS NOT DISTINCT FROM blocked_locks.objid\n AND blocking_locks.objsubid IS NOT DISTINCT FROM blocked_locks.objsubid\n AND blocking_locks.pid != blocked_locks.pid\n JOIN pg_catalog.pg_stat_activity blocking_activity ON blocking_activity.pid = blocking_locks.pid\n WHERE NOT blocked_locks.granted\n metrics:\n - total:\n usage: \"GAUGE\"\n description: \"Total number of backends that are currently waiting on other queries\"\n\npg_database:\n query: |\n SELECT datname\n , pg_catalog.pg_database_size(datname) AS size_bytes\n , pg_catalog.age(datfrozenxid) AS xid_age\n , pg_catalog.mxid_age(datminmxid) AS mxid_age\n FROM pg_catalog.pg_database\n WHERE datallowconn\n metrics:\n - datname:\n usage: \"LABEL\"\n description: \"Name of the database\"\n - size_bytes:\n usage: \"GAUGE\"\n description: \"Disk space used by the database\"\n - xid_age:\n usage: \"GAUGE\"\n description: \"Number of transactions from the frozen XID to the current one\"\n - mxid_age:\n usage: \"GAUGE\"\n description: \"Number of multiple transactions (Multixact) from the frozen XID to the current one\"\n\npg_postmaster:\n query: |\n SELECT EXTRACT(EPOCH FROM pg_postmaster_start_time) AS start_time\n FROM pg_catalog.pg_postmaster_start_time()\n metrics:\n - start_time:\n usage: \"GAUGE\"\n description: \"Time at which postgres started (based on epoch)\"\n\npg_replication:\n query: \"SELECT CASE WHEN (\n NOT pg_catalog.pg_is_in_recovery()\n OR pg_catalog.pg_last_wal_receive_lsn() = pg_catalog.pg_last_wal_replay_lsn())\n THEN 0\n ELSE GREATEST (0,\n EXTRACT(EPOCH FROM (now() - pg_catalog.pg_last_xact_replay_timestamp())))\n END AS lag,\n pg_catalog.pg_is_in_recovery() AS in_recovery,\n EXISTS (TABLE pg_stat_wal_receiver) AS is_wal_receiver_up,\n (SELECT count(*) FROM pg_catalog.pg_stat_replication) AS streaming_replicas\"\n metrics:\n - lag:\n usage: \"GAUGE\"\n description: \"Replication lag behind primary in seconds\"\n - in_recovery:\n usage: \"GAUGE\"\n description: \"Whether the instance is in recovery\"\n - is_wal_receiver_up:\n usage: \"GAUGE\"\n description: \"Whether the instance wal_receiver is up\"\n - streaming_replicas:\n usage: \"GAUGE\"\n description: \"Number of streaming replicas connected to the instance\"\n\npg_replication_slots:\n query: |\n SELECT slot_name,\n slot_type,\n database,\n active,\n (CASE pg_catalog.pg_is_in_recovery()\n WHEN TRUE THEN pg_catalog.pg_wal_lsn_diff(pg_catalog.pg_last_wal_receive_lsn(), restart_lsn)\n ELSE pg_catalog.pg_wal_lsn_diff(pg_catalog.pg_current_wal_lsn(), restart_lsn)\n END) as pg_wal_lsn_diff\n FROM pg_catalog.pg_replication_slots\n WHERE NOT temporary\n metrics:\n - slot_name:\n usage: \"LABEL\"\n description: \"Name of the replication slot\"\n - slot_type:\n usage: \"LABEL\"\n description: \"Type of the replication slot\"\n - database:\n usage: \"LABEL\"\n description: \"Name of the database\"\n - active:\n usage: \"GAUGE\"\n description: \"Flag indicating whether the slot is active\"\n - pg_wal_lsn_diff:\n usage: \"GAUGE\"\n description: \"Replication lag in bytes\"\n\npg_stat_archiver:\n query: |\n SELECT archived_count\n , failed_count\n , COALESCE(EXTRACT(EPOCH FROM (now() - last_archived_time)), -1) AS seconds_since_last_archival\n , COALESCE(EXTRACT(EPOCH FROM (now() - last_failed_time)), -1) AS seconds_since_last_failure\n , COALESCE(EXTRACT(EPOCH FROM last_archived_time), -1) AS last_archived_time\n , COALESCE(EXTRACT(EPOCH FROM last_failed_time), -1) AS last_failed_time\n , COALESCE(CAST(CAST('x'||pg_catalog.right(pg_catalog.split_part(last_archived_wal, '.', 1), 16) AS pg_catalog.bit(64)) AS pg_catalog.int8), -1) AS last_archived_wal_start_lsn\n , COALESCE(CAST(CAST('x'||pg_catalog.right(pg_catalog.split_part(last_failed_wal, '.', 1), 16) AS pg_catalog.bit(64)) AS pg_catalog.int8), -1) AS last_failed_wal_start_lsn\n , EXTRACT(EPOCH FROM stats_reset) AS stats_reset_time\n FROM pg_catalog.pg_stat_archiver\n metrics:\n - archived_count:\n usage: \"COUNTER\"\n description: \"Number of WAL files that have been successfully archived\"\n - failed_count:\n usage: \"COUNTER\"\n description: \"Number of failed attempts for archiving WAL files\"\n - seconds_since_last_archival:\n usage: \"GAUGE\"\n description: \"Seconds since the last successful archival operation\"\n - seconds_since_last_failure:\n usage: \"GAUGE\"\n description: \"Seconds since the last failed archival operation\"\n - last_archived_time:\n usage: \"GAUGE\"\n description: \"Epoch of the last time WAL archiving succeeded\"\n - last_failed_time:\n usage: \"GAUGE\"\n description: \"Epoch of the last time WAL archiving failed\"\n - last_archived_wal_start_lsn:\n usage: \"GAUGE\"\n description: \"Archived WAL start LSN\"\n - last_failed_wal_start_lsn:\n usage: \"GAUGE\"\n description: \"Last failed WAL LSN\"\n - stats_reset_time:\n usage: \"GAUGE\"\n description: \"Time at which these statistics were last reset\"\n\npg_stat_bgwriter:\n runonserver: \"<17.0.0\"\n query: |\n SELECT checkpoints_timed\n , checkpoints_req\n , checkpoint_write_time\n , checkpoint_sync_time\n , buffers_checkpoint\n , buffers_clean\n , maxwritten_clean\n , buffers_backend\n , buffers_backend_fsync\n , buffers_alloc\n FROM pg_catalog.pg_stat_bgwriter\n metrics:\n - checkpoints_timed:\n usage: \"COUNTER\"\n description: \"Number of scheduled checkpoints that have been performed\"\n - checkpoints_req:\n usage: \"COUNTER\"\n description: \"Number of requested checkpoints that have been performed\"\n - checkpoint_write_time:\n usage: \"COUNTER\"\n description: \"Total amount of time that has been spent in the portion of checkpoint processing where files are written to disk, in milliseconds\"\n - checkpoint_sync_time:\n usage: \"COUNTER\"\n description: \"Total amount of time that has been spent in the portion of checkpoint processing where files are synchronized to disk, in milliseconds\"\n - buffers_checkpoint:\n usage: \"COUNTER\"\n description: \"Number of buffers written during checkpoints\"\n - buffers_clean:\n usage: \"COUNTER\"\n description: \"Number of buffers written by the background writer\"\n - maxwritten_clean:\n usage: \"COUNTER\"\n description: \"Number of times the background writer stopped a cleaning scan because it had written too many buffers\"\n - buffers_backend:\n usage: \"COUNTER\"\n description: \"Number of buffers written directly by a backend\"\n - buffers_backend_fsync:\n usage: \"COUNTER\"\n description: \"Number of times a backend had to execute its own fsync call (normally the background writer handles those even when the backend does its own write)\"\n - buffers_alloc:\n usage: \"COUNTER\"\n description: \"Number of buffers allocated\"\n\npg_stat_bgwriter_17:\n runonserver: \">=17.0.0\"\n name: pg_stat_bgwriter\n query: |\n SELECT buffers_clean\n , maxwritten_clean\n , buffers_alloc\n , EXTRACT(EPOCH FROM stats_reset) AS stats_reset_time\n FROM pg_catalog.pg_stat_bgwriter\n metrics:\n - buffers_clean:\n usage: \"COUNTER\"\n description: \"Number of buffers written by the background writer\"\n - maxwritten_clean:\n usage: \"COUNTER\"\n description: \"Number of times the background writer stopped a cleaning scan because it had written too many buffers\"\n - buffers_alloc:\n usage: \"COUNTER\"\n description: \"Number of buffers allocated\"\n - stats_reset_time:\n usage: \"GAUGE\"\n description: \"Time at which these statistics were last reset\"\n\npg_stat_checkpointer:\n runonserver: \">=17.0.0\"\n query: |\n SELECT num_timed AS checkpoints_timed\n , num_requested AS checkpoints_req\n , restartpoints_timed\n , restartpoints_req\n , restartpoints_done\n , write_time\n , sync_time\n , buffers_written\n , EXTRACT(EPOCH FROM stats_reset) AS stats_reset_time\n FROM pg_catalog.pg_stat_checkpointer\n metrics:\n - checkpoints_timed:\n usage: \"COUNTER\"\n description: \"Number of scheduled checkpoints that have been performed\"\n - checkpoints_req:\n usage: \"COUNTER\"\n description: \"Number of requested checkpoints that have been performed\"\n - restartpoints_timed:\n usage: \"COUNTER\"\n description: \"Number of scheduled restartpoints due to timeout or after a failed attempt to perform it\"\n - restartpoints_req:\n usage: \"COUNTER\"\n description: \"Number of requested restartpoints that have been performed\"\n - restartpoints_done:\n usage: \"COUNTER\"\n description: \"Number of restartpoints that have been performed\"\n - write_time:\n usage: \"COUNTER\"\n description: \"Total amount of time that has been spent in the portion of processing checkpoints and restartpoints where files are written to disk, in milliseconds\"\n - sync_time:\n usage: \"COUNTER\"\n description: \"Total amount of time that has been spent in the portion of processing checkpoints and restartpoints where files are synchronized to disk, in milliseconds\"\n - buffers_written:\n usage: \"COUNTER\"\n description: \"Number of buffers written during checkpoints and restartpoints\"\n - stats_reset_time:\n usage: \"GAUGE\"\n description: \"Time at which these statistics were last reset\"\n\npg_stat_database:\n query: |\n SELECT datname\n , xact_commit\n , xact_rollback\n , blks_read\n , blks_hit\n , tup_returned\n , tup_fetched\n , tup_inserted\n , tup_updated\n , tup_deleted\n , conflicts\n , temp_files\n , temp_bytes\n , deadlocks\n , blk_read_time\n , blk_write_time\n FROM pg_catalog.pg_stat_database\n metrics:\n - datname:\n usage: \"LABEL\"\n description: \"Name of this database\"\n - xact_commit:\n usage: \"COUNTER\"\n description: \"Number of transactions in this database that have been committed\"\n - xact_rollback:\n usage: \"COUNTER\"\n description: \"Number of transactions in this database that have been rolled back\"\n - blks_read:\n usage: \"COUNTER\"\n description: \"Number of disk blocks read in this database\"\n - blks_hit:\n usage: \"COUNTER\"\n description: \"Number of times disk blocks were found already in the buffer cache, so that a read was not necessary (this only includes hits in the PostgreSQL buffer cache, not the operating system's file system cache)\"\n - tup_returned:\n usage: \"COUNTER\"\n description: \"Number of rows returned by queries in this database\"\n - tup_fetched:\n usage: \"COUNTER\"\n description: \"Number of rows fetched by queries in this database\"\n - tup_inserted:\n usage: \"COUNTER\"\n description: \"Number of rows inserted by queries in this database\"\n - tup_updated:\n usage: \"COUNTER\"\n description: \"Number of rows updated by queries in this database\"\n - tup_deleted:\n usage: \"COUNTER\"\n description: \"Number of rows deleted by queries in this database\"\n - conflicts:\n usage: \"COUNTER\"\n description: \"Number of queries canceled due to conflicts with recovery in this database\"\n - temp_files:\n usage: \"COUNTER\"\n description: \"Number of temporary files created by queries in this database\"\n - temp_bytes:\n usage: \"COUNTER\"\n description: \"Total amount of data written to temporary files by queries in this database\"\n - deadlocks:\n usage: \"COUNTER\"\n description: \"Number of deadlocks detected in this database\"\n - blk_read_time:\n usage: \"COUNTER\"\n description: \"Time spent reading data file blocks by backends in this database, in milliseconds\"\n - blk_write_time:\n usage: \"COUNTER\"\n description: \"Time spent writing data file blocks by backends in this database, in milliseconds\"\n\npg_stat_replication:\n primary: true\n query: |\n SELECT usename\n , COALESCE(application_name, '') AS application_name\n , COALESCE(client_addr::text, '') AS client_addr\n , COALESCE(client_port::text, '') AS client_port\n , EXTRACT(EPOCH FROM backend_start) AS backend_start\n , COALESCE(pg_catalog.age(backend_xmin), 0) AS backend_xmin_age\n , pg_catalog.pg_wal_lsn_diff(pg_catalog.pg_current_wal_lsn(), sent_lsn) AS sent_diff_bytes\n , pg_catalog.pg_wal_lsn_diff(pg_catalog.pg_current_wal_lsn(), write_lsn) AS write_diff_bytes\n , pg_catalog.pg_wal_lsn_diff(pg_catalog.pg_current_wal_lsn(), flush_lsn) AS flush_diff_bytes\n , COALESCE(pg_catalog.pg_wal_lsn_diff(pg_catalog.pg_current_wal_lsn(), replay_lsn),0) AS replay_diff_bytes\n , COALESCE((EXTRACT(EPOCH FROM write_lag)),0)::float AS write_lag_seconds\n , COALESCE((EXTRACT(EPOCH FROM flush_lag)),0)::float AS flush_lag_seconds\n , COALESCE((EXTRACT(EPOCH FROM replay_lag)),0)::float AS replay_lag_seconds\n FROM pg_catalog.pg_stat_replication\n metrics:\n - usename:\n usage: \"LABEL\"\n description: \"Name of the replication user\"\n - application_name:\n usage: \"LABEL\"\n description: \"Name of the application\"\n - client_addr:\n usage: \"LABEL\"\n description: \"Client IP address\"\n - client_port:\n usage: \"LABEL\"\n description: \"Client TCP port\"\n - backend_start:\n usage: \"COUNTER\"\n description: \"Time when this process was started\"\n - backend_xmin_age:\n usage: \"COUNTER\"\n description: \"The age of this standby's xmin horizon\"\n - sent_diff_bytes:\n usage: \"GAUGE\"\n description: \"Difference in bytes from the last write-ahead log location sent on this connection\"\n - write_diff_bytes:\n usage: \"GAUGE\"\n description: \"Difference in bytes from the last write-ahead log location written to disk by this standby server\"\n - flush_diff_bytes:\n usage: \"GAUGE\"\n description: \"Difference in bytes from the last write-ahead log location flushed to disk by this standby server\"\n - replay_diff_bytes:\n usage: \"GAUGE\"\n description: \"Difference in bytes from the last write-ahead log location replayed into the database on this standby server\"\n - write_lag_seconds:\n usage: \"GAUGE\"\n description: \"Time elapsed between flushing recent WAL locally and receiving notification that this standby server has written it\"\n - flush_lag_seconds:\n usage: \"GAUGE\"\n description: \"Time elapsed between flushing recent WAL locally and receiving notification that this standby server has written and flushed it\"\n - replay_lag_seconds:\n usage: \"GAUGE\"\n description: \"Time elapsed between flushing recent WAL locally and receiving notification that this standby server has written, flushed and applied it\"\n\npg_settings:\n query: |\n SELECT name,\n CASE setting WHEN 'on' THEN '1' WHEN 'off' THEN '0' ELSE setting END AS setting\n FROM pg_catalog.pg_settings\n WHERE vartype IN ('integer', 'real', 'bool')\n ORDER BY 1\n metrics:\n - name:\n usage: \"LABEL\"\n description: \"Name of the setting\"\n - setting:\n usage: \"GAUGE\"\n description: \"Setting value\"\n"` | A string representation of a YAML defining monitoring queries. | | nameOverride | string | `""` | | | namespaceOverride | string | `""` | | | nodeSelector | object | `{}` | Nodeselector for the operator to be installed. | @@ -77,7 +77,5 @@ CloudNativePG Operator Helm Chart | serviceAccount.create | bool | `true` | Specifies whether the service account should be created. | | serviceAccount.name | string | `""` | The name of the service account to use. If not set and create is true, a name is generated using the fullname template. | | tolerations | list | `[]` | Tolerations for the operator to be installed. | -| topologySpreadConstraints | list | `[]` | Topology Spread Constraints for the operator to be installed. | -| updateStrategy | object | `{}` | Update strategy for the operator. ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy For example: type: RollingUpdate rollingUpdate: maxSurge: 25% maxUnavailable: 25% | -| webhook | object | `{"livenessProbe":{"initialDelaySeconds":3},"mutating":{"create":true,"failurePolicy":"Fail"},"port":9443,"readinessProbe":{"initialDelaySeconds":3},"startupProbe":{"failureThreshold":6,"periodSeconds":5},"validating":{"create":true,"failurePolicy":"Fail"}}` | The webhook configuration. | +| webhook | object | `{"livenessProbe":{"initialDelaySeconds":3},"mutating":{"create":true,"failurePolicy":"Fail"},"port":9443,"readinessProbe":{"initialDelaySeconds":3},"validating":{"create":true,"failurePolicy":"Fail"}}` | The webhook configuration. | diff --git a/packages/system/postgres-operator/charts/cloudnative-pg/templates/NOTES.txt b/packages/system/postgres-operator/charts/cloudnative-pg/templates/NOTES.txt index 4c8ef66d..d0b65b9b 100644 --- a/packages/system/postgres-operator/charts/cloudnative-pg/templates/NOTES.txt +++ b/packages/system/postgres-operator/charts/cloudnative-pg/templates/NOTES.txt @@ -1,5 +1,5 @@ -CloudNativePG operator should be installed in namespace "{{ include "cloudnative-pg.namespace" . }}". +CloudNativePG operator should be installed in namespace "{{ .Release.Namespace }}". You can now create a PostgreSQL cluster with 3 nodes as follows: cat <= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. @@ -4382,41 +4322,41 @@ spec: type: string modulus: description: |- - modulus to take of the hash of the source label values. + Modulus to take of the hash of the source label values. Only applicable when the action is `HashMod`. format: int64 type: integer regex: - description: regex defines the regular expression against - which the extracted value is matched. + description: Regular expression against which the extracted + value is matched. type: string replacement: description: |- - replacement value against which a Replace action is performed if the + Replacement value against which a Replace action is performed if the regular expression matches. Regex capture groups are available. type: string separator: - description: separator defines the string between concatenated + description: Separator is the string between concatenated SourceLabels. type: string sourceLabels: description: |- - sourceLabels defines the source labels select values from existing labels. Their content is + The source labels select values from existing labels. Their content is concatenated using the configured Separator and matched against the configured regular expression. items: description: |- - LabelName is a valid Prometheus label name. - For Prometheus 3.x, a label name is valid if it contains UTF-8 characters. - For Prometheus 2.x, a label name is only valid if it contains ASCII characters, letters, numbers, as well as underscores. + LabelName is a valid Prometheus label name which may only contain ASCII + letters, numbers, as well as underscores. + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ type: string type: array targetLabel: description: |- - targetLabel defines the label to which the resulting string is written in a replacement. + Label to which the resulting string is written in a replacement. It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. @@ -4426,11 +4366,8 @@ spec: type: object type: array podMonitorRelabelings: - description: |- - The list of relabelings for the `PodMonitor`. Applied to samples before scraping. - - Deprecated: This feature will be removed in an upcoming release. If - you need this functionality, you can create a PodMonitor manually. + description: The list of relabelings for the `PodMonitor`. Applied + to samples before scraping. items: description: |- RelabelConfig allows dynamic rewriting of the label set for targets, alerts, @@ -4441,7 +4378,7 @@ spec: action: default: replace description: |- - action to perform based on the regex matching. + Action to perform based on the regex matching. `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. @@ -4473,41 +4410,41 @@ spec: type: string modulus: description: |- - modulus to take of the hash of the source label values. + Modulus to take of the hash of the source label values. Only applicable when the action is `HashMod`. format: int64 type: integer regex: - description: regex defines the regular expression against - which the extracted value is matched. + description: Regular expression against which the extracted + value is matched. type: string replacement: description: |- - replacement value against which a Replace action is performed if the + Replacement value against which a Replace action is performed if the regular expression matches. Regex capture groups are available. type: string separator: - description: separator defines the string between concatenated + description: Separator is the string between concatenated SourceLabels. type: string sourceLabels: description: |- - sourceLabels defines the source labels select values from existing labels. Their content is + The source labels select values from existing labels. Their content is concatenated using the configured Separator and matched against the configured regular expression. items: description: |- - LabelName is a valid Prometheus label name. - For Prometheus 3.x, a label name is valid if it contains UTF-8 characters. - For Prometheus 2.x, a label name is only valid if it contains ASCII characters, letters, numbers, as well as underscores. + LabelName is a valid Prometheus label name which may only contain ASCII + letters, numbers, as well as underscores. + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ type: string type: array targetLabel: description: |- - targetLabel defines the label to which the resulting string is written in a replacement. + Label to which the resulting string is written in a replacement. It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. @@ -4556,13 +4493,6 @@ spec: default: true description: Enabled is true if this plugin will be used type: boolean - isWALArchiver: - default: false - description: |- - Marks the plugin as the WAL archiver. At most one plugin can be - designated as a WAL archiver. This cannot be enabled if the - `.spec.backup.barmanObjectStore` configuration is present. - type: boolean name: description: Name is the plugin name type: string @@ -4597,67 +4527,6 @@ spec: This should only be used for debugging and troubleshooting. Defaults to false. type: boolean - extensions: - description: The configuration of the extensions to be added - items: - description: |- - ExtensionConfiguration is the configuration used to add - PostgreSQL extensions to the Cluster. - properties: - dynamic_library_path: - description: |- - The list of directories inside the image which should be added to dynamic_library_path. - If not defined, defaults to "/lib". - items: - type: string - type: array - extension_control_path: - description: |- - The list of directories inside the image which should be added to extension_control_path. - If not defined, defaults to "/share". - items: - type: string - type: array - image: - description: The image containing the extension, required - properties: - pullPolicy: - description: |- - Policy for pulling OCI objects. Possible values are: - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. - Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. - type: string - reference: - description: |- - Required: Image or artifact reference to be used. - Behaves in the same way as pod.spec.containers[*].image. - Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. - More info: https://kubernetes.io/docs/concepts/containers/images - This field is optional to allow higher level config management to default or override - container images in workload controllers like Deployments and StatefulSets. - type: string - type: object - x-kubernetes-validations: - - message: An image reference is required - rule: has(self.reference) - ld_library_path: - description: The list of directories inside the image which - should be added to ld_library_path. - items: - type: string - type: array - name: - description: The name of the extension, required - minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - required: - - image - - name - type: object - type: array ldap: description: Options to specify LDAP configuration properties: @@ -4785,6 +4654,7 @@ spec: feature properties: dataDurability: + default: required description: |- If set to "required", data durability is strictly enforced. Write operations with synchronous commit settings (`on`, `remote_write`, or `remote_apply`) will @@ -4896,30 +4766,6 @@ spec: More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer - isolationCheck: - description: |- - Configure the feature that extends the liveness probe for a primary - instance. In addition to the basic checks, this verifies whether the - primary is isolated from the Kubernetes API server and from its - replicas, ensuring that it can be safely shut down if network - partition or API unavailability is detected. Enabled by default. - properties: - connectionTimeout: - default: 1000 - description: Timeout in milliseconds for connections during - the primary isolation check - type: integer - enabled: - default: true - description: Whether primary isolation checking is enabled - for the liveness probe - type: boolean - requestTimeout: - default: 1000 - description: Timeout in milliseconds for requests during - the primary isolation check - type: integer - type: object periodSeconds: description: |- How often (in seconds) to perform the probe. @@ -4969,13 +4815,6 @@ spec: More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer - maximumLag: - anyOf: - - type: integer - - type: string - description: Lag limit. Used only for `streaming` strategy - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true periodSeconds: description: |- How often (in seconds) to perform the probe. @@ -5009,13 +4848,6 @@ spec: More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer - type: - description: The probe strategy - enum: - - pg_isready - - streaming - - query - type: string type: object startup: description: The startup probe configuration @@ -5032,13 +4864,6 @@ spec: More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer - maximumLag: - anyOf: - - type: integer - - type: string - description: Lag limit. Used only for `streaming` strategy - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true periodSeconds: description: |- How often (in seconds) to perform the probe. @@ -5072,13 +4897,6 @@ spec: More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes format: int32 type: integer - type: - description: The probe strategy - enum: - - pg_isready - - streaming - - query - type: string type: object type: object projectedVolumeTemplate: @@ -5329,111 +5147,6 @@ spec: type: array x-kubernetes-list-type: atomic type: object - podCertificate: - description: |- - Projects an auto-rotating credential bundle (private key and certificate - chain) that the pod can use either as a TLS client or server. - - Kubelet generates a private key and uses it to send a - PodCertificateRequest to the named signer. Once the signer approves the - request and issues a certificate chain, Kubelet writes the key and - certificate chain to the pod filesystem. The pod does not start until - certificates have been issued for each podCertificate projected volume - source in its spec. - - Kubelet will begin trying to rotate the certificate at the time indicated - by the signer using the PodCertificateRequest.Status.BeginRefreshAt - timestamp. - - Kubelet can write a single file, indicated by the credentialBundlePath - field, or separate files, indicated by the keyPath and - certificateChainPath fields. - - The credential bundle is a single file in PEM format. The first PEM - entry is the private key (in PKCS#8 format), and the remaining PEM - entries are the certificate chain issued by the signer (typically, - signers will return their certificate chain in leaf-to-root order). - - Prefer using the credential bundle format, since your application code - can read it atomically. If you use keyPath and certificateChainPath, - your application must make two separate file reads. If these coincide - with a certificate rotation, it is possible that the private key and leaf - certificate you read may not correspond to each other. Your application - will need to check for this condition, and re-read until they are - consistent. - - The named signer controls chooses the format of the certificate it - issues; consult the signer implementation's documentation to learn how to - use the certificates it issues. - properties: - certificateChainPath: - description: |- - Write the certificate chain at this path in the projected volume. - - Most applications should use credentialBundlePath. When using keyPath - and certificateChainPath, your application needs to check that the key - and leaf certificate are consistent, because it is possible to read the - files mid-rotation. - type: string - credentialBundlePath: - description: |- - Write the credential bundle at this path in the projected volume. - - The credential bundle is a single file that contains multiple PEM blocks. - The first PEM block is a PRIVATE KEY block, containing a PKCS#8 private - key. - - The remaining blocks are CERTIFICATE blocks, containing the issued - certificate chain from the signer (leaf and any intermediates). - - Using credentialBundlePath lets your Pod's application code make a single - atomic read that retrieves a consistent key and certificate chain. If you - project them to separate files, your application code will need to - additionally check that the leaf certificate was issued to the key. - type: string - keyPath: - description: |- - Write the key at this path in the projected volume. - - Most applications should use credentialBundlePath. When using keyPath - and certificateChainPath, your application needs to check that the key - and leaf certificate are consistent, because it is possible to read the - files mid-rotation. - type: string - keyType: - description: |- - The type of keypair Kubelet will generate for the pod. - - Valid values are "RSA3072", "RSA4096", "ECDSAP256", "ECDSAP384", - "ECDSAP521", and "ED25519". - type: string - maxExpirationSeconds: - description: |- - maxExpirationSeconds is the maximum lifetime permitted for the - certificate. - - Kubelet copies this value verbatim into the PodCertificateRequests it - generates for this projection. - - If omitted, kube-apiserver will set it to 86400(24 hours). kube-apiserver - will reject values shorter than 3600 (1 hour). The maximum allowable - value is 7862400 (91 days). - - The signer implementation is then free to issue a certificate with any - lifetime *shorter* than MaxExpirationSeconds, but no shorter than 3600 - seconds (1 hour). This constraint is enforced by kube-apiserver. - `kubernetes.io` signers will never issue certificates with a lifetime - longer than 24 hours. - format: int32 - type: integer - signerName: - description: Kubelet's generated CSRs will be addressed - to this signer. - type: string - required: - - keyType - - signerName - type: object secret: description: secret information about the secret data to project @@ -5596,15 +5309,6 @@ spec: This can only be set at creation time. By default set to `_cnpg_`. pattern: ^[0-9a-z_]*$ type: string - synchronizeLogicalDecoding: - description: |- - When enabled, the operator automatically manages synchronization of logical - decoding (replication) slots across high-availability clusters. - - Requires one of the following conditions: - - PostgreSQL version 17 or later - - PostgreSQL version < 17 with pg_failover_slots extension enabled - type: boolean type: object synchronizeReplicas: description: Configures the synchronization of the user defined @@ -5644,7 +5348,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This field depends on the + This is an alpha field and requires enabling the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -5765,7 +5469,7 @@ spec: description: |- The time in seconds that controls the window of time reserved for the smart shutdown of Postgres to complete. Make sure you reserve enough time for the operator to request a fast shutdown of Postgres - (that is: `stopDelay` - `smartShutdownTimeout`). Default is 180 seconds. + (that is: `stopDelay` - `smartShutdownTimeout`). format: int32 type: integer startDelay: @@ -5965,13 +5669,15 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string or nil value indicates that no - VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, - this field can be reset to its previous value (including nil) to cancel the modification. + it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass + will be applied to the claim but it's not allowed to reset this field to empty string once it is set. + If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass + will be set by the persistentvolume controller if it exists. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ + (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -6221,13 +5927,15 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string or nil value indicates that no - VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, - this field can be reset to its previous value (including nil) to cancel the modification. + it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass + will be applied to the claim but it's not allowed to reset this field to empty string once it is set. + If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass + will be set by the persistentvolume controller if it exists. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ + (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -6396,6 +6104,7 @@ spec: - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. If this value is nil, the behavior is equivalent to the Honor policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. type: string nodeTaintsPolicy: description: |- @@ -6406,6 +6115,7 @@ spec: - Ignore: node taints are ignored. All nodes are included. If this value is nil, the behavior is equivalent to the Ignore policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. type: string topologyKey: description: |- @@ -6629,13 +6339,15 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string or nil value indicates that no - VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, - this field can be reset to its previous value (including nil) to cancel the modification. + it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass + will be applied to the claim but it's not allowed to reset this field to empty string once it is set. + If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass + will be set by the persistentvolume controller if it exists. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ + (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -6695,6 +6407,10 @@ spec: - hash type: object type: array + azurePVCUpdateEnabled: + description: AzurePVCUpdateEnabled shows if the PVC online upgrade + is enabled for this cluster + type: boolean certificates: description: The configuration for the CA and related certificates, initialized with defaults. @@ -6856,18 +6572,14 @@ spec: firstRecoverabilityPoint: description: |- The first recoverability point, stored as a date in RFC3339 format. - This field is calculated from the content of FirstRecoverabilityPointByMethod. - - Deprecated: the field is not set for backup plugins. + This field is calculated from the content of FirstRecoverabilityPointByMethod type: string firstRecoverabilityPointByMethod: additionalProperties: format: date-time type: string - description: |- - The first recoverability point, stored as a date in RFC3339 format, per backup method type. - - Deprecated: the field is not set for backup plugins. + description: The first recoverability point, stored as a date in RFC3339 + format, per backup method type type: object healthyPVC: description: List of all the PVCs not dangling nor initializing @@ -6897,9 +6609,6 @@ spec: description: InstanceReportedState describes the last reported state of an instance during a reconciliation loop properties: - ip: - description: IP address of the instance - type: string isPrimary: description: indicates if an instance is the primary one type: boolean @@ -6925,10 +6634,7 @@ spec: format: int32 type: integer lastFailedBackup: - description: |- - Last failed backup, stored as a date in RFC3339 format. - - Deprecated: the field is not set for backup plugins. + description: Stored as a date in RFC3339 format type: string lastPromotionToken: description: |- @@ -6937,19 +6643,15 @@ spec: type: string lastSuccessfulBackup: description: |- - Last successful backup, stored as a date in RFC3339 format. - This field is calculated from the content of LastSuccessfulBackupByMethod. - - Deprecated: the field is not set for backup plugins. + Last successful backup, stored as a date in RFC3339 format + This field is calculated from the content of LastSuccessfulBackupByMethod type: string lastSuccessfulBackupByMethod: additionalProperties: format: date-time type: string - description: |- - Last successful backup, stored as a date in RFC3339 format, per backup method type. - - Deprecated: the field is not set for backup plugins. + description: Last successful backup, stored as a date in RFC3339 format, + per backup method type type: object latestGeneratedNode: description: ID of the latest generated node (used to avoid node name @@ -6997,20 +6699,6 @@ spec: description: OnlineUpdateEnabled shows if the online upgrade is enabled inside the cluster type: boolean - pgDataImageInfo: - description: PGDataImageInfo contains the details of the latest image - that has run on the current data directory. - properties: - image: - description: Image is the image name - type: string - majorVersion: - description: MajorVersion is the major version of the image - type: integer - required: - - image - - majorVersion - type: object phase: description: Current phase of the cluster type: string @@ -7166,9 +6854,6 @@ spec: of switching a cluster to a replica cluster. type: boolean type: object - systemID: - description: SystemID is the latest detected PostgreSQL SystemID - type: string tablespacesStatus: description: TablespacesStatus reports the state of the declarative tablespaces in the cluster @@ -7258,7 +6943,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.19.0 + controller-gen.kubebuilder.io/version: v0.16.5 helm.sh/resource-policy: keep name: databases.postgresql.cnpg.io spec: @@ -7380,43 +7065,6 @@ spec: - present - absent type: string - extensions: - description: The list of extensions to be managed in the database - items: - description: ExtensionSpec configures an extension in a database - properties: - ensure: - default: present - description: |- - Specifies whether an extension/schema should be present or absent in - the database. If set to `present`, the extension/schema will be - created if it does not exist. If set to `absent`, the - extension/schema will be removed if it exists. - enum: - - present - - absent - type: string - name: - description: Name of the extension/schema - type: string - schema: - description: |- - The name of the schema in which to install the extension's objects, - in case the extension allows its contents to be relocated. If not - specified (default), and the extension's control file does not - specify a schema either, the current default object creation schema - is used. - type: string - version: - description: |- - The version of the extension to install. If empty, the operator will - install the default version (whatever is specified in the - extension's control file) - type: string - required: - - name - type: object - type: array icuLocale: description: |- Maps to the `ICU_LOCALE` parameter of `CREATE DATABASE`. This @@ -7496,35 +7144,6 @@ spec: Maps to the `OWNER TO` command of `ALTER DATABASE`. The role name of the user who owns the database inside PostgreSQL. type: string - schemas: - description: The list of schemas to be managed in the database - items: - description: SchemaSpec configures a schema in a database - properties: - ensure: - default: present - description: |- - Specifies whether an extension/schema should be present or absent in - the database. If set to `present`, the extension/schema will be - created if it does not exist. If set to `absent`, the - extension/schema will be removed if it exists. - enum: - - present - - absent - type: string - name: - description: Name of the extension/schema - type: string - owner: - description: |- - The role name of the user who owns the schema inside PostgreSQL. - It maps to the `AUTHORIZATION` parameter of `CREATE SCHEMA` and the - `OWNER TO` command of `ALTER SCHEMA`. - type: string - required: - - name - type: object - type: array tablespace: description: |- Maps to the `TABLESPACE` parameter of `CREATE DATABASE`. @@ -7564,28 +7183,6 @@ spec: applied: description: Applied is true if the database was reconciled correctly type: boolean - extensions: - description: Extensions is the status of the managed extensions - items: - description: DatabaseObjectStatus is the status of the managed database - objects - properties: - applied: - description: |- - True of the object has been installed successfully in - the database - type: boolean - message: - description: Message is the object reconciliation message - type: string - name: - description: The name of the object - type: string - required: - - applied - - name - type: object - type: array message: description: Message is the reconciliation output message type: string @@ -7595,28 +7192,6 @@ spec: desired state that was synchronized format: int64 type: integer - schemas: - description: Schemas is the status of the managed schemas - items: - description: DatabaseObjectStatus is the status of the managed database - objects - properties: - applied: - description: |- - True of the object has been installed successfully in - the database - type: boolean - message: - description: Message is the object reconciliation message - type: string - name: - description: The name of the object - type: string - required: - - applied - - name - type: object - type: array type: object required: - metadata @@ -7631,85 +7206,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.19.0 - helm.sh/resource-policy: keep - name: failoverquorums.postgresql.cnpg.io -spec: - group: postgresql.cnpg.io - names: - kind: FailoverQuorum - listKind: FailoverQuorumList - plural: failoverquorums - singular: failoverquorum - scope: Namespaced - versions: - - additionalPrinterColumns: - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1 - schema: - openAPIV3Schema: - description: |- - FailoverQuorum contains the information about the current failover - quorum status of a PG cluster. It is updated by the instance manager - of the primary node and reset to zero by the operator to trigger - an update. - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - status: - description: Most recently observed status of the failover quorum. - properties: - method: - description: Contains the latest reported Method value. - type: string - primary: - description: |- - Primary is the name of the primary instance that updated - this object the latest time. - type: string - standbyNames: - description: |- - StandbyNames is the list of potentially synchronous - instance names. - items: - type: string - type: array - standbyNumber: - description: |- - StandbyNumber is the number of synchronous standbys that transactions - need to wait for replies from. - type: integer - type: object - required: - - metadata - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.19.0 + controller-gen.kubebuilder.io/version: v0.16.5 helm.sh/resource-policy: keep name: imagecatalogs.postgresql.cnpg.io spec: @@ -7790,7 +7287,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.19.0 + controller-gen.kubebuilder.io/version: v0.16.5 helm.sh/resource-policy: keep name: poolers.postgresql.cnpg.io spec: @@ -7904,11 +7401,8 @@ spec: format: int32 type: integer monitoring: - description: |- - The configuration of the monitoring infrastructure of this pooler. - - Deprecated: This feature will be removed in an upcoming release. If - you need this functionality, you can create a PodMonitor manually. + description: The configuration of the monitoring infrastructure of + this pooler. properties: enablePodMonitor: default: false @@ -7927,7 +7421,7 @@ spec: action: default: replace description: |- - action to perform based on the regex matching. + Action to perform based on the regex matching. `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. @@ -7959,41 +7453,41 @@ spec: type: string modulus: description: |- - modulus to take of the hash of the source label values. + Modulus to take of the hash of the source label values. Only applicable when the action is `HashMod`. format: int64 type: integer regex: - description: regex defines the regular expression against - which the extracted value is matched. + description: Regular expression against which the extracted + value is matched. type: string replacement: description: |- - replacement value against which a Replace action is performed if the + Replacement value against which a Replace action is performed if the regular expression matches. Regex capture groups are available. type: string separator: - description: separator defines the string between concatenated + description: Separator is the string between concatenated SourceLabels. type: string sourceLabels: description: |- - sourceLabels defines the source labels select values from existing labels. Their content is + The source labels select values from existing labels. Their content is concatenated using the configured Separator and matched against the configured regular expression. items: description: |- - LabelName is a valid Prometheus label name. - For Prometheus 3.x, a label name is valid if it contains UTF-8 characters. - For Prometheus 2.x, a label name is only valid if it contains ASCII characters, letters, numbers, as well as underscores. + LabelName is a valid Prometheus label name which may only contain ASCII + letters, numbers, as well as underscores. + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ type: string type: array targetLabel: description: |- - targetLabel defines the label to which the resulting string is written in a replacement. + Label to which the resulting string is written in a replacement. It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. @@ -8015,7 +7509,7 @@ spec: action: default: replace description: |- - action to perform based on the regex matching. + Action to perform based on the regex matching. `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. @@ -8047,41 +7541,41 @@ spec: type: string modulus: description: |- - modulus to take of the hash of the source label values. + Modulus to take of the hash of the source label values. Only applicable when the action is `HashMod`. format: int64 type: integer regex: - description: regex defines the regular expression against - which the extracted value is matched. + description: Regular expression against which the extracted + value is matched. type: string replacement: description: |- - replacement value against which a Replace action is performed if the + Replacement value against which a Replace action is performed if the regular expression matches. Regex capture groups are available. type: string separator: - description: separator defines the string between concatenated + description: Separator is the string between concatenated SourceLabels. type: string sourceLabels: description: |- - sourceLabels defines the source labels select values from existing labels. Their content is + The source labels select values from existing labels. Their content is concatenated using the configured Separator and matched against the configured regular expression. items: description: |- - LabelName is a valid Prometheus label name. - For Prometheus 3.x, a label name is valid if it contains UTF-8 characters. - For Prometheus 2.x, a label name is only valid if it contains ASCII characters, letters, numbers, as well as underscores. + LabelName is a valid Prometheus label name which may only contain ASCII + letters, numbers, as well as underscores. + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ type: string type: array targetLabel: description: |- - targetLabel defines the label to which the resulting string is written in a replacement. + Label to which the resulting string is written in a replacement. It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. @@ -8492,12 +7986,13 @@ spec: type: object trafficDistribution: description: |- - TrafficDistribution offers a way to express preferences for how traffic - is distributed to Service endpoints. Implementations can use this field - as a hint, but are not required to guarantee strict adherence. If the - field is not set, the implementation will apply its default routing - strategy. If set to "PreferClose", implementations should prioritize - endpoints that are in the same zone. + TrafficDistribution offers a way to express preferences for how traffic is + distributed to Service endpoints. Implementations can use this field as a + hint, but are not required to guarantee strict adherence. If the field is + not set, the implementation will apply its default routing strategy. If set + to "PreferClose", implementations should prioritize endpoints that are + topologically close (e.g., same zone). + This is a beta field and requires enabling ServiceTrafficDistribution feature. type: string type: description: |- @@ -8851,6 +8346,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -8865,6 +8361,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -9032,6 +8529,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -9046,6 +8544,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -9139,8 +8638,8 @@ spec: most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), - compute a sum by iterating through the elements of this field and subtracting - "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: description: The weights of all of the matched WeightedPodAffinityTerm @@ -9211,6 +8710,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -9225,6 +8725,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -9392,6 +8893,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -9406,6 +8908,7 @@ spec: pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). items: type: string type: array @@ -9538,9 +9041,8 @@ spec: present in a Container. properties: name: - description: |- - Name of the environment variable. - May consist of any printable ASCII characters except '='. + description: Name of the environment variable. + Must be a C_IDENTIFIER. type: string value: description: |- @@ -9599,43 +9101,6 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic - fileKeyRef: - description: |- - FileKeyRef selects a key of the env file. - Requires the EnvFiles feature gate to be enabled. - properties: - key: - description: |- - The key within the env file. An invalid key will prevent the pod from starting. - The keys defined within a source may consist of any printable ASCII characters except '='. - During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. - type: string - optional: - default: false - description: |- - Specify whether the file or its key must be defined. If the file or key - does not exist, then the env var is not published. - If optional is set to true and the specified key does not exist, - the environment variable will not be set in the Pod's containers. - - If optional is set to false and the specified key does not exist, - an error will be returned during Pod creation. - type: boolean - path: - description: |- - The path within the volume from which to select the file. - Must be relative and may not contain the '..' path or start with '..'. - type: string - volumeName: - description: The name of the volume mount - containing the env file. - type: string - required: - - key - - path - - volumeName - type: object - x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -9698,14 +9163,14 @@ spec: envFrom: description: |- List of sources to populate environment variables in the container. - The keys defined within a source may consist of any printable ASCII characters except '='. - When a key exists in multiple + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated. items: description: EnvFromSource represents the source of - a set of ConfigMaps or Secrets + a set of ConfigMaps properties: configMapRef: description: The ConfigMap to select from @@ -9726,9 +9191,8 @@ spec: type: object x-kubernetes-map-type: atomic prefix: - description: |- - Optional text to prepend to the name of each environment variable. - May consist of any printable ASCII characters except '='. + description: An optional identifier to prepend + to each key in the ConfigMap. Must be a C_IDENTIFIER. type: string secretRef: description: The Secret to select from @@ -9994,12 +9458,6 @@ spec: - port type: object type: object - stopSignal: - description: |- - StopSignal defines which signal will be sent to a container when it is being stopped. - If not specified, the default is defined by the container runtime in use. - StopSignal can only be set for Pods with a non-empty .spec.os.name - type: string type: object livenessProbe: description: |- @@ -10407,7 +9865,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This field depends on the + This is an alpha field and requires enabling the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -10462,10 +9920,10 @@ spec: restartPolicy: description: |- RestartPolicy defines the restart behavior of individual containers in a pod. - This overrides the pod-level restart policy. When this field is not specified, + This field may only be set for init containers, and the only allowed value is "Always". + For non-init containers or when this field is not specified, the restart behavior is defined by the Pod's restart policy and the container type. - Additionally, setting the RestartPolicy as "Always" for the init container will - have the following effect: + Setting the RestartPolicy as "Always" for the init container will have the following effect: this init container will be continually restarted on exit until all regular containers have terminated. Once all regular containers have completed, all init containers with restartPolicy "Always" @@ -10477,59 +9935,6 @@ spec: init container is started, or after any startupProbe has successfully completed. type: string - restartPolicyRules: - description: |- - Represents a list of rules to be checked to determine if the - container should be restarted on exit. The rules are evaluated in - order. Once a rule matches a container exit condition, the remaining - rules are ignored. If no rule matches the container exit condition, - the Container-level restart policy determines the whether the container - is restarted or not. Constraints on the rules: - - At most 20 rules are allowed. - - Rules can have the same action. - - Identical rules are not forbidden in validations. - When rules are specified, container MUST set RestartPolicy explicitly - even it if matches the Pod's RestartPolicy. - items: - description: ContainerRestartRule describes how a - container exit is handled. - properties: - action: - description: |- - Specifies the action taken on a container exit if the requirements - are satisfied. The only possible value is "Restart" to restart the - container. - type: string - exitCodes: - description: Represents the exit codes to check - on container exits. - properties: - operator: - description: |- - Represents the relationship between the container exit code(s) and the - specified values. Possible values are: - - In: the requirement is satisfied if the container exit code is in the - set of specified values. - - NotIn: the requirement is satisfied if the container exit code is - not in the set of specified values. - type: string - values: - description: |- - Specifies the set of values to check for container exit codes. - At most 255 elements are allowed. - items: - format: int32 - type: integer - type: array - x-kubernetes-list-type: set - required: - - operator - type: object - required: - - action - type: object - type: array - x-kubernetes-list-type: atomic securityContext: description: |- SecurityContext defines the security options the container should be run with. @@ -11150,9 +10555,8 @@ spec: present in a Container. properties: name: - description: |- - Name of the environment variable. - May consist of any printable ASCII characters except '='. + description: Name of the environment variable. + Must be a C_IDENTIFIER. type: string value: description: |- @@ -11211,43 +10615,6 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic - fileKeyRef: - description: |- - FileKeyRef selects a key of the env file. - Requires the EnvFiles feature gate to be enabled. - properties: - key: - description: |- - The key within the env file. An invalid key will prevent the pod from starting. - The keys defined within a source may consist of any printable ASCII characters except '='. - During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. - type: string - optional: - default: false - description: |- - Specify whether the file or its key must be defined. If the file or key - does not exist, then the env var is not published. - If optional is set to true and the specified key does not exist, - the environment variable will not be set in the Pod's containers. - - If optional is set to false and the specified key does not exist, - an error will be returned during Pod creation. - type: boolean - path: - description: |- - The path within the volume from which to select the file. - Must be relative and may not contain the '..' path or start with '..'. - type: string - volumeName: - description: The name of the volume mount - containing the env file. - type: string - required: - - key - - path - - volumeName - type: object - x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -11310,14 +10677,14 @@ spec: envFrom: description: |- List of sources to populate environment variables in the container. - The keys defined within a source may consist of any printable ASCII characters except '='. - When a key exists in multiple + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated. items: description: EnvFromSource represents the source of - a set of ConfigMaps or Secrets + a set of ConfigMaps properties: configMapRef: description: The ConfigMap to select from @@ -11338,9 +10705,8 @@ spec: type: object x-kubernetes-map-type: atomic prefix: - description: |- - Optional text to prepend to the name of each environment variable. - May consist of any printable ASCII characters except '='. + description: An optional identifier to prepend + to each key in the ConfigMap. Must be a C_IDENTIFIER. type: string secretRef: description: The Secret to select from @@ -11603,12 +10969,6 @@ spec: - port type: object type: object - stopSignal: - description: |- - StopSignal defines which signal will be sent to a container when it is being stopped. - If not specified, the default is defined by the container runtime in use. - StopSignal can only be set for Pods with a non-empty .spec.os.name - type: string type: object livenessProbe: description: Probes are not allowed for ephemeral containers. @@ -11999,7 +11359,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This field depends on the + This is an alpha field and requires enabling the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -12055,53 +11415,9 @@ spec: description: |- Restart policy for the container to manage the restart behavior of each container within a pod. - You cannot set this field on ephemeral containers. - type: string - restartPolicyRules: - description: |- - Represents a list of rules to be checked to determine if the - container should be restarted on exit. You cannot set this field on + This may only be set for init containers. You cannot set this field on ephemeral containers. - items: - description: ContainerRestartRule describes how a - container exit is handled. - properties: - action: - description: |- - Specifies the action taken on a container exit if the requirements - are satisfied. The only possible value is "Restart" to restart the - container. - type: string - exitCodes: - description: Represents the exit codes to check - on container exits. - properties: - operator: - description: |- - Represents the relationship between the container exit code(s) and the - specified values. Possible values are: - - In: the requirement is satisfied if the container exit code is in the - set of specified values. - - NotIn: the requirement is satisfied if the container exit code is - not in the set of specified values. - type: string - values: - description: |- - Specifies the set of values to check for container exit codes. - At most 255 elements are allowed. - items: - format: int32 - type: integer - type: array - x-kubernetes-list-type: set - required: - - operator - type: object - required: - - action - type: object - type: array - x-kubernetes-list-type: atomic + type: string securityContext: description: |- Optional: SecurityContext defines the security options the ephemeral container should be run with. @@ -12640,9 +11956,7 @@ spec: hostNetwork: description: |- Host networking requested for this pod. Use the host's network namespace. - When using HostNetwork you should specify ports so the scheduler is aware. - When `hostNetwork` is true, specified `hostPort` fields in port definitions must match `containerPort`, - and unspecified `hostPort` fields in port definitions are defaulted to match `containerPort`. + If this option is set, the ports that will be used must be specified. Default to false. type: boolean hostPID: @@ -12667,19 +11981,6 @@ spec: Specifies the hostname of the Pod If not specified, the pod's hostname will be set to a system-defined value. type: string - hostnameOverride: - description: |- - HostnameOverride specifies an explicit override for the pod's hostname as perceived by the pod. - This field only specifies the pod's hostname and does not affect its DNS records. - When this field is set to a non-empty string: - - It takes precedence over the values set in `hostname` and `subdomain`. - - The Pod's hostname will be set to this value. - - `setHostnameAsFQDN` must be nil or set to false. - - `hostNetwork` must be set to false. - - This field must be a valid DNS subdomain as defined in RFC 1123 and contain at most 64 characters. - Requires the HostnameOverride feature gate to be enabled. - type: string imagePullSecrets: description: |- ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images used by this PodSpec. @@ -12715,7 +12016,7 @@ spec: Init containers may not have Lifecycle actions, Readiness probes, Liveness probes, or Startup probes. The resourceRequirements of an init container are taken into account during scheduling by finding the highest request/limit for each resource type, and then using the max of - that value or the sum of the normal containers. Limits are applied to init containers + of that value or the sum of the normal containers. Limits are applied to init containers in a similar fashion. Init containers cannot currently be added or removed. Cannot be updated. @@ -12761,9 +12062,8 @@ spec: present in a Container. properties: name: - description: |- - Name of the environment variable. - May consist of any printable ASCII characters except '='. + description: Name of the environment variable. + Must be a C_IDENTIFIER. type: string value: description: |- @@ -12822,43 +12122,6 @@ spec: - fieldPath type: object x-kubernetes-map-type: atomic - fileKeyRef: - description: |- - FileKeyRef selects a key of the env file. - Requires the EnvFiles feature gate to be enabled. - properties: - key: - description: |- - The key within the env file. An invalid key will prevent the pod from starting. - The keys defined within a source may consist of any printable ASCII characters except '='. - During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. - type: string - optional: - default: false - description: |- - Specify whether the file or its key must be defined. If the file or key - does not exist, then the env var is not published. - If optional is set to true and the specified key does not exist, - the environment variable will not be set in the Pod's containers. - - If optional is set to false and the specified key does not exist, - an error will be returned during Pod creation. - type: boolean - path: - description: |- - The path within the volume from which to select the file. - Must be relative and may not contain the '..' path or start with '..'. - type: string - volumeName: - description: The name of the volume mount - containing the env file. - type: string - required: - - key - - path - - volumeName - type: object - x-kubernetes-map-type: atomic resourceFieldRef: description: |- Selects a resource of the container: only resources limits and requests @@ -12921,14 +12184,14 @@ spec: envFrom: description: |- List of sources to populate environment variables in the container. - The keys defined within a source may consist of any printable ASCII characters except '='. - When a key exists in multiple + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated. items: description: EnvFromSource represents the source of - a set of ConfigMaps or Secrets + a set of ConfigMaps properties: configMapRef: description: The ConfigMap to select from @@ -12949,9 +12212,8 @@ spec: type: object x-kubernetes-map-type: atomic prefix: - description: |- - Optional text to prepend to the name of each environment variable. - May consist of any printable ASCII characters except '='. + description: An optional identifier to prepend + to each key in the ConfigMap. Must be a C_IDENTIFIER. type: string secretRef: description: The Secret to select from @@ -13217,12 +12479,6 @@ spec: - port type: object type: object - stopSignal: - description: |- - StopSignal defines which signal will be sent to a container when it is being stopped. - If not specified, the default is defined by the container runtime in use. - StopSignal can only be set for Pods with a non-empty .spec.os.name - type: string type: object livenessProbe: description: |- @@ -13630,7 +12886,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This field depends on the + This is an alpha field and requires enabling the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -13685,10 +12941,10 @@ spec: restartPolicy: description: |- RestartPolicy defines the restart behavior of individual containers in a pod. - This overrides the pod-level restart policy. When this field is not specified, + This field may only be set for init containers, and the only allowed value is "Always". + For non-init containers or when this field is not specified, the restart behavior is defined by the Pod's restart policy and the container type. - Additionally, setting the RestartPolicy as "Always" for the init container will - have the following effect: + Setting the RestartPolicy as "Always" for the init container will have the following effect: this init container will be continually restarted on exit until all regular containers have terminated. Once all regular containers have completed, all init containers with restartPolicy "Always" @@ -13700,59 +12956,6 @@ spec: init container is started, or after any startupProbe has successfully completed. type: string - restartPolicyRules: - description: |- - Represents a list of rules to be checked to determine if the - container should be restarted on exit. The rules are evaluated in - order. Once a rule matches a container exit condition, the remaining - rules are ignored. If no rule matches the container exit condition, - the Container-level restart policy determines the whether the container - is restarted or not. Constraints on the rules: - - At most 20 rules are allowed. - - Rules can have the same action. - - Identical rules are not forbidden in validations. - When rules are specified, container MUST set RestartPolicy explicitly - even it if matches the Pod's RestartPolicy. - items: - description: ContainerRestartRule describes how a - container exit is handled. - properties: - action: - description: |- - Specifies the action taken on a container exit if the requirements - are satisfied. The only possible value is "Restart" to restart the - container. - type: string - exitCodes: - description: Represents the exit codes to check - on container exits. - properties: - operator: - description: |- - Represents the relationship between the container exit code(s) and the - specified values. Possible values are: - - In: the requirement is satisfied if the container exit code is in the - set of specified values. - - NotIn: the requirement is satisfied if the container exit code is - not in the set of specified values. - type: string - values: - description: |- - Specifies the set of values to check for container exit codes. - At most 255 elements are allowed. - items: - format: int32 - type: integer - type: array - x-kubernetes-list-type: set - required: - - operator - type: object - required: - - action - type: object - type: array - x-kubernetes-list-type: atomic securityContext: description: |- SecurityContext defines the security options the container should be run with. @@ -14286,7 +13489,6 @@ spec: - spec.hostPID - spec.hostIPC - spec.hostUsers - - spec.resources - spec.securityContext.appArmorProfile - spec.securityContext.seLinuxOptions - spec.securityContext.seccompProfile @@ -14440,7 +13642,7 @@ spec: description: |- Resources is the total amount of CPU and Memory resources required by all containers in the pod. It supports specifying Requests and Limits for - "cpu", "memory" and "hugepages-" resource names only. ResourceClaims are not supported. + "cpu" and "memory" resource names only. ResourceClaims are not supported. This field enables fine-grained control over resource allocation for the entire pod, allowing resource sharing among containers in a pod. @@ -14453,7 +13655,7 @@ spec: Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. - This field depends on the + This is an alpha field and requires enabling the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. @@ -14991,6 +14193,7 @@ spec: - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. If this value is nil, the behavior is equivalent to the Honor policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. type: string nodeTaintsPolicy: description: |- @@ -15001,6 +14204,7 @@ spec: - Ignore: node taints are ignored. All nodes are included. If this value is nil, the behavior is equivalent to the Ignore policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. type: string topologyKey: description: |- @@ -15730,13 +14934,15 @@ spec: volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, - it can be changed after the claim is created. An empty string or nil value indicates that no - VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, - this field can be reset to its previous value (including nil) to cancel the modification. + it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass + will be applied to the claim but it's not allowed to reset this field to empty string once it is set. + If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass + will be set by the persistentvolume controller if it exists. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ + (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). type: string volumeMode: description: |- @@ -15918,10 +15124,12 @@ spec: description: |- glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported. + More info: https://examples.k8s.io/volumes/glusterfs/README.md properties: endpoints: - description: endpoints is the endpoint name that - details Glusterfs topology. + description: |- + endpoints is the endpoint name that details Glusterfs topology. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod type: string path: description: |- @@ -15975,7 +15183,7 @@ spec: The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. The volume will be mounted read-only (ro) and non-executable files (noexec). - Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath) before 1.33. + Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath). The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type. properties: pullPolicy: @@ -16000,7 +15208,7 @@ spec: description: |- iscsi represents an ISCSI Disk resource that is attached to a kubelet's host machine and then exposed to the pod. - More info: https://kubernetes.io/docs/concepts/storage/volumes/#iscsi + More info: https://examples.k8s.io/volumes/iscsi/README.md properties: chapAuthDiscovery: description: chapAuthDiscovery defines whether support @@ -16426,111 +15634,6 @@ spec: type: array x-kubernetes-list-type: atomic type: object - podCertificate: - description: |- - Projects an auto-rotating credential bundle (private key and certificate - chain) that the pod can use either as a TLS client or server. - - Kubelet generates a private key and uses it to send a - PodCertificateRequest to the named signer. Once the signer approves the - request and issues a certificate chain, Kubelet writes the key and - certificate chain to the pod filesystem. The pod does not start until - certificates have been issued for each podCertificate projected volume - source in its spec. - - Kubelet will begin trying to rotate the certificate at the time indicated - by the signer using the PodCertificateRequest.Status.BeginRefreshAt - timestamp. - - Kubelet can write a single file, indicated by the credentialBundlePath - field, or separate files, indicated by the keyPath and - certificateChainPath fields. - - The credential bundle is a single file in PEM format. The first PEM - entry is the private key (in PKCS#8 format), and the remaining PEM - entries are the certificate chain issued by the signer (typically, - signers will return their certificate chain in leaf-to-root order). - - Prefer using the credential bundle format, since your application code - can read it atomically. If you use keyPath and certificateChainPath, - your application must make two separate file reads. If these coincide - with a certificate rotation, it is possible that the private key and leaf - certificate you read may not correspond to each other. Your application - will need to check for this condition, and re-read until they are - consistent. - - The named signer controls chooses the format of the certificate it - issues; consult the signer implementation's documentation to learn how to - use the certificates it issues. - properties: - certificateChainPath: - description: |- - Write the certificate chain at this path in the projected volume. - - Most applications should use credentialBundlePath. When using keyPath - and certificateChainPath, your application needs to check that the key - and leaf certificate are consistent, because it is possible to read the - files mid-rotation. - type: string - credentialBundlePath: - description: |- - Write the credential bundle at this path in the projected volume. - - The credential bundle is a single file that contains multiple PEM blocks. - The first PEM block is a PRIVATE KEY block, containing a PKCS#8 private - key. - - The remaining blocks are CERTIFICATE blocks, containing the issued - certificate chain from the signer (leaf and any intermediates). - - Using credentialBundlePath lets your Pod's application code make a single - atomic read that retrieves a consistent key and certificate chain. If you - project them to separate files, your application code will need to - additionally check that the leaf certificate was issued to the key. - type: string - keyPath: - description: |- - Write the key at this path in the projected volume. - - Most applications should use credentialBundlePath. When using keyPath - and certificateChainPath, your application needs to check that the key - and leaf certificate are consistent, because it is possible to read the - files mid-rotation. - type: string - keyType: - description: |- - The type of keypair Kubelet will generate for the pod. - - Valid values are "RSA3072", "RSA4096", "ECDSAP256", "ECDSAP384", - "ECDSAP521", and "ED25519". - type: string - maxExpirationSeconds: - description: |- - maxExpirationSeconds is the maximum lifetime permitted for the - certificate. - - Kubelet copies this value verbatim into the PodCertificateRequests it - generates for this projection. - - If omitted, kube-apiserver will set it to 86400(24 hours). kube-apiserver - will reject values shorter than 3600 (1 hour). The maximum allowable - value is 7862400 (91 days). - - The signer implementation is then free to issue a certificate with any - lifetime *shorter* than MaxExpirationSeconds, but no shorter than 3600 - seconds (1 hour). This constraint is enforced by kube-apiserver. - `kubernetes.io` signers will never issue certificates with a lifetime - longer than 24 hours. - format: int32 - type: integer - signerName: - description: Kubelet's generated CSRs - will be addressed to this signer. - type: string - required: - - keyType - - signerName - type: object secret: description: secret information about the secret data to project @@ -16665,6 +15768,7 @@ spec: description: |- rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported. + More info: https://examples.k8s.io/volumes/rbd/README.md properties: fsType: description: |- @@ -16962,7 +16066,6 @@ spec: enum: - rw - ro - - r type: string required: - cluster @@ -17043,7 +16146,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.19.0 + controller-gen.kubebuilder.io/version: v0.16.5 helm.sh/resource-policy: keep name: publications.postgresql.cnpg.io spec: @@ -17239,7 +16342,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.19.0 + controller-gen.kubebuilder.io/version: v0.16.5 helm.sh/resource-policy: keep name: scheduledbackups.postgresql.cnpg.io spec: @@ -17431,7 +16534,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.19.0 + controller-gen.kubebuilder.io/version: v0.16.5 helm.sh/resource-policy: keep name: subscriptions.postgresql.cnpg.io spec: @@ -17522,11 +16625,8 @@ spec: additionalProperties: type: string description: |- - Subscription parameters included in the `WITH` clause of the PostgreSQL - `CREATE SUBSCRIPTION` command. Most parameters cannot be changed - after the subscription is created and will be ignored if modified - later, except for a limited set documented at: - https://www.postgresql.org/docs/current/sql-altersubscription.html#SQL-ALTERSUBSCRIPTION-PARAMS-SET + Subscription parameters part of the `WITH` clause as expected by + PostgreSQL `CREATE SUBSCRIPTION` command type: object publicationDBName: description: |- diff --git a/packages/system/postgres-operator/charts/cloudnative-pg/templates/deployment.yaml b/packages/system/postgres-operator/charts/cloudnative-pg/templates/deployment.yaml index 760b3768..7e0bce72 100644 --- a/packages/system/postgres-operator/charts/cloudnative-pg/templates/deployment.yaml +++ b/packages/system/postgres-operator/charts/cloudnative-pg/templates/deployment.yaml @@ -1,6 +1,5 @@ # -# Copyright © contributors to CloudNativePG, established as -# CloudNativePG a Series of LF Projects, LLC. +# Copyright The CloudNativePG Contributors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# SPDX-License-Identifier: Apache-2.0 -# --- apiVersion: apps/v1 kind: Deployment @@ -33,10 +30,6 @@ spec: selector: matchLabels: {{- include "cloudnative-pg.selectorLabels" . | nindent 6 }} - {{- if .Values.updateStrategy }} - strategy: - {{- toYaml .Values.updateStrategy | nindent 4 }} - {{- end }} template: metadata: annotations: @@ -91,7 +84,7 @@ spec: value: "{{ .Values.monitoringQueriesConfigMap.name }}" {{- if not .Values.config.clusterWide }} - name: WATCH_NAMESPACE - value: "{{ include "cloudnative-pg.namespace" . }}" + value: "{{ .Release.Namespace }}" {{- end }} {{- if .Values.additionalEnv }} {{- tpl (.Values.additionalEnv | toYaml) . | nindent 8 }} @@ -126,17 +119,6 @@ spec: {{- toYaml .Values.resources | nindent 10 }} securityContext: {{- toYaml .Values.containerSecurityContext | nindent 10 }} - startupProbe: - {{- if .Values.webhook.startupProbe.failureThreshold }} - failureThreshold: {{ .Values.webhook.startupProbe.failureThreshold }} - {{- end }} - httpGet: - path: /readyz - port: {{ .Values.webhook.port }} - scheme: HTTPS - {{- if .Values.webhook.startupProbe.periodSeconds }} - periodSeconds: {{ .Values.webhook.startupProbe.periodSeconds }} - {{- end }} volumeMounts: - mountPath: /controller name: scratch-data @@ -153,10 +135,6 @@ spec: nodeSelector: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.topologySpreadConstraints }} - topologySpreadConstraints: - {{- toYaml . | nindent 8 }} - {{- end }} {{- with .Values.affinity }} affinity: {{- toYaml . | nindent 8 }} @@ -173,3 +151,5 @@ spec: defaultMode: 420 optional: true secretName: cnpg-webhook-cert + + diff --git a/packages/system/postgres-operator/charts/cloudnative-pg/templates/monitoring-configmap.yaml b/packages/system/postgres-operator/charts/cloudnative-pg/templates/monitoring-configmap.yaml index d47bc744..aa5937d5 100644 --- a/packages/system/postgres-operator/charts/cloudnative-pg/templates/monitoring-configmap.yaml +++ b/packages/system/postgres-operator/charts/cloudnative-pg/templates/monitoring-configmap.yaml @@ -1,6 +1,5 @@ # -# Copyright © contributors to CloudNativePG, established as -# CloudNativePG a Series of LF Projects, LLC. +# Copyright The CloudNativePG Contributors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# SPDX-License-Identifier: Apache-2.0 -# --- apiVersion: v1 kind: ConfigMap diff --git a/packages/system/postgres-operator/charts/cloudnative-pg/templates/mutatingwebhookconfiguration.yaml b/packages/system/postgres-operator/charts/cloudnative-pg/templates/mutatingwebhookconfiguration.yaml index c58628a6..200695b1 100644 --- a/packages/system/postgres-operator/charts/cloudnative-pg/templates/mutatingwebhookconfiguration.yaml +++ b/packages/system/postgres-operator/charts/cloudnative-pg/templates/mutatingwebhookconfiguration.yaml @@ -1,6 +1,5 @@ # -# Copyright © contributors to CloudNativePG, established as -# CloudNativePG a Series of LF Projects, LLC. +# Copyright The CloudNativePG Contributors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# SPDX-License-Identifier: Apache-2.0 -# {{- if .Values.webhook.mutating.create }} --- apiVersion: admissionregistration.k8s.io/v1 @@ -34,7 +31,7 @@ webhooks: clientConfig: service: name: {{ .Values.service.name }} - namespace: {{ include "cloudnative-pg.namespace" . }} + namespace: {{ .Release.Namespace }} path: /mutate-postgresql-cnpg-io-v1-backup port: {{ .Values.service.port }} failurePolicy: {{ .Values.webhook.mutating.failurePolicy }} @@ -55,7 +52,7 @@ webhooks: clientConfig: service: name: {{ .Values.service.name }} - namespace: {{ include "cloudnative-pg.namespace" . }} + namespace: {{ .Release.Namespace }} path: /mutate-postgresql-cnpg-io-v1-cluster port: {{ .Values.service.port }} failurePolicy: {{ .Values.webhook.mutating.failurePolicy }} @@ -76,28 +73,7 @@ webhooks: clientConfig: service: name: {{ .Values.service.name }} - namespace: {{ include "cloudnative-pg.namespace" . }} - path: /mutate-postgresql-cnpg-io-v1-database - port: {{ .Values.service.port }} - failurePolicy: {{ .Values.webhook.mutating.failurePolicy }} - name: mdatabase.cnpg.io - rules: - - apiGroups: - - postgresql.cnpg.io - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - databases - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: {{ .Values.service.name }} - namespace: {{ include "cloudnative-pg.namespace" . }} + namespace: {{ .Release.Namespace }} path: /mutate-postgresql-cnpg-io-v1-scheduledbackup port: {{ .Values.service.port }} failurePolicy: {{ .Values.webhook.mutating.failurePolicy }} diff --git a/packages/system/postgres-operator/charts/cloudnative-pg/templates/podmonitor.yaml b/packages/system/postgres-operator/charts/cloudnative-pg/templates/podmonitor.yaml index 0d22102b..8984c00f 100644 --- a/packages/system/postgres-operator/charts/cloudnative-pg/templates/podmonitor.yaml +++ b/packages/system/postgres-operator/charts/cloudnative-pg/templates/podmonitor.yaml @@ -1,6 +1,5 @@ # -# Copyright © contributors to CloudNativePG, established as -# CloudNativePG a Series of LF Projects, LLC. +# Copyright The CloudNativePG Contributors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# SPDX-License-Identifier: Apache-2.0 -# {{- if .Values.monitoring.podMonitorEnabled }} apiVersion: monitoring.coreos.com/v1 kind: PodMonitor diff --git a/packages/system/postgres-operator/charts/cloudnative-pg/templates/rbac.yaml b/packages/system/postgres-operator/charts/cloudnative-pg/templates/rbac.yaml index 0dc17080..cf213f35 100644 --- a/packages/system/postgres-operator/charts/cloudnative-pg/templates/rbac.yaml +++ b/packages/system/postgres-operator/charts/cloudnative-pg/templates/rbac.yaml @@ -1,6 +1,5 @@ # -# Copyright © contributors to CloudNativePG, established as -# CloudNativePG a Series of LF Projects, LLC. +# Copyright The CloudNativePG Contributors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# SPDX-License-Identifier: Apache-2.0 -# {{- if .Values.serviceAccount.create }} --- apiVersion: v1 @@ -70,7 +67,7 @@ roleRef: subjects: - kind: ServiceAccount name: {{ include "cloudnative-pg.serviceAccountName" . }} - namespace: {{ include "cloudnative-pg.namespace" . }} + namespace: {{ .Release.Namespace }} {{/* If we're doing a single-namespace installation we create a Role with the common rules for the operator, @@ -111,7 +108,7 @@ roleRef: subjects: - kind: ServiceAccount name: {{ include "cloudnative-pg.serviceAccountName" . }} - namespace: {{ include "cloudnative-pg.namespace" . }} + namespace: {{ .Release.Namespace }} {{- end }} --- apiVersion: rbac.authorization.k8s.io/v1 @@ -131,14 +128,10 @@ rules: resources: - backups - clusters - - clusters/status - databases - - failoverquorums - poolers - publications - scheduledbackups - - imagecatalogs - - clusterimagecatalogs - subscriptions verbs: - get @@ -161,14 +154,10 @@ rules: resources: - backups - clusters - - clusters/status - databases - - failoverquorums - poolers - publications - scheduledbackups - - imagecatalogs - - clusterimagecatalogs - subscriptions verbs: - create diff --git a/packages/system/postgres-operator/charts/cloudnative-pg/templates/service.yaml b/packages/system/postgres-operator/charts/cloudnative-pg/templates/service.yaml index 8afa02fc..13be46ae 100644 --- a/packages/system/postgres-operator/charts/cloudnative-pg/templates/service.yaml +++ b/packages/system/postgres-operator/charts/cloudnative-pg/templates/service.yaml @@ -1,6 +1,5 @@ # -# Copyright © contributors to CloudNativePG, established as -# CloudNativePG a Series of LF Projects, LLC. +# Copyright The CloudNativePG Contributors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# SPDX-License-Identifier: Apache-2.0 -# --- apiVersion: v1 kind: Service diff --git a/packages/system/postgres-operator/charts/cloudnative-pg/templates/validatingwebhookconfiguration.yaml b/packages/system/postgres-operator/charts/cloudnative-pg/templates/validatingwebhookconfiguration.yaml index c171c911..be9fff18 100644 --- a/packages/system/postgres-operator/charts/cloudnative-pg/templates/validatingwebhookconfiguration.yaml +++ b/packages/system/postgres-operator/charts/cloudnative-pg/templates/validatingwebhookconfiguration.yaml @@ -1,6 +1,5 @@ # -# Copyright © contributors to CloudNativePG, established as -# CloudNativePG a Series of LF Projects, LLC. +# Copyright The CloudNativePG Contributors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# SPDX-License-Identifier: Apache-2.0 -# {{- if .Values.webhook.validating.create }} --- apiVersion: admissionregistration.k8s.io/v1 @@ -34,7 +31,7 @@ webhooks: clientConfig: service: name: {{ .Values.service.name }} - namespace: {{ include "cloudnative-pg.namespace" . }} + namespace: {{ .Release.Namespace }} path: /validate-postgresql-cnpg-io-v1-backup port: {{ .Values.service.port }} failurePolicy: {{ .Values.webhook.validating.failurePolicy }} @@ -55,7 +52,7 @@ webhooks: clientConfig: service: name: {{ .Values.service.name }} - namespace: {{ include "cloudnative-pg.namespace" . }} + namespace: {{ .Release.Namespace }} path: /validate-postgresql-cnpg-io-v1-cluster port: {{ .Values.service.port }} failurePolicy: {{ .Values.webhook.validating.failurePolicy }} @@ -76,7 +73,7 @@ webhooks: clientConfig: service: name: {{ .Values.service.name }} - namespace: {{ include "cloudnative-pg.namespace" . }} + namespace: {{ .Release.Namespace }} path: /validate-postgresql-cnpg-io-v1-scheduledbackup port: {{ .Values.service.port }} failurePolicy: {{ .Values.webhook.validating.failurePolicy }} @@ -92,33 +89,12 @@ webhooks: resources: - scheduledbackups sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - service: - name: {{ .Values.service.name }} - namespace: {{ include "cloudnative-pg.namespace" . }} - path: /validate-postgresql-cnpg-io-v1-database - port: {{ .Values.service.port }} - failurePolicy: {{ .Values.webhook.validating.failurePolicy }} - name: vdatabase.cnpg.io - rules: - - apiGroups: - - postgresql.cnpg.io - apiVersions: - - v1 - operations: - - CREATE - - UPDATE - resources: - - databases - sideEffects: None - admissionReviewVersions: - v1 clientConfig: service: name: {{ .Values.service.name }} - namespace: {{ include "cloudnative-pg.namespace" . }} + namespace: {{ .Release.Namespace }} path: /validate-postgresql-cnpg-io-v1-pooler port: {{ .Values.service.port }} failurePolicy: {{ .Values.webhook.validating.failurePolicy }} diff --git a/packages/system/postgres-operator/charts/cloudnative-pg/values.schema.json b/packages/system/postgres-operator/charts/cloudnative-pg/values.schema.json index 4c69aae6..4ba70818 100644 --- a/packages/system/postgres-operator/charts/cloudnative-pg/values.schema.json +++ b/packages/system/postgres-operator/charts/cloudnative-pg/values.schema.json @@ -246,12 +246,6 @@ "tolerations": { "type": "array" }, - "topologySpreadConstraints": { - "type": "array" - }, - "updateStrategy": { - "type": "object" - }, "webhook": { "type": "object", "properties": { @@ -285,17 +279,6 @@ } } }, - "startupProbe": { - "type": "object", - "properties": { - "failureThreshold": { - "type": "integer" - }, - "periodSeconds": { - "type": "integer" - } - } - }, "validating": { "type": "object", "properties": { diff --git a/packages/system/postgres-operator/charts/cloudnative-pg/values.yaml b/packages/system/postgres-operator/charts/cloudnative-pg/values.yaml index cbba7505..cdfb4cf8 100644 --- a/packages/system/postgres-operator/charts/cloudnative-pg/values.yaml +++ b/packages/system/postgres-operator/charts/cloudnative-pg/values.yaml @@ -1,6 +1,5 @@ # -# Copyright © contributors to CloudNativePG, established as -# CloudNativePG a Series of LF Projects, LLC. +# Copyright The CloudNativePG Contributors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# SPDX-License-Identifier: Apache-2.0 -# # Default values for CloudNativePG. # This is a YAML-formatted file. # Please declare variables to be passed to your templates. @@ -36,15 +33,6 @@ namespaceOverride: "" hostNetwork: false dnsPolicy: "" -# -- Update strategy for the operator. -# ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy -# For example: -# type: RollingUpdate -# rollingUpdate: -# maxSurge: 25% -# maxUnavailable: 25% -updateStrategy: {} - crds: # -- Specifies whether the CRDs should be created when installing the chart. create: true @@ -62,9 +50,6 @@ webhook: initialDelaySeconds: 3 readinessProbe: initialDelaySeconds: 3 - startupProbe: - failureThreshold: 6 - periodSeconds: 5 # Operator configuration. config: @@ -88,7 +73,7 @@ config: # -- The maximum number of concurrent reconciles. Defaults to 10. maxConcurrentReconciles: 10 -# -- Additional arguments to be added to the operator's args list. +# -- Additinal arguments to be added to the operator's args list. additionalArgs: [] # -- Array containing extra environment variables which can be templated. @@ -167,9 +152,6 @@ resources: {} # -- Nodeselector for the operator to be installed. nodeSelector: {} -# -- Topology Spread Constraints for the operator to be installed. -topologySpreadConstraints: [] - # -- Tolerations for the operator to be installed. tolerations: [] @@ -655,35 +637,3 @@ monitoringQueriesConfigMap: - setting: usage: "GAUGE" description: "Setting value" - - pg_extensions: - query: | - SELECT - current_database() as datname, - name as extname, - default_version, - installed_version, - CASE - WHEN default_version = installed_version THEN 0 - ELSE 1 - END AS update_available - FROM pg_catalog.pg_available_extensions - WHERE installed_version IS NOT NULL - metrics: - - datname: - usage: "LABEL" - description: "Name of the database" - - extname: - usage: "LABEL" - description: "Extension name" - - default_version: - usage: "LABEL" - description: "Default version" - - installed_version: - usage: "LABEL" - description: "Installed version" - - update_available: - usage: "GAUGE" - description: "An update is available" - target_databases: - - '*' diff --git a/packages/system/postgres-operator/templates/webhook-ready-hook.yaml b/packages/system/postgres-operator/templates/webhook-ready-hook.yaml deleted file mode 100644 index 51a8ba81..00000000 --- a/packages/system/postgres-operator/templates/webhook-ready-hook.yaml +++ /dev/null @@ -1,148 +0,0 @@ -{{- /* - Post-install gate: block the HelmRelease from reporting Ready until the - cnpg admission webhook actually serves through the cluster Service. Helm - --wait on the controller pod passes once its readinessProbe passes, but - EndpointSlice propagation and kube-proxy/cilium data-plane programming - can lag by a second or two — long enough for any HelmRelease that - depends on postgres-operator (e.g. cozy-keycloak, tenant Postgres apps) - to fire its own install and have kube-apiserver hit the mcluster.cnpg.io - mutating webhook with "dial tcp :443: connect: connection refused". - - The Job uses the apiserver service proxy, which exercises the same - endpoint-resolution and apiserver-initiated pod dial that the admission - webhook path uses. Once /readyz answers through the proxy the data-plane - race is resolved. It does not verify the webhook's TLS CA bundle, so - this gate is scoped to reachability regressions, not cert rotation. - - The service name and port name are hardcoded literals. Upstream cnpg - pins the service name in charts/cloudnative-pg/values.yaml with a - comment "DO NOT CHANGE THE SERVICE NAME as it is currently used to - generate the certificate and can not be configured". The port name is - fixed in charts/cloudnative-pg/templates/service.yaml (ports[0].name: - webhook-server). If a future `make update` ever changes either literal - upstream, the sync-check helm-unittest test - (tests/webhook-ready-hook_test.yaml) renders the subchart Service and - fails if the literal drifts — forcing this template to be updated in - the same change. -*/}} -{{- $_ := required "webhookReady.image.repository must be set to the container image providing kubectl for the post-install readiness Job" .Values.webhookReady.image.repository -}} -{{- $_ := required "webhookReady.image.tag must be set for the post-install readiness Job" .Values.webhookReady.image.tag -}} -{{- /* $svcName and $portName are hardcoded literals; see header comment. */ -}} -{{- $svcName := "cnpg-webhook-service" -}} -{{- /* $portName is the service port NAME, not number — matches ports[0].name in the vendored subchart's Service. */ -}} -{{- $portName := "webhook-server" -}} -{{- $resourceName := printf "https:%s:%s" $svcName $portName -}} -{{- $maxAttempts := .Values.webhookReady.maxAttempts | default 60 -}} -{{- $sleepSeconds := .Values.webhookReady.sleepSeconds | default 2 -}} -{{- /* Derive activeDeadlineSeconds from retries + 60s slack so a values override that raises maxAttempts doesn't get silently cut. */ -}} -{{- $deadline := add (mul (int $maxAttempts) (int $sleepSeconds)) 60 -}} -{{- $image := printf "%s:%s" .Values.webhookReady.image.repository .Values.webhookReady.image.tag -}} -{{- if .Values.webhookReady.image.digest }} -{{- $image = printf "%s@%s" $image .Values.webhookReady.image.digest -}} -{{- end }} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ .Release.Name }}-webhook-ready - namespace: {{ .Release.Namespace }} - annotations: - helm.sh/hook: post-install,post-upgrade - helm.sh/hook-weight: "0" - helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: {{ .Release.Name }}-webhook-ready - namespace: {{ .Release.Namespace }} - annotations: - helm.sh/hook: post-install,post-upgrade - helm.sh/hook-weight: "0" - helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded -rules: - - apiGroups: [""] - resources: ["services/proxy"] - resourceNames: [{{ $resourceName | quote }}] - verbs: ["get"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ .Release.Name }}-webhook-ready - namespace: {{ .Release.Namespace }} - annotations: - helm.sh/hook: post-install,post-upgrade - helm.sh/hook-weight: "0" - helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: {{ .Release.Name }}-webhook-ready -subjects: - - kind: ServiceAccount - name: {{ .Release.Name }}-webhook-ready - namespace: {{ .Release.Namespace }} ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: {{ .Release.Name }}-webhook-ready - namespace: {{ .Release.Namespace }} - annotations: - helm.sh/hook: post-install,post-upgrade - helm.sh/hook-weight: "10" - helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded -spec: - backoffLimit: {{ .Values.webhookReady.backoffLimit | default 2 }} - activeDeadlineSeconds: {{ $deadline }} - template: - spec: - restartPolicy: Never - serviceAccountName: {{ .Release.Name }}-webhook-ready - securityContext: - runAsNonRoot: true - runAsUser: 65532 - runAsGroup: 65532 - seccompProfile: - type: RuntimeDefault - containers: - - name: wait - image: {{ $image }} - imagePullPolicy: {{ if .Values.webhookReady.image.digest }}IfNotPresent{{ else }}Always{{ end }} - resources: - requests: - cpu: 10m - memory: 32Mi - limits: - cpu: 100m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - capabilities: - drop: ["ALL"] - command: - - sh - - -c - - | - set -e - ns={{ .Release.Namespace }} - proxy="/api/v1/namespaces/${ns}/services/{{ $resourceName }}/proxy/readyz" - max_attempts={{ $maxAttempts }} - sleep_seconds={{ $sleepSeconds }} - i=0 - last_err="" - until last_err=$(kubectl get --raw "$proxy" 2>&1 >/dev/null); do - i=$((i + 1)) - if [ $i -gt $max_attempts ]; then - echo "timeout: cnpg webhook did not respond through the apiserver proxy after ${max_attempts} attempts (${sleep_seconds}s each)" - echo "last error: ${last_err}" - exit 1 - fi - if [ $((i % 10)) -eq 1 ]; then - echo "attempt $i/${max_attempts}: ${last_err}" - fi - sleep "$sleep_seconds" - done - echo "cnpg webhook is ready" diff --git a/packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml b/packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml deleted file mode 100644 index 3253c663..00000000 --- a/packages/system/postgres-operator/tests/webhook-ready-hook_test.yaml +++ /dev/null @@ -1,328 +0,0 @@ -suite: cnpg webhook post-install readiness gate - -templates: - - templates/webhook-ready-hook.yaml - - charts/cloudnative-pg/templates/service.yaml - -release: - name: postgres-operator - namespace: cozy-postgres-operator - -tests: - - it: renders four hook objects (SA + Role + RoleBinding + Job) - template: templates/webhook-ready-hook.yaml - asserts: - - hasDocuments: - count: 4 - - - it: every rendered object carries post-install and post-upgrade hook annotations - template: templates/webhook-ready-hook.yaml - asserts: - - documentIndex: 0 - equal: - path: metadata.annotations["helm.sh/hook"] - value: post-install,post-upgrade - - documentIndex: 1 - equal: - path: metadata.annotations["helm.sh/hook"] - value: post-install,post-upgrade - - documentIndex: 2 - equal: - path: metadata.annotations["helm.sh/hook"] - value: post-install,post-upgrade - - documentIndex: 3 - equal: - path: metadata.annotations["helm.sh/hook"] - value: post-install,post-upgrade - - - it: RBAC is created before the Job (hook-weight ordering) - template: templates/webhook-ready-hook.yaml - asserts: - - documentIndex: 0 - equal: - path: kind - value: ServiceAccount - - documentIndex: 0 - equal: - path: metadata.annotations["helm.sh/hook-weight"] - value: "0" - - documentIndex: 1 - equal: - path: kind - value: Role - - documentIndex: 1 - equal: - path: metadata.annotations["helm.sh/hook-weight"] - value: "0" - - documentIndex: 2 - equal: - path: kind - value: RoleBinding - - documentIndex: 2 - equal: - path: metadata.annotations["helm.sh/hook-weight"] - value: "0" - - documentIndex: 3 - equal: - path: kind - value: Job - - documentIndex: 3 - equal: - path: metadata.annotations["helm.sh/hook-weight"] - value: "10" - - - it: RBAC resourceName matches the exact proxy URL segment the Job probes - template: templates/webhook-ready-hook.yaml - asserts: - - documentIndex: 1 - equal: - path: rules[0].resources[0] - value: services/proxy - - documentIndex: 1 - equal: - path: rules[0].resourceNames[0] - value: https:cnpg-webhook-service:webhook-server - - documentIndex: 3 - matchRegex: - path: spec.template.spec.containers[0].command[2] - pattern: "/api/v1/namespaces/\\$\\{ns\\}/services/https:cnpg-webhook-service:webhook-server/proxy/readyz" - - - it: hardcoded service name in the hook matches the vendored cnpg subchart Service (drift guard for make update) - template: charts/cloudnative-pg/templates/service.yaml - asserts: - - equal: - path: metadata.name - value: cnpg-webhook-service - - equal: - path: spec.ports[0].name - value: webhook-server - - - it: Job calls kubectl get --raw on the proxy path - template: templates/webhook-ready-hook.yaml - asserts: - - documentIndex: 3 - matchRegex: - path: spec.template.spec.containers[0].command[2] - pattern: "kubectl get --raw" - - - it: Job image is digest-pinned when webhookReady.image.digest is set - template: templates/webhook-ready-hook.yaml - set: - webhookReady: - image: - repository: docker.io/clastix/kubectl - tag: v1.32 - digest: sha256:b9ef7d8dbe65bcc81a46c09b8dc7543103055021c4f43287bf59e92a8f4fe05c - asserts: - - documentIndex: 3 - equal: - path: spec.template.spec.containers[0].image - value: docker.io/clastix/kubectl:v1.32@sha256:b9ef7d8dbe65bcc81a46c09b8dc7543103055021c4f43287bf59e92a8f4fe05c - - - it: Job image falls back to tag-only when digest is not configured - template: templates/webhook-ready-hook.yaml - set: - webhookReady: - image: - repository: docker.io/clastix/kubectl - tag: v1.32 - digest: "" - asserts: - - documentIndex: 3 - equal: - path: spec.template.spec.containers[0].image - value: docker.io/clastix/kubectl:v1.32 - - - it: backoffLimit defaults to 2 so transient pod-level failures retry instead of killing the HelmRelease - template: templates/webhook-ready-hook.yaml - asserts: - - documentIndex: 3 - equal: - path: spec.backoffLimit - value: 2 - - - it: backoffLimit is overridable - template: templates/webhook-ready-hook.yaml - set: - webhookReady: - backoffLimit: 5 - asserts: - - documentIndex: 3 - equal: - path: spec.backoffLimit - value: 5 - - - it: chart render fails when webhookReady.image.repository is empty - template: templates/webhook-ready-hook.yaml - set: - webhookReady: - image: - repository: "" - tag: v1.32 - asserts: - - failedTemplate: - errorMessage: "webhookReady.image.repository must be set to the container image providing kubectl for the post-install readiness Job" - - - it: chart render fails when webhookReady.image.tag is empty - template: templates/webhook-ready-hook.yaml - set: - webhookReady: - image: - repository: docker.io/clastix/kubectl - tag: "" - asserts: - - failedTemplate: - errorMessage: "webhookReady.image.tag must be set for the post-install readiness Job" - - - it: retry loop bounds default when blanked so a wiped override still produces a working Job - template: templates/webhook-ready-hook.yaml - set: - webhookReady: - image: - repository: docker.io/clastix/kubectl - tag: v1.32 - maxAttempts: null - sleepSeconds: null - asserts: - - documentIndex: 3 - matchRegex: - path: spec.template.spec.containers[0].command[2] - pattern: "max_attempts=60" - - documentIndex: 3 - matchRegex: - path: spec.template.spec.containers[0].command[2] - pattern: "sleep_seconds=2" - - - it: Job pod runs non-root with seccomp RuntimeDefault for restricted-PSA clusters - template: templates/webhook-ready-hook.yaml - asserts: - - documentIndex: 3 - equal: - path: spec.template.spec.securityContext.runAsNonRoot - value: true - - documentIndex: 3 - equal: - path: spec.template.spec.securityContext.seccompProfile.type - value: RuntimeDefault - - - it: wait container declares resource requests and limits for predictable scheduling - template: templates/webhook-ready-hook.yaml - asserts: - - documentIndex: 3 - equal: - path: spec.template.spec.containers[0].resources.requests.cpu - value: 10m - - documentIndex: 3 - equal: - path: spec.template.spec.containers[0].resources.requests.memory - value: 32Mi - - documentIndex: 3 - equal: - path: spec.template.spec.containers[0].resources.limits.cpu - value: 100m - - documentIndex: 3 - equal: - path: spec.template.spec.containers[0].resources.limits.memory - value: 64Mi - - - it: Job container drops all capabilities and runs read-only rootfs - template: templates/webhook-ready-hook.yaml - asserts: - - documentIndex: 3 - equal: - path: spec.template.spec.containers[0].securityContext.allowPrivilegeEscalation - value: false - - documentIndex: 3 - equal: - path: spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem - value: true - - documentIndex: 3 - equal: - path: spec.template.spec.containers[0].securityContext.capabilities.drop[0] - value: ALL - - - it: imagePullPolicy is IfNotPresent when digest-pinned, Always when tag-only - template: templates/webhook-ready-hook.yaml - set: - webhookReady: - image: - repository: docker.io/clastix/kubectl - tag: v1.32 - digest: sha256:b9ef7d8dbe65bcc81a46c09b8dc7543103055021c4f43287bf59e92a8f4fe05c - asserts: - - documentIndex: 3 - equal: - path: spec.template.spec.containers[0].imagePullPolicy - value: IfNotPresent - - - it: imagePullPolicy is Always when no digest is configured - template: templates/webhook-ready-hook.yaml - set: - webhookReady: - image: - repository: docker.io/clastix/kubectl - tag: v1.32 - digest: "" - asserts: - - documentIndex: 3 - equal: - path: spec.template.spec.containers[0].imagePullPolicy - value: Always - - - it: retry loop captures and surfaces the last kubectl error message on timeout - template: templates/webhook-ready-hook.yaml - asserts: - - documentIndex: 3 - matchRegex: - path: spec.template.spec.containers[0].command[2] - pattern: 'last_err=\$\(kubectl get --raw' - - documentIndex: 3 - matchRegex: - path: spec.template.spec.containers[0].command[2] - pattern: 'last error: \$\{last_err\}' - - - it: retry loop error message stays in sync when maxAttempts is bumped - template: templates/webhook-ready-hook.yaml - set: - webhookReady: - image: - repository: docker.io/clastix/kubectl - tag: v1.32 - maxAttempts: 90 - sleepSeconds: 3 - asserts: - - documentIndex: 3 - matchRegex: - path: spec.template.spec.containers[0].command[2] - pattern: "max_attempts=90" - - documentIndex: 3 - matchRegex: - path: spec.template.spec.containers[0].command[2] - pattern: "sleep_seconds=3" - - documentIndex: 3 - matchRegex: - path: spec.template.spec.containers[0].command[2] - pattern: 'after \$\{max_attempts\} attempts' - - - it: activeDeadlineSeconds scales with retry bounds so an override raise does not silently cut - template: templates/webhook-ready-hook.yaml - set: - webhookReady: - image: - repository: docker.io/clastix/kubectl - tag: v1.32 - maxAttempts: 180 - sleepSeconds: 3 - asserts: - - documentIndex: 3 - equal: - path: spec.activeDeadlineSeconds - value: 600 - - - it: activeDeadlineSeconds defaults include 60s slack over the default retry window - template: templates/webhook-ready-hook.yaml - asserts: - - documentIndex: 3 - equal: - path: spec.activeDeadlineSeconds - value: 180 diff --git a/packages/system/postgres-operator/values.yaml b/packages/system/postgres-operator/values.yaml index c408220e..6bc9595b 100644 --- a/packages/system/postgres-operator/values.yaml +++ b/packages/system/postgres-operator/values.yaml @@ -1,32 +1,3 @@ cloudnative-pg: crds: create: true - image: - tag: "1.27.3" -# Image used by the post-install webhook-readiness Job (see templates/webhook-ready-hook.yaml). -# Any image with kubectl on PATH works; the Job calls the apiserver service proxy with the -# hook ServiceAccount's token to confirm the mcluster.cnpg.io webhook is reachable end-to-end -# before the HelmRelease reports Ready, closing the "connection refused" bootstrap race. -# -# The tag is digest-pinned so an upstream retag does not change what runs on every install -# and upgrade across the fleet. Refresh by resolving the current manifest-list digest -# (`docker manifest inspect docker.io/clastix/kubectl:v1.32`) and updating `digest` below. -# renovate: datasource=docker depName=docker.io/clastix/kubectl -webhookReady: - image: - repository: docker.io/clastix/kubectl - tag: v1.32 - digest: sha256:b9ef7d8dbe65bcc81a46c09b8dc7543103055021c4f43287bf59e92a8f4fe05c - # Retry loop bounds for the readiness probe. Defaults total ~120s wall clock. - # Both are `default`-coerced in the template so an override that blanks them still - # produces a working Job. - maxAttempts: 60 - sleepSeconds: 2 - # Pod-level retry budget for transient node/registry failures (image pull rate limit, - # OOM, CNI hiccup). The shell loop inside the container only covers reachability retries. - # Pod retries and activeDeadlineSeconds (wall-clock bound on the whole Job across all - # pod retries) are ANDed, so activeDeadlineSeconds is the shorter of the two gates with - # the defaults above: the 60*2+60 = 180s deadline cuts the Job before backoffLimit=2 - # ever matters if pod-level failures eat more than ~60s of the budget. Raise - # maxAttempts/sleepSeconds alongside backoffLimit when tuning for slow image pulls. - backoffLimit: 2 diff --git a/packages/system/postgres-rd/cozyrds/postgres.yaml b/packages/system/postgres-rd/cozyrds/postgres.yaml index 72d013f1..c74c5783 100644 --- a/packages/system/postgres-rd/cozyrds/postgres.yaml +++ b/packages/system/postgres-rd/cozyrds/postgres.yaml @@ -8,7 +8,7 @@ spec: singular: postgres plural: postgreses openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"replicas":{"description":"Number of Postgres replicas.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration for each PostgreSQL replica. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume Claim size available for application data.","default":"10Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":""},"external":{"description":"Enable external access from outside the cluster.","type":"boolean","default":false},"version":{"description":"PostgreSQL major version to deploy","type":"string","default":"v18","enum":["v18","v17","v16","v15","v14","v13"]},"postgresql":{"description":"PostgreSQL server configuration.","type":"object","default":{},"properties":{"parameters":{"description":"PostgreSQL server parameters.","type":"object","default":{},"properties":{"max_connections":{"description":"Maximum number of concurrent connections to the database server.","type":"integer","default":100}}}}},"quorum":{"description":"Quorum configuration for synchronous replication.","type":"object","default":{},"required":["maxSyncReplicas","minSyncReplicas"],"properties":{"maxSyncReplicas":{"description":"Maximum number of synchronous replicas allowed (must be less than total replicas).","type":"integer","default":0},"minSyncReplicas":{"description":"Minimum number of synchronous replicas required for commit.","type":"integer","default":0}}},"users":{"description":"Users configuration map.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"password":{"description":"Password for the user.","type":"string"},"replication":{"description":"Whether the user has replication privileges.","type":"boolean"}}}},"databases":{"description":"Databases configuration map.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"extensions":{"description":"List of enabled PostgreSQL extensions.","type":"array","items":{"type":"string"}},"roles":{"description":"Roles assigned to users.","type":"object","properties":{"admin":{"description":"List of users with admin privileges.","type":"array","items":{"type":"string"}},"readonly":{"description":"List of users with read-only privileges.","type":"array","items":{"type":"string"}}}}}}},"backup":{"description":"Backup configuration.","type":"object","default":{},"required":["enabled"],"properties":{"destinationPath":{"description":"Destination path for backups (e.g. s3://bucket/path/).","type":"string","default":"s3://bucket/path/to/folder/"},"enabled":{"description":"Enable regular backups.","type":"boolean","default":false},"endpointURL":{"description":"S3 endpoint URL for uploads.","type":"string","default":"http://minio-gateway-service:9000"},"retentionPolicy":{"description":"Retention policy (e.g. \"30d\").","type":"string","default":"30d"},"s3AccessKey":{"description":"Access key for S3 authentication.","type":"string","default":""},"s3SecretKey":{"description":"Secret key for S3 authentication.","type":"string","default":""},"schedule":{"description":"Cron schedule for automated backups.","type":"string","default":"0 2 * * * *"}}},"bootstrap":{"description":"Bootstrap configuration.","type":"object","default":{},"required":["enabled","oldName"],"properties":{"enabled":{"description":"Whether to restore from a backup.","type":"boolean","default":false},"oldName":{"description":"Previous cluster name before deletion.","type":"string","default":""},"recoveryTime":{"description":"Timestamp (RFC3339) for point-in-time recovery; empty means latest.","type":"string","default":""},"serverName":{"description":"Barman server name (S3 path prefix) used by the original cluster when writing backups. Set this only when the original cluster had an explicit barmanObjectStore.serverName that differed from its Kubernetes resource name.","type":"string","default":""}}}}} + {"title":"Chart Values","type":"object","properties":{"replicas":{"description":"Number of Postgres replicas.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration for each PostgreSQL replica. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"micro","enum":["nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume Claim size available for application data.","default":"10Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":""},"external":{"description":"Enable external access from outside the cluster.","type":"boolean","default":false},"version":{"description":"PostgreSQL major version to deploy","type":"string","default":"v18","enum":["v18","v17","v16","v15","v14","v13"]},"postgresql":{"description":"PostgreSQL server configuration.","type":"object","default":{},"properties":{"parameters":{"description":"PostgreSQL server parameters.","type":"object","default":{},"properties":{"max_connections":{"description":"Maximum number of concurrent connections to the database server.","type":"integer","default":100}}}}},"quorum":{"description":"Quorum configuration for synchronous replication.","type":"object","default":{},"required":["maxSyncReplicas","minSyncReplicas"],"properties":{"maxSyncReplicas":{"description":"Maximum number of synchronous replicas allowed (must be less than total replicas).","type":"integer","default":0},"minSyncReplicas":{"description":"Minimum number of synchronous replicas required for commit.","type":"integer","default":0}}},"users":{"description":"Users configuration map.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"password":{"description":"Password for the user.","type":"string"},"replication":{"description":"Whether the user has replication privileges.","type":"boolean"}}}},"databases":{"description":"Databases configuration map.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"extensions":{"description":"List of enabled PostgreSQL extensions.","type":"array","items":{"type":"string"}},"roles":{"description":"Roles assigned to users.","type":"object","properties":{"admin":{"description":"List of users with admin privileges.","type":"array","items":{"type":"string"}},"readonly":{"description":"List of users with read-only privileges.","type":"array","items":{"type":"string"}}}}}}},"backup":{"description":"Backup configuration.","type":"object","default":{},"required":["enabled"],"properties":{"destinationPath":{"description":"Destination path for backups (e.g. s3://bucket/path/).","type":"string","default":"s3://bucket/path/to/folder/"},"enabled":{"description":"Enable regular backups.","type":"boolean","default":false},"endpointURL":{"description":"S3 endpoint URL for uploads.","type":"string","default":"http://minio-gateway-service:9000"},"retentionPolicy":{"description":"Retention policy (e.g. \"30d\").","type":"string","default":"30d"},"s3AccessKey":{"description":"Access key for S3 authentication.","type":"string","default":""},"s3SecretKey":{"description":"Secret key for S3 authentication.","type":"string","default":""},"schedule":{"description":"Cron schedule for automated backups.","type":"string","default":"0 2 * * * *"}}},"bootstrap":{"description":"Bootstrap configuration.","type":"object","default":{},"required":["enabled","oldName"],"properties":{"enabled":{"description":"Whether to restore from a backup.","type":"boolean","default":false},"oldName":{"description":"Previous cluster name before deletion.","type":"string","default":""},"recoveryTime":{"description":"Timestamp (RFC3339) for point-in-time recovery; empty means latest.","type":"string","default":""}}}}} release: prefix: postgres- labels: @@ -33,7 +33,7 @@ spec: # labelSelector: # helm.toolkit.fluxcd.io/name: "{reqs[0]['metadata','name']}" - keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "replicas"], ["spec", "resources"], ["spec", "resourcesPreset"], ["spec", "size"], ["spec", "storageClass"], ["spec", "external"], ["spec", "version"], ["spec", "postgresql"], ["spec", "postgresql", "parameters"], ["spec", "postgresql", "parameters", "max_connections"], ["spec", "quorum"], ["spec", "quorum", "minSyncReplicas"], ["spec", "quorum", "maxSyncReplicas"], ["spec", "users"], ["spec", "databases"], ["spec", "backup"], ["spec", "backup", "enabled"], ["spec", "backup", "retentionPolicy"], ["spec", "backup", "destinationPath"], ["spec", "backup", "endpointURL"], ["spec", "backup", "schedule"], ["spec", "backup", "s3AccessKey"], ["spec", "backup", "s3SecretKey"], ["spec", "bootstrap"], ["spec", "bootstrap", "enabled"], ["spec", "bootstrap", "recoveryTime"], ["spec", "bootstrap", "oldName"], ["spec", "bootstrap", "serverName"]] + keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "replicas"], ["spec", "resources"], ["spec", "resourcesPreset"], ["spec", "size"], ["spec", "storageClass"], ["spec", "external"], ["spec", "version"], ["spec", "postgresql"], ["spec", "postgresql", "parameters"], ["spec", "postgresql", "parameters", "max_connections"], ["spec", "quorum"], ["spec", "quorum", "minSyncReplicas"], ["spec", "quorum", "maxSyncReplicas"], ["spec", "users"], ["spec", "databases"], ["spec", "backup"], ["spec", "backup", "enabled"], ["spec", "backup", "retentionPolicy"], ["spec", "backup", "destinationPath"], ["spec", "backup", "endpointURL"], ["spec", "backup", "schedule"], ["spec", "backup", "s3AccessKey"], ["spec", "backup", "s3SecretKey"], ["spec", "bootstrap"], ["spec", "bootstrap", "enabled"], ["spec", "bootstrap", "recoveryTime"], ["spec", "bootstrap", "oldName"]] secrets: exclude: [] include: diff --git a/packages/system/seaweedfs/templates/database.yaml b/packages/system/seaweedfs/templates/database.yaml index b9b50c49..1c116dd0 100644 --- a/packages/system/seaweedfs/templates/database.yaml +++ b/packages/system/seaweedfs/templates/database.yaml @@ -5,9 +5,7 @@ metadata: name: seaweedfs-db spec: instances: {{ .Values.db.replicas }} - {{- $existingCluster := lookup "postgresql.cnpg.io/v1" "Cluster" .Release.Namespace "seaweedfs-db" }} - {{- $image := dig "spec" "imageName" "ghcr.io/cloudnative-pg/postgresql:17.7-standard-trixie" $existingCluster }} - imageName: {{ if regexMatch ":17\\." $image }}ghcr.io/cloudnative-pg/postgresql:17.7-standard-trixie{{ else if regexMatch ":[0-9]+\\.[0-9]+$" $image }}{{ printf "%s-standard-trixie" $image }}{{ else }}{{ $image }}{{ end }} + imageName: ghcr.io/cloudnative-pg/postgresql:17.7 storage: size: {{ .Values.db.size }} {{- with .Values.db.storageClass }} diff --git a/packages/system/seaweedfs/values.yaml b/packages/system/seaweedfs/values.yaml index 3542a77f..7d598649 100644 --- a/packages/system/seaweedfs/values.yaml +++ b/packages/system/seaweedfs/values.yaml @@ -111,7 +111,7 @@ seaweedfs: nginx.ingress.kubernetes.io/client-body-timeout: "3600" nginx.ingress.kubernetes.io/client-header-timeout: "120" nginx.ingress.kubernetes.io/service-upstream: "true" - acme.cert-manager.io/http01-ingress-ingressclassname: tenant-root + acme.cert-manager.io/http01-ingress-class: tenant-root cert-manager.io/cluster-issuer: letsencrypt-prod tls: - hosts: @@ -177,7 +177,7 @@ seaweedfs: bucketClassName: "seaweedfs" region: "" sidecar: - image: "ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.3.0@sha256:5bcccbdb13979a16cee535eb5fbcdf0d87973689010a10b7b25e55c5be3edaa6" + image: "ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.2.1@sha256:1ad07fb9e96477e5c322f240f6023ceedd8762261d8379ec784dde92797ee206" certificates: commonName: "SeaweedFS CA" ipAddresses: [] diff --git a/packages/system/velero/templates/backupstrategy-controller-rbac.yaml b/packages/system/velero/templates/backupstrategy-controller-rbac.yaml deleted file mode 100644 index 43c51cd7..00000000 --- a/packages/system/velero/templates/backupstrategy-controller-rbac.yaml +++ /dev/null @@ -1,26 +0,0 @@ -# Grants the backupstrategy-controller permission to manage ResourceModifier -# ConfigMaps in the Velero install namespace. Lives here (not in the -# backupstrategy-controller chart) so that the Role is only created when -# velero is actually installed — otherwise the chart would try to create it -# in a namespace that does not exist. -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: backups.cozystack.io:strategy-controller:velero-configmaps -rules: -- apiGroups: [""] - resources: ["configmaps"] - verbs: ["create", "get", "list", "delete", "deletecollection", "patch", "update"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: backups.cozystack.io:strategy-controller:velero-configmaps -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: backups.cozystack.io:strategy-controller:velero-configmaps -subjects: -- kind: ServiceAccount - name: backupstrategy-controller - namespace: cozy-backup-controller diff --git a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/Chart.lock b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/Chart.lock index 2868e66f..1b0e2a46 100644 --- a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/Chart.lock +++ b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/Chart.lock @@ -1,9 +1,9 @@ dependencies: - name: victoria-metrics-common - repository: oci://ghcr.io/victoriametrics/helm-charts - version: 0.3.0 + repository: https://victoriametrics.github.io/helm-charts + version: 0.0.46 - name: crds repository: "" version: 0.0.* -digest: sha256:adec954bf2814f744f5b69fcf67c8b246bf21f416adb0ab37d91c9b42f72d353 -generated: "2026-04-16T10:14:10.795552631Z" +digest: sha256:43d3d210a9d1a2234e6c56518f8c477125a5ad5e8ed08d46209528f19acd9c89 +generated: "2025-12-23T12:58:01.428960334Z" diff --git a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/Chart.yaml b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/Chart.yaml index 030af8f0..bb9253b1 100644 --- a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/Chart.yaml +++ b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/Chart.yaml @@ -1,7 +1,7 @@ annotations: artifacthub.io/category: monitoring-logging artifacthub.io/changes: | - - revert change in Deployment's matchLabels, that was introduced in release 0.60.0 + - updates operator to [v0.68.1](https://github.com/VictoriaMetrics/operator/releases/tag/v0.68.1) version artifacthub.io/license: Apache-2.0 artifacthub.io/links: | - name: Sources @@ -16,14 +16,13 @@ annotations: artifacthub.io/readme: | # VictoriaMetrics Operator Helm chart - Chart documentation is available [here](https://docs.victoriametrics.com/helm/victoria-metrics-operator/). - Changelog is [here](https://docs.victoriametrics.com/helm/victoria-metrics-operator/changelog/). + Chart documentation is available [here](https://docs.victoriametrics.com/helm/victoria-metrics-operator/) apiVersion: v2 -appVersion: v0.68.4 +appVersion: v0.68.1 dependencies: - name: victoria-metrics-common - repository: oci://ghcr.io/victoriametrics/helm-charts - version: 0.3.* + repository: https://victoriametrics.github.io/helm-charts + version: 0.0.* - condition: crds.plain name: crds repository: "" @@ -47,4 +46,4 @@ sources: - https://github.com/VictoriaMetrics/helm-charts - https://github.com/VictoriaMetrics/operator type: application -version: 0.61.0 +version: 0.59.1 diff --git a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/README.md b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/README.md index 6a080424..49a06de2 100644 --- a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/README.md +++ b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/README.md @@ -1,4 +1,3 @@ # VictoriaMetrics Operator Helm chart -Chart documentation is available [here](https://docs.victoriametrics.com/helm/victoria-metrics-operator/). -Changelog is [here](https://docs.victoriametrics.com/helm/victoria-metrics-operator/changelog/). +Chart documentation is available [here](https://docs.victoriametrics.com/helm/victoria-metrics-operator/) diff --git a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/RELEASE_NOTES b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/RELEASE_NOTES index 3529783a..bf22fd3f 100644 --- a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/RELEASE_NOTES +++ b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/RELEASE_NOTES @@ -1,7 +1,7 @@ -# Release notes for version 0.61.0 +# Release notes for version 0.59.1 -**Release date:** 16 Apr 2026 +**Release date:** 03 Mar 2026 -![Helm: v3](https://img.shields.io/badge/Helm-v3.14%2B-informational?color=informational&logo=helm&link=https%3A%2F%2Fgithub.com%2Fhelm%2Fhelm%2Freleases%2Ftag%2Fv3.14.0) ![AppVersion: v0.68.4](https://img.shields.io/badge/v0.68.4-success?logo=VictoriaMetrics&labelColor=gray&link=https%3A%2F%2Fdocs.victoriametrics.com%2Foperator%2Fchangelog%2F%23v0684) +![Helm: v3](https://img.shields.io/badge/Helm-v3.14%2B-informational?color=informational&logo=helm&link=https%3A%2F%2Fgithub.com%2Fhelm%2Fhelm%2Freleases%2Ftag%2Fv3.14.0) ![AppVersion: v0.68.1](https://img.shields.io/badge/v0.68.1-success?logo=VictoriaMetrics&labelColor=gray&link=https%3A%2F%2Fdocs.victoriametrics.com%2Foperator%2Fchangelog%2F%23v0681) -- revert change in Deployment's matchLabels, that was introduced in release 0.60.0 +- updates operator to [v0.68.1](https://github.com/VictoriaMetrics/operator/releases/tag/v0.68.1) version diff --git a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/crds/crds/crd.yaml b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/crds/crds/crd.yaml index bdf7e8d0..c4138be8 100644 --- a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/crds/crds/crd.yaml +++ b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/crds/crds/crd.yaml @@ -6079,7 +6079,6 @@ spec: - spec type: object shardCount: - format: int32 type: integer startupProbe: type: object @@ -6088,14 +6087,6 @@ spec: type: boolean statefulRollingUpdateStrategy: type: string - statefulRollingUpdateStrategyBehavior: - properties: - maxUnavailable: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - type: object statefulStorage: properties: emptyDir: @@ -12335,7 +12326,6 @@ spec: type: object x-kubernetes-preserve-unknown-fields: true shardCount: - format: int32 type: integer startupProbe: type: object @@ -16818,9 +16808,6 @@ spec: type: boolean vmauth: properties: - enabled: - default: true - type: boolean name: type: string spec: @@ -24740,14 +24727,6 @@ spec: type: boolean statefulRollingUpdateStrategy: type: string - statefulRollingUpdateStrategyBehavior: - properties: - maxUnavailable: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - type: object statefulStorage: properties: emptyDir: @@ -28649,14 +28628,6 @@ spec: type: boolean statefulRollingUpdateStrategy: type: string - statefulRollingUpdateStrategyBehavior: - properties: - maxUnavailable: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - type: object statefulStorage: properties: emptyDir: diff --git a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/crds/files/crd.yaml.bz2 b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/crds/files/crd.yaml.bz2 index ab7c4d95..7f518a91 100644 Binary files a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/crds/files/crd.yaml.bz2 and b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/crds/files/crd.yaml.bz2 differ diff --git a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/Chart.yaml b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/Chart.yaml index e8f67b95..04aba91f 100644 --- a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/Chart.yaml +++ b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/Chart.yaml @@ -1,7 +1,7 @@ annotations: artifacthub.io/category: monitoring-logging artifacthub.io/changes: | - - reverted usage of `app` label in `vm.selectorLabels` + - allow setting global labels for each resource using `.Values.global.extraLabels`. See [#2607](https://github.com/VictoriaMetrics/helm-charts/issues/2607). artifacthub.io/license: Apache-2.0 artifacthub.io/links: | - name: Sources @@ -11,8 +11,7 @@ annotations: artifacthub.io/readme: | # VictoriaMetrics Common Helm chart - Chart documentation is available [here](https://docs.victoriametrics.com/helm/victoria-metrics-common/). - Changelog is [here](https://docs.victoriametrics.com/helm/victoria-metrics-common/changelog/). + Chart documentation is available [here](https://docs.victoriametrics.com/helm/victoria-metrics-common/) apiVersion: v2 description: VictoriaMetrics Common - contains shared templates for all Victoria Metrics helm charts @@ -30,4 +29,4 @@ name: victoria-metrics-common sources: - https://github.com/VictoriaMetrics/helm-charts type: library -version: 0.3.0 +version: 0.0.46 diff --git a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/README.md b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/README.md index c8ee7051..9ba0b4f5 100644 --- a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/README.md +++ b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/README.md @@ -1,4 +1,3 @@ # VictoriaMetrics Common Helm chart -Chart documentation is available [here](https://docs.victoriametrics.com/helm/victoria-metrics-common/). -Changelog is [here](https://docs.victoriametrics.com/helm/victoria-metrics-common/changelog/). +Chart documentation is available [here](https://docs.victoriametrics.com/helm/victoria-metrics-common/) diff --git a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/RELEASE_NOTES b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/RELEASE_NOTES index ad0cdfa9..055cd4c5 100644 --- a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/RELEASE_NOTES +++ b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/RELEASE_NOTES @@ -1,7 +1,7 @@ -# Release notes for version 0.3.0 +# Release notes for version 0.0.46 -**Release date:** 16 Apr 2026 +**Release date:** 23 Dec 2025 ![Helm: v3](https://img.shields.io/badge/Helm-v3.14%2B-informational?color=informational&logo=helm&link=https%3A%2F%2Fgithub.com%2Fhelm%2Fhelm%2Freleases%2Ftag%2Fv3.14.0) -- reverted usage of `app` label in `vm.selectorLabels` +- allow setting global labels for each resource using `.Values.global.extraLabels`. See [#2607](https://github.com/VictoriaMetrics/helm-charts/issues/2607). diff --git a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/templates/_helpers.tpl b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/templates/_helpers.tpl index bd945678..99ac30d5 100644 --- a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/templates/_helpers.tpl +++ b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/templates/_helpers.tpl @@ -125,8 +125,8 @@ If release name contains chart name it will be used as a full name. {{- $ctx := . -}} {{- $values := $Values -}} {{- range $ak := $appKey }} - {{- $values = ternary (dict) (index $values $ak | default dict) (empty $values) -}} - {{- $ctx = ternary (dict) (index $ctx $ak | default dict) (empty $ctx) -}} + {{- $values = ternary (default dict) (index $values $ak | default dict) (empty $values) -}} + {{- $ctx = ternary (default dict) (index $ctx $ak | default dict) (empty $ctx) -}} {{- if and (empty $values) (empty $ctx) -}} {{- fail (printf "No data for appKey %s" (join "->" $appKey)) -}} {{- end -}} @@ -187,9 +187,6 @@ If release name contains chart name it will be used as a full name. {{- include "vm.validate.args" . -}} {{- $Release := (.helm).Release | default .Release -}} {{- $labels := fromYaml (include "vm.selectorLabels" .) -}} - {{- with $labels.app -}} - {{- $_ := set $labels "app.kubernetes.io/component" . -}} - {{- end -}} {{- $labels = mergeOverwrite $labels (.extraLabels | default dict) -}} {{- $_ := set $labels "app.kubernetes.io/managed-by" $Release.Service -}} {{- toYaml $labels -}} @@ -200,7 +197,7 @@ If release name contains chart name it will be used as a full name. {{- include "vm.validate.args" . -}} {{- $Values := (.helm).Values | default .Values -}} {{- $globalLabels := deepCopy (($Values.global).extraLabels | default dict) -}} - {{- $labels := fromYaml (include "vm.commonLabels" .) -}} + {{- $labels := fromYaml (include "vm.selectorLabels" .) -}} {{- $labels = mergeOverwrite $globalLabels $labels (fromYaml (include "vm.metaLabels" .)) -}} {{- with (include "vm.image.tag" .) }} {{- $_ := set $labels "app.kubernetes.io/version" (regexReplaceAll "(.*)(@sha.*)" . "${1}") -}} @@ -234,16 +231,11 @@ If release name contains chart name it will be used as a full name. {{- $_ := set $labels "app.kubernetes.io/name" (include "vm.name" .) -}} {{- $_ := set $labels "app.kubernetes.io/instance" (include "vm.release" .) -}} {{- with (include "vm.app.name" .) -}} - {{- $_ := set $labels "app" . -}} + {{- if eq $.style "managed" -}} + {{- $_ := set $labels "app.kubernetes.io/component" (printf "%s-%s" (include "vm.name" $) .) -}} + {{- else -}} + {{- $_ := set $labels "app" . -}} + {{- end -}} {{- end -}} {{- toYaml $labels -}} {{- end }} - -{{- define "vm.commonLabels" -}} - {{- $labels := fromYaml (include "vm.selectorLabels" . ) -}} - {{- with $labels.app -}} - {{- $_ := set $labels "app.kubernetes.io/component" . -}} - {{- $_ := unset $labels "app" -}} - {{- end -}} - {{- toYaml $labels -}} -{{- end -}} diff --git a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/templates/_image.tpl b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/templates/_image.tpl index 66d019e3..6d30bd03 100644 --- a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/templates/_image.tpl +++ b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/templates/_image.tpl @@ -42,8 +42,8 @@ Victoria Metrics Image {{- with .appKey -}} {{- $appKey := ternary (list .) . (kindIs "string" .) -}} {{- range $ak := $appKey -}} - {{- $values = ternary (dict) (index $values $ak | default dict) (empty $values) -}} - {{- $ctx = ternary (dict) (index $ctx $ak | default dict) (empty $ctx) -}} + {{- $values = ternary (default dict) (index $values $ak | default dict) (empty $values) -}} + {{- $ctx = ternary (default dict) (index $ctx $ak | default dict) (empty $ctx) -}} {{- if and (empty $values) (empty $ctx) -}} {{- fail (printf "No data for appKey %s" (join "->" $appKey)) -}} {{- end -}} diff --git a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/templates/_pod.tpl b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/templates/_pod.tpl index 62981dfb..7534ae2d 100644 --- a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/templates/_pod.tpl +++ b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/templates/_pod.tpl @@ -39,9 +39,9 @@ Render probe {{- define "vm.probe" -}} {{- /* undefined value */ -}} {{- $null := (fromYaml "value: null").value -}} - {{- $probe := dig .type (dict) .app.probe -}} + {{- $probe := dig .type (default dict) .app.probe -}} {{- $probeType := "" -}} - {{- $defaultProbe := dict -}} + {{- $defaultProbe := default dict -}} {{- if ne (dig "httpGet" $null $probe) $null -}} {{- /* httpGet probe */ -}} {{- $defaultProbe = dict "path" (include "vm.probe.http.path" .) "scheme" (include "vm.probe.http.scheme" .) "port" (include "vm.probe.port" .) -}} @@ -51,7 +51,7 @@ Render probe {{- $defaultProbe = dict "port" (include "vm.probe.port" .) -}} {{- $probeType = "tcpSocket" -}} {{- end -}} - {{- $defaultProbe = ternary (dict) (dict $probeType $defaultProbe) (empty $probeType) -}} + {{- $defaultProbe = ternary (default dict) (dict $probeType $defaultProbe) (empty $probeType) -}} {{- $probe = mergeOverwrite $defaultProbe $probe -}} {{- range $key, $value := $probe -}} {{- if and (has (kindOf $value) (list "object" "map")) (ne $key $probeType) -}} @@ -100,7 +100,7 @@ Net probe port command line arguments */ -}} {{- define "vm.args" -}} - {{- $args := list -}} + {{- $args := default list -}} {{- range $key, $value := . -}} {{- if not $key -}} {{- fail "Empty key in command line args is not allowed" -}} diff --git a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/templates/_service.tpl b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/templates/_service.tpl index 77cdeee7..77a1365f 100644 --- a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/templates/_service.tpl +++ b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/charts/victoria-metrics-common/templates/_service.tpl @@ -37,10 +37,10 @@ {{- $values := $Values -}} {{- $ctx := . -}} {{- range $ak := $appKey -}} - {{- $values = ternary (dict) (index $values $ak | default dict) (empty $values) -}} - {{- $ctx = ternary (dict) (index $ctx $ak | default dict) (empty $ctx) -}} + {{- $values = ternary (default dict) (index $values $ak | default dict) (empty $values) -}} + {{- $ctx = ternary (default dict) (index $ctx $ak | default dict) (empty $ctx) -}} {{- end -}} - {{- $spec := dict -}} + {{- $spec := default dict -}} {{- if $ctx -}} {{- $spec = $ctx -}} {{- else if $values -}} @@ -69,10 +69,10 @@ {{- $values := $Values -}} {{- $ctx := . -}} {{- range $ak := $appKey -}} - {{- $values = ternary (dict) (index $values $ak | default dict) (empty $values) -}} - {{- $ctx = ternary (dict) (index $ctx $ak | default dict) (empty $ctx) -}} + {{- $values = ternary (default dict) (index $values $ak | default dict) (empty $values) -}} + {{- $ctx = ternary (default dict) (index $ctx $ak | default dict) (empty $ctx) -}} {{- end -}} - {{- $spec := dict -}} + {{- $spec := default dict -}} {{- if $values -}} {{- $spec = $values -}} {{- else if $ctx -}} diff --git a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/templates/NOTES.txt b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/templates/NOTES.txt index 41f39937..7fb3fbd9 100644 --- a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/templates/NOTES.txt +++ b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/templates/NOTES.txt @@ -1,5 +1,5 @@ {{ include "vm.name" . }} has been installed. Check its status by running: - kubectl --namespace {{ include "vm.namespace" . }} get pods -l "app.kubernetes.io/name={{ include "vm.name" . }}" -l "app.kubernetes.io/instance={{ include "vm.release" . }}" + kubectl --namespace {{ include "vm.namespace" . }} get pods -l "app.kubernetes.io/instance={{ $.Release.Name }}" Get more information on https://github.com/VictoriaMetrics/helm-charts/tree/master/charts/victoria-metrics-operator. See "Getting started guide for VM Operator" on https://docs.victoriametrics.com/guides/getting-started-with-vm-operator diff --git a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/templates/pdb.yaml b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/templates/pdb.yaml index 9ec3c2c2..8abd9dea 100644 --- a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/templates/pdb.yaml +++ b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/templates/pdb.yaml @@ -18,9 +18,6 @@ spec: {{- with $pdb.maxUnavailable }} maxUnavailable: {{ . }} {{- end }} - {{- with $pdb.unhealthyPodEvictionPolicy }} - unhealthyPodEvictionPolicy: {{ . }} - {{- end }} selector: matchLabels: {{ include "vm.selectorLabels" $ctx | nindent 6 }} {{- end }} diff --git a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/templates/server.yaml b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/templates/server.yaml index 9d7fd7ed..e224c074 100644 --- a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/templates/server.yaml +++ b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/templates/server.yaml @@ -18,12 +18,9 @@ metadata: annotations: {{ toYaml . | nindent 4 }} {{- end }} spec: - replicas: {{ .Values.replicaCount }} + replicas: {{.Values.replicaCount }} selector: matchLabels: {{ include "vm.selectorLabels" $ctx | nindent 6 }} - {{- with .Values.strategy }} - strategy: {{ toYaml . | nindent 4 }} - {{- end }} template: metadata: {{- with .Values.annotations }} diff --git a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/templates/webhook.yaml b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/templates/webhook.yaml index e38f7448..51039cae 100644 --- a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/templates/webhook.yaml +++ b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/templates/webhook.yaml @@ -1,31 +1,22 @@ -{{- $webhooks := .Values.admissionWebhooks }} -{{- if $webhooks.enabled }} +{{- if .Values.admissionWebhooks.enabled }} {{- $ctx := dict "helm" . "extraLabels" .Values.extraLabels }} {{- $tls := fromYaml (include "vm-operator.certs" $ctx) }} {{- $fullname := include "vm.plain.fullname" $ctx }} {{- $domain := ((.Values.global).cluster).dnsDomain }} {{- $ns := include "vm.namespace" $ctx }} -{{- $certManager := $webhooks.certManager }} +{{- $certManager := .Values.admissionWebhooks.certManager }} {{- $files := .Files }} {{- $crds := $files.Get "crd.yaml" | splitList "---" }} -{{- $disabledFor := $webhooks.disabledFor | default list }} +{{- $disabledFor := .Values.admissionWebhooks.disabledFor | default list }} --- apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: name: {{ $fullname }}-admission - {{- if or $certManager.enabled $webhooks.annotations }} + {{- if $certManager.enabled }} annotations: - {{- if $certManager.enabled }} certmanager.k8s.io/inject-ca-from: {{ printf "%s/%s-validation" $ns $fullname | quote }} cert-manager.io/inject-ca-from: {{ printf "%s/%s-validation" $ns $fullname | quote }} - {{- end }} - {{- with $certManager.annotations }} - {{- toYaml . | nindent 4 }} - {{- end }} - {{- with $webhooks.annotations }} - {{- toYaml . | nindent 4 }} - {{- end }} {{- end }} labels: {{ include "vm.labels" $ctx | nindent 4 }} webhooks: @@ -43,7 +34,7 @@ webhooks: {{- if not $certManager.enabled }} caBundle: {{ $tls.caCert }} {{- end }} - failurePolicy: {{ $webhooks.policy }} + failurePolicy: {{ $.Values.admissionWebhooks.policy }} name: '{{ $crd.metadata.name }}' admissionReviewVersions: - v1 diff --git a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/values.yaml b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/values.yaml index 3a1027f2..279ceee9 100644 --- a/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/values.yaml +++ b/packages/system/victoria-metrics-operator/charts/victoria-metrics-operator/values.yaml @@ -125,7 +125,7 @@ crds: # whenUnsatisfiable: DoNotSchedule # labelSelector: # matchLabels: - # app.kubernetes.io/component: alertmanager + # app: alertmanager # -- Labels to add to the upgrade job labels: {} @@ -276,12 +276,8 @@ service: # -- See `kubectl explain poddisruptionbudget.spec` for more or check [these docs](https://kubernetes.io/docs/tasks/run-application/configure-pdb/) podDisruptionBudget: enabled: false - # -- min number or percentage of pods that can be unavailable - minAvailable: 0 - # -- max number or percentage of pods that can be unavailable - maxUnavailable: 0 - # -- Defines criteria when unhealthy pods should be considered for eviction - unhealthyPodEvictionPolicy: + # minAvailable: 1 + # maxUnavailable: 1 labels: {} # -- Graceful pod termination timeout. See [this article](https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#hook-handler-execution) for details. @@ -303,13 +299,6 @@ resources: # -- Pod's node selector. Details are [here](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) nodeSelector: {} -# -- Deployment strategy, set to standard k8s default -strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 25% - maxUnavailable: 25% - # -- Name of Priority Class priorityClassName: "" @@ -375,8 +364,6 @@ shareProcessNamespace: false admissionWebhooks: # -- Enables validation webhook. enabled: true - # -- Annotations for webhook. Can be used to define Helm or ArgoCD annotations. - annotations: {} # -- List of CRD names to disable validation for disabledFor: [] # - vmagent diff --git a/packages/system/vm-default-images/Chart.yaml b/packages/system/vm-default-images/Chart.yaml deleted file mode 100644 index aac0ccca..00000000 --- a/packages/system/vm-default-images/Chart.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v2 -name: vm-default-images -description: Global Golden Image collection for virtual machines -type: application -version: 0.0.0 # Placeholder, the actual version will be automatically set during the build process diff --git a/packages/system/vm-default-images/Makefile b/packages/system/vm-default-images/Makefile deleted file mode 100644 index a3f9fac7..00000000 --- a/packages/system/vm-default-images/Makefile +++ /dev/null @@ -1,4 +0,0 @@ -export NAME=vm-default-images -export NAMESPACE=cozy-system - -include ../../../hack/package.mk diff --git a/packages/system/vm-default-images/templates/dv.yaml b/packages/system/vm-default-images/templates/dv.yaml deleted file mode 100644 index 84b9a758..00000000 --- a/packages/system/vm-default-images/templates/dv.yaml +++ /dev/null @@ -1,45 +0,0 @@ -{{- range .Values.images }} ---- -apiVersion: cdi.kubevirt.io/v1beta1 -kind: DataVolume -metadata: - annotations: - vm-default-images.cozystack.io/name: {{ .name | quote }} - {{- with .os }} - {{- with .family }} - vm-default-images.cozystack.io/os-family: {{ . | quote }} - {{- end }} - {{- with .name }} - vm-default-images.cozystack.io/os-name: {{ . | quote }} - {{- end }} - {{- with .version }} - vm-default-images.cozystack.io/os-version: {{ . | quote }} - {{- end }} - {{- end }} - {{- with .architecture }} - vm-default-images.cozystack.io/architecture: {{ . | quote }} - {{- end }} - {{- with .description }} - vm-default-images.cozystack.io/description: {{ . | quote }} - {{- end }} - cdi.kubevirt.io/storage.bind.immediate.requested: "true" - cdi.kubevirt.io/storage.usePopulator: "true" - labels: - app.kubernetes.io/managed-by: cozystack - name: vm-default-images-{{ required "A valid .name entry required for each image!" .name }} - namespace: cozy-public -spec: - contentType: kubevirt - source: - http: - url: {{ required (printf "A valid .url entry required for image '%s'!" .name) .url | quote }} - storage: - resources: - requests: - storage: {{ default "5Gi" .storage }} - {{- if .storageClass }} - storageClassName: {{ .storageClass }} - {{- else if $.Values.storageClass }} - storageClassName: {{ $.Values.storageClass }} - {{- end }} -{{- end }} diff --git a/packages/system/vm-default-images/values.yaml b/packages/system/vm-default-images/values.yaml deleted file mode 100644 index 9d777f0b..00000000 --- a/packages/system/vm-default-images/values.yaml +++ /dev/null @@ -1,169 +0,0 @@ -## -## @section Global Golden Image Collection -## - -## @param {string} storageClass - Default StorageClass for all images. If empty, uses the cluster default StorageClass. -## NOTE: The default set of images requires approximately 320Gi of storage (16 images × 20Gi each). -## Adjust the image list or per-image storage sizes to match your cluster capacity. -storageClass: "replicated" - -## @typedef {struct} ImageOS - Operating system metadata for a Golden Image. -## @field {string} [family] - OS family (e.g. "Linux", "Windows"). -## @field {string} [name] - OS distribution name (e.g. "Ubuntu", "Fedora"). -## @field {string} [version] - OS version (e.g. "24.04", "40"). - -## @typedef {struct} GlobalImage - A global Golden Image entry. -## @field {string} name - Unique image name used to reference this image in vm-disk (e.g. "ubuntu"). -## @field {string} url - HTTP(S) URL to download the image from. -## @field {quantity} storage - Storage size to allocate for this image. -## @field {string} storageClass - StorageClass used to store the image (overrides global default). -## @field {ImageOS} [os] - Operating system metadata. -## @field {string} [architecture] - CPU architecture (e.g. "amd64", "arm64"). -## @field {string} [description] - Human-readable description. - -## @param {[]GlobalImage} images - List of global Golden Images to provision in the cozy-public namespace. -images: - - name: ubuntu-22.04 - url: https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64.img - storage: 20Gi - os: - family: Linux - name: Ubuntu - version: "22.04" - architecture: amd64 - description: "Ubuntu 22.04 LTS (Jammy Jellyfish) cloud image" - - name: ubuntu-24.04 - url: https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img - storage: 20Gi - os: - family: Linux - name: Ubuntu - version: "24.04" - architecture: amd64 - description: "Ubuntu 24.04 LTS (Noble Numbat) cloud image" - - name: rocky-8 - url: https://dl.rockylinux.org/pub/rocky/8/images/x86_64/Rocky-8-GenericCloud-Base.latest.x86_64.qcow2 - storage: 20Gi - os: - family: Linux - name: Rocky Linux - version: "8" - architecture: amd64 - description: "Rocky Linux 8 GenericCloud image" - - name: rocky-9 - url: https://dl.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud-Base.latest.x86_64.qcow2 - storage: 20Gi - os: - family: Linux - name: Rocky Linux - version: "9" - architecture: amd64 - description: "Rocky Linux 9 GenericCloud image" - - name: rocky-10 - url: https://dl.rockylinux.org/pub/rocky/10/images/x86_64/Rocky-10-GenericCloud-Base.latest.x86_64.qcow2 - storage: 20Gi - os: - family: Linux - name: Rocky Linux - version: "10" - architecture: amd64 - description: "Rocky Linux 10 GenericCloud image" - - name: almalinux-8 - url: https://repo.almalinux.org/almalinux/8/cloud/x86_64/images/AlmaLinux-8-GenericCloud-latest.x86_64.qcow2 - storage: 20Gi - os: - family: Linux - name: AlmaLinux - version: "8" - architecture: amd64 - description: "AlmaLinux 8 GenericCloud image" - - name: almalinux-9 - url: https://repo.almalinux.org/almalinux/9/cloud/x86_64/images/AlmaLinux-9-GenericCloud-latest.x86_64.qcow2 - storage: 20Gi - os: - family: Linux - name: AlmaLinux - version: "9" - architecture: amd64 - description: "AlmaLinux 9 GenericCloud image" - - name: almalinux-10 - url: https://repo.almalinux.org/almalinux/10/cloud/x86_64/images/AlmaLinux-10-GenericCloud-latest.x86_64.qcow2 - storage: 20Gi - os: - family: Linux - name: AlmaLinux - version: "10" - architecture: amd64 - description: "AlmaLinux 10 GenericCloud image" - - name: debian-12 - url: https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-generic-amd64.qcow2 - storage: 20Gi - os: - family: Linux - name: Debian - version: "12" - architecture: amd64 - description: "Debian 12 (Bookworm) generic cloud image" - - name: debian-13 - url: https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2 - storage: 20Gi - os: - family: Linux - name: Debian - version: "13" - architecture: amd64 - description: "Debian 13 (Trixie) generic cloud image" - - name: centos-stream-9 - url: https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-9-latest.x86_64.qcow2 - storage: 20Gi - os: - family: Linux - name: CentOS - version: "9" - architecture: amd64 - description: "CentOS Stream 9 GenericCloud image" - - name: centos-stream-10 - url: https://cloud.centos.org/centos/10-stream/x86_64/images/CentOS-Stream-GenericCloud-10-latest.x86_64.qcow2 - storage: 20Gi - os: - family: Linux - name: CentOS - version: "10" - architecture: amd64 - description: "CentOS Stream 10 GenericCloud image" - - name: opensuse-leap-15.6 - url: https://download.opensuse.org/repositories/Cloud:/Images:/Leap_15.6/images/openSUSE-Leap-15.6.x86_64-NoCloud.qcow2 - storage: 20Gi - os: - family: Linux - name: openSUSE - version: "15.6" - architecture: amd64 - description: "openSUSE Leap 15.6 NoCloud image" - - name: opensuse-leap-16.0 - url: https://download.opensuse.org/repositories/openSUSE:/Leap:/16.0:/Images/images/Leap-16.0-Minimal-VM.x86_64-Cloud.qcow2 - storage: 20Gi - os: - family: Linux - name: openSUSE - version: "16.0" - architecture: amd64 - description: "openSUSE Leap 16.0 Minimal VM cloud image" - - name: ubuntu-20.04 - url: https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img - storage: 20Gi - os: - family: Linux - name: Ubuntu - version: "20.04" - architecture: amd64 - description: "Ubuntu 20.04 LTS (Focal Fossa) cloud image" - - name: alpine-3.21 - url: https://dl-cdn.alpinelinux.org/alpine/v3.21/releases/cloud/nocloud_alpine-3.21.6-x86_64-bios-cloudinit-r0.qcow2 - storage: 20Gi - os: - family: Linux - name: Alpine - version: "3.21" - architecture: amd64 - description: "Alpine Linux 3.21 cloud image (cloud-init)" diff --git a/packages/system/vm-disk-rd/cozyrds/vm-disk.yaml b/packages/system/vm-disk-rd/cozyrds/vm-disk.yaml index 3ce046ed..4d0d7e37 100644 --- a/packages/system/vm-disk-rd/cozyrds/vm-disk.yaml +++ b/packages/system/vm-disk-rd/cozyrds/vm-disk.yaml @@ -8,7 +8,7 @@ spec: singular: vmdisk plural: vmdisks openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"source":{"description":"The source image location used to create a disk.","type":"object","default":{},"properties":{"disk":{"description":"Clone an existing vm-disk.","type":"object","required":["name"],"properties":{"name":{"description":"Name of the vm-disk to clone.","type":"string"}}},"http":{"description":"Download image from an HTTP source.","type":"object","required":["url"],"properties":{"url":{"description":"URL to download the image.","type":"string"}}},"image":{"description":"Use image by name from default collection.","type":"object","required":["name"],"properties":{"name":{"description":"Name of the image to use.","type":"string"}}},"upload":{"description":"Upload local image.","type":"object"}}},"optical":{"description":"Defines if disk should be considered optical.","type":"boolean","default":false},"storage":{"description":"The size of the disk allocated for the virtual machine.","default":"5Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"replicated"}}} + {"title":"Chart Values","type":"object","properties":{"source":{"description":"The source image location used to create a disk.","type":"object","default":{},"properties":{"http":{"description":"Download image from an HTTP source.","type":"object","required":["url"],"properties":{"url":{"description":"URL to download the image.","type":"string"}}},"image":{"description":"Use image by name.","type":"object","required":["name"],"properties":{"name":{"description":"Name of the image to use (uploaded as \"golden image\" or from the list: `ubuntu`, `fedora`, `cirros`, `alpine`, `talos`).","type":"string"}}},"upload":{"description":"Upload local image.","type":"object"}}},"optical":{"description":"Defines if disk should be considered optical.","type":"boolean","default":false},"storage":{"description":"The size of the disk allocated for the virtual machine.","default":"5Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"replicated"}}} release: prefix: vm-disk- labels: diff --git a/packages/system/vm-instance-rd/cozyrds/vm-instance.yaml b/packages/system/vm-instance-rd/cozyrds/vm-instance.yaml index e56bd9dd..e93fa86b 100644 --- a/packages/system/vm-instance-rd/cozyrds/vm-instance.yaml +++ b/packages/system/vm-instance-rd/cozyrds/vm-instance.yaml @@ -8,7 +8,7 @@ spec: singular: vminstance plural: vminstances openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"external":{"description":"Enable external access from outside the cluster.","type":"boolean","default":false},"externalMethod":{"description":"Method to pass through traffic to the VM.","type":"string","default":"PortList","enum":["PortList","WholeIP"]},"externalPorts":{"description":"Ports to forward from outside the cluster.","type":"array","default":[22],"items":{"type":"integer"}},"externalAllowICMP":{"description":"Whether to accept ICMP traffic to the VM in PortList mode (preserves ping and PMTU discovery). No effect in WholeIP mode. Default true so ping behaves as users expect even when port filtering is in effect.","type":"boolean","default":true},"runStrategy":{"description":"Requested running state of the VirtualMachineInstance","type":"string","default":"Always","enum":["Always","Halted","Manual","RerunOnFailure","Once"]},"instanceType":{"description":"Virtual Machine instance type.","type":"string","default":"u1.medium"},"instanceProfile":{"description":"Virtual Machine preferences profile.","type":"string","default":"ubuntu","enum":["alpine","centos.7","centos.7.desktop","centos.stream10","centos.stream10.desktop","centos.stream8","centos.stream8.desktop","centos.stream8.dpdk","centos.stream9","centos.stream9.desktop","centos.stream9.dpdk","cirros","fedora","fedora.arm64","opensuse.leap","opensuse.tumbleweed","rhel.10","rhel.10.arm64","rhel.7","rhel.7.desktop","rhel.8","rhel.8.desktop","rhel.8.dpdk","rhel.9","rhel.9.arm64","rhel.9.desktop","rhel.9.dpdk","rhel.9.realtime","sles","ubuntu","windows.10","windows.10.virtio","windows.11","windows.11.virtio","windows.2k16","windows.2k16.virtio","windows.2k19","windows.2k19.virtio","windows.2k22","windows.2k22.virtio","windows.2k25","windows.2k25.virtio",""]},"disks":{"description":"List of disks to attach.","type":"array","default":[],"items":{"type":"object","required":["name"],"properties":{"bus":{"description":"Disk bus type (e.g. \"sata\").","type":"string"},"name":{"description":"Disk name.","type":"string"}}}},"networks":{"description":"Networks to attach the VM to.","type":"array","default":[],"items":{"type":"object","properties":{"name":{"description":"Network attachment name.","type":"string"}}}},"subnets":{"description":"Deprecated: use networks instead.","type":"array","default":[],"items":{"type":"object","properties":{"name":{"description":"Network attachment name.","type":"string"}}}},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","default":[],"items":{"type":"object","required":["name"],"properties":{"name":{"description":"The name of the GPU resource to attach.","type":"string"}}}},"cpuModel":{"description":"Model specifies the CPU model inside the VMI. List of available models https://github.com/libvirt/libvirt/tree/master/src/cpu_map","type":"string","default":""},"resources":{"description":"Resource configuration for the virtual machine.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"sockets":{"description":"Number of CPU sockets (vCPU topology).","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"sshKeys":{"description":"List of SSH public keys for authentication.","type":"array","default":[],"items":{"type":"string"}},"cloudInit":{"description":"Cloud-init user data.","type":"string","default":""},"cloudInitSeed":{"description":"Seed string to generate SMBIOS UUID for the VM.","type":"string","default":""}}} + {"title":"Chart Values","type":"object","properties":{"external":{"description":"Enable external access from outside the cluster.","type":"boolean","default":false},"externalMethod":{"description":"Method to pass through traffic to the VM.","type":"string","default":"PortList","enum":["PortList","WholeIP"]},"externalPorts":{"description":"Ports to forward from outside the cluster.","type":"array","default":[22],"items":{"type":"integer"}},"runStrategy":{"description":"Requested running state of the VirtualMachineInstance","type":"string","default":"Always","enum":["Always","Halted","Manual","RerunOnFailure","Once"]},"instanceType":{"description":"Virtual Machine instance type.","type":"string","default":"u1.medium"},"instanceProfile":{"description":"Virtual Machine preferences profile.","type":"string","default":"ubuntu","enum":["alpine","centos.7","centos.7.desktop","centos.stream10","centos.stream10.desktop","centos.stream8","centos.stream8.desktop","centos.stream8.dpdk","centos.stream9","centos.stream9.desktop","centos.stream9.dpdk","cirros","fedora","fedora.arm64","opensuse.leap","opensuse.tumbleweed","rhel.10","rhel.10.arm64","rhel.7","rhel.7.desktop","rhel.8","rhel.8.desktop","rhel.8.dpdk","rhel.9","rhel.9.arm64","rhel.9.desktop","rhel.9.dpdk","rhel.9.realtime","sles","ubuntu","windows.10","windows.10.virtio","windows.11","windows.11.virtio","windows.2k16","windows.2k16.virtio","windows.2k19","windows.2k19.virtio","windows.2k22","windows.2k22.virtio","windows.2k25","windows.2k25.virtio",""]},"disks":{"description":"List of disks to attach.","type":"array","default":[],"items":{"type":"object","required":["name"],"properties":{"bus":{"description":"Disk bus type (e.g. \"sata\").","type":"string"},"name":{"description":"Disk name.","type":"string"}}}},"networks":{"description":"Networks to attach the VM to.","type":"array","default":[],"items":{"type":"object","properties":{"name":{"description":"Network attachment name.","type":"string"}}}},"subnets":{"description":"Deprecated: use networks instead.","type":"array","default":[],"items":{"type":"object","properties":{"name":{"description":"Network attachment name.","type":"string"}}}},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","default":[],"items":{"type":"object","required":["name"],"properties":{"name":{"description":"The name of the GPU resource to attach.","type":"string"}}}},"cpuModel":{"description":"Model specifies the CPU model inside the VMI. List of available models https://github.com/libvirt/libvirt/tree/master/src/cpu_map","type":"string","default":""},"resources":{"description":"Resource configuration for the virtual machine.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"sockets":{"description":"Number of CPU sockets (vCPU topology).","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"sshKeys":{"description":"List of SSH public keys for authentication.","type":"array","default":[],"items":{"type":"string"}},"cloudInit":{"description":"Cloud-init user data.","type":"string","default":""},"cloudInitSeed":{"description":"Seed string to generate SMBIOS UUID for the VM.","type":"string","default":""}}} release: prefix: vm-instance- labels: @@ -26,7 +26,7 @@ spec: tags: - compute icon: PHN2ZyB3aWR0aD0iMTQ0IiBoZWlnaHQ9IjE0NCIgdmlld0JveD0iMCAwIDE0NCAxNDQiIGZpbGw9Im5vbmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CjxyZWN0IHdpZHRoPSIxNDQiIGhlaWdodD0iMTQ0IiByeD0iMjQiIGZpbGw9InVybCgjcGFpbnQwX2xpbmVhcl82ODdfMzQ1NCkiLz4KPGcgY2xpcC1wYXRoPSJ1cmwoI2NsaXAwXzY4N18zNDU0KSI+CjxwYXRoIGQ9Ik04OS41MDM5IDExMS43MDdINTQuNDk3QzU0LjE3MjcgMTExLjcwNyA1NC4wMTA4IDExMS4yMjEgNTQuMzM0OSAxMTEuMDU5TDU3LjI1MjIgMTA4Ljk1MkM2MC4zMzE0IDEwNi42ODMgNjEuOTUyMiAxMDIuNjMxIDYwLjk3OTcgOTguNzQxMkg4My4wMjFDODIuMDQ4NSAxMDIuNjMxIDgzLjY2OTMgMTA2LjY4MyA4Ni43NDg1IDEwOC45NTJMODkuNjY1OCAxMTEuMDU5Qzg5Ljk5IDExMS4yMjEgODkuODI3OSAxMTEuNzA3IDg5LjUwMzkgMTExLjcwN1oiIGZpbGw9IiNCMEI2QkIiLz4KPHBhdGggZD0iTTExMy4zMjggOTguNzQxSDMwLjY3MjVDMjcuNTkzMSA5OC43NDEgMjUgOTYuMTQ4IDI1IDkzLjA2ODdWMzMuMTAzMkMyNSAzMC4wMjM5IDI3LjU5MzEgMjcuNDMwNyAzMC42NzI1IDI3LjQzMDdIMTEzLjMyOEMxMTYuNDA3IDI3LjQzMDcgMTE5IDMwLjAyMzcgMTE5IDMzLjEwMzJWOTMuMDY4N0MxMTkgOTYuMTQ4IDExNi40MDcgOTguNzQxIDExMy4zMjggOTguNzQxWiIgZmlsbD0iI0U4RURFRSIvPgo8cGF0aCBkPSJNMTE5IDg0LjE1NDlIMjVWMzMuMTAzMkMyNSAzMC4wMjM5IDI3LjU5MzEgMjcuNDMwNyAzMC42NzI1IDI3LjQzMDdIMTEzLjMyOEMxMTYuNDA3IDI3LjQzMDcgMTE5IDMwLjAyMzcgMTE5IDMzLjEwMzJMMTE5IDg0LjE1NDlaIiBmaWxsPSIjMDBCM0ZGIi8+CjxwYXRoIGQ9Ik05MC42Mzc0IDExNi41NjlINTMuMzYxNkM1Mi4wNjUxIDExNi41NjkgNTAuOTMwNyAxMTUuNDM1IDUwLjkzMDcgMTE0LjEzOEM1MC45MzA3IDExMi44NDEgNTIuMDY1MSAxMTEuNzA3IDUzLjM2MTYgMTExLjcwN0g5MC42Mzc0QzkxLjkzMzkgMTExLjcwNyA5My4wNjg0IDExMi44NDEgOTMuMDY4NCAxMTQuMTM4QzkzLjA2ODQgMTE1LjQzNSA5MS45MzM5IDExNi41NjkgOTAuNjM3NCAxMTYuNTY5WiIgZmlsbD0iI0U4RURFRSIvPgo8L2c+CjxwYXRoIGQ9Ik03Mi41Mjc1IDUzLjgzNjdDNzIuNDQzMSA1My44MzUxIDcyLjM2MDUgNTMuODEyMiA3Mi4yODczIDUzLjc3MDFMNTYuNDY5OSA0NC43OTM0QzU2LjM5ODMgNDQuNzUxOSA1Ni4zMzg4IDQ0LjY5MjMgNTYuMjk3MyA0NC42MjA3QzU2LjI1NTkgNDQuNTQ5IDU2LjIzMzggNDQuNDY3OCA1Ni4yMzM0IDQ0LjM4NUM1Ni4yMzM0IDQ0LjIxNjkgNTYuMzI1OCA0NC4wNjE3IDU2LjQ2OTkgNDMuOTc4NUw3Mi4xOTEyIDM1LjA2MDlDNzIuMjYzNyAzNS4wMjEgNzIuMzQ1IDM1IDcyLjQyNzcgMzVDNzIuNTEwNSAzNSA3Mi41OTE4IDM1LjAyMSA3Mi42NjQzIDM1LjA2MDlMODguNDg3MiA0NC4wMzk1Qzg4LjU1OTEgNDQuMDgwMSA4OC42MTg4IDQ0LjEzOTIgODguNjYgNDQuMjEwN0M4OC43MDEzIDQ0LjI4MjIgODguNzIyNyA0NC4zNjM1IDg4LjcyMTkgNDQuNDQ2Qzg4LjcyMjUgNDQuNTI4NSA4OC43MDEgNDQuNjA5NyA4OC42NTk4IDQ0LjY4MTJDODguNjE4NSA0NC43NTI2IDg4LjU1ODkgNDQuODExOCA4OC40ODcyIDQ0Ljg1MjVMNzIuNzcxNCA1My43NjgzQzcyLjY5NzIgNTMuODExNCA3Mi42MTMzIDUzLjgzNDkgNzIuNTI3NSA1My44MzY3IiBmaWxsPSJ3aGl0ZSIvPgo8cGF0aCBvcGFjaXR5PSIwLjciIGQ9Ik03MC4yNTUzIDc1LjY1MTdDNzAuMTcxIDc1LjY1MzUgNzAuMDg3OCA3NS42MzE3IDcwLjAxNTEgNzUuNTg4OEw1NC4yNDU4IDY2LjY0MTdDNTQuMTcxNSA2Ni42MDI0IDU0LjEwOTUgNjYuNTQzNiA1NC4wNjYxIDY2LjQ3MTZDNTQuMDIyOCA2Ni4zOTk3IDU0IDY2LjMxNzMgNTQgNjYuMjMzM1Y0OC4yNzhDNTQgNDguMTA4IDU0LjA5MjQgNDcuOTU0NiA1NC4yNDM5IDQ3Ljg2OTZDNTQuMzE3MiA0Ny44MjcxIDU0LjQwMDQgNDcuODA0NyA1NC40ODUxIDQ3LjgwNDdDNTQuNTY5NyA0Ny44MDQ3IDU0LjY1MjkgNDcuODI3MSA1NC43MjYyIDQ3Ljg2OTZMNzAuNDkzNyA1Ni44MTMxQzcwLjU2NDIgNTYuODU2NSA3MC42MjI1IDU2LjkxNyA3MC42NjMyIDU2Ljk4OTFDNzAuNzAzOSA1Ny4wNjEyIDcwLjcyNTcgNTcuMTQyNCA3MC43MjY1IDU3LjIyNTFWNzUuMTgwNUM3MC43MjU5IDc1LjI2MjggNzAuNzA0MiA3NS4zNDM2IDcwLjY2MzUgNzUuNDE1MUM3MC42MjI3IDc1LjQ4NjYgNzAuNTY0MiA3NS41NDY0IDcwLjQ5MzcgNzUuNTg4OEM3MC40MjA2IDc1LjYyOTEgNzAuMzM4NyA3NS42NTA3IDcwLjI1NTMgNzUuNjUxNyIgZmlsbD0id2hpdGUiLz4KPHBhdGggb3BhY2l0eT0iMC40IiBkPSJNNzQuNzE5OCA3NS42NTExQzc0LjYzMzMgNzUuNjUxMiA3NC41NDgyIDc1LjYyOTYgNzQuNDcyMiA3NS41ODgzQzc0LjQwMTYgNzUuNTQ2MSA3NC4zNDMyIDc1LjQ4NjIgNzQuMzAyNyA3NS40MTQ3Qzc0LjI2MjMgNzUuMzQzMSA3NC4yNDExIDc1LjI2MjIgNzQuMjQxMiA3NS4xOFY1Ny4zMzczQzc0LjI0MTIgNTcuMTcxIDc0LjMzMzYgNTcuMDE1OCA3NC40NzIyIDU2LjkyOUw5MC4yMzk3IDQ3Ljk4NTVDOTAuMzExOSA0Ny45NDM4IDkwLjM5MzggNDcuOTIxOSA5MC40NzcxIDQ3LjkyMTlDOTAuNTYwNSA0Ny45MjE5IDkwLjY0MjQgNDcuOTQzOCA5MC43MTQ2IDQ3Ljk4NTVDOTAuNzg3NiA0OC4wMjU1IDkwLjg0ODUgNDguMDg0MiA5MC44OTExIDQ4LjE1NTdDOTAuOTMzNyA0OC4yMjcyIDkwLjk1NjMgNDguMzA4OCA5MC45NTY2IDQ4LjM5MlY2Ni4yMzI4QzkwLjk1NyA2Ni4zMTY0IDkwLjkzNDcgNjYuMzk4NSA5MC44OTIxIDY2LjQ3MDRDOTAuODQ5NSA2Ni41NDI0IDkwLjc4ODEgNjYuNjAxNCA5MC43MTQ2IDY2LjY0MTFMNzQuOTUyNiA3NS41ODgzQzc0Ljg4MjUgNzUuNjMwNyA3NC44MDE4IDc1LjY1MjUgNzQuNzE5OCA3NS42NTExIiBmaWxsPSJ3aGl0ZSIvPgo8ZGVmcz4KPGxpbmVhckdyYWRpZW50IGlkPSJwYWludDBfbGluZWFyXzY4N18zNDU0IiB4MT0iMTYxIiB5MT0iMTgwIiB4Mj0iMy41OTI4NGUtMDciIHkyPSI0Ljk5OTk4IiBncmFkaWVudFVuaXRzPSJ1c2VyU3BhY2VPblVzZSI+CjxzdG9wLz4KPHN0b3Agb2Zmc2V0PSIxIiBzdG9wLWNvbG9yPSIjNTk1NjU2Ii8+CjwvbGluZWFyR3JhZGllbnQ+CjxjbGlwUGF0aCBpZD0iY2xpcDBfNjg3XzM0NTQiPgo8cmVjdCB3aWR0aD0iOTQiIGhlaWdodD0iOTQiIGZpbGw9IndoaXRlIiB0cmFuc2Zvcm09InRyYW5zbGF0ZSgyNSAyNSkiLz4KPC9jbGlwUGF0aD4KPC9kZWZzPgo8L3N2Zz4K - keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "external"], ["spec", "externalMethod"], ["spec", "externalPorts"], ["spec", "externalAllowICMP"], ["spec", "runStrategy"], ["spec", "instanceType"], ["spec", "instanceProfile"], ["spec", "disks"], ["spec", "networks"], ["spec", "subnets"], ["spec", "gpus"], ["spec", "cpuModel"], ["spec", "resources"], ["spec", "sshKeys"], ["spec", "cloudInit"], ["spec", "cloudInitSeed"]] + keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "external"], ["spec", "externalMethod"], ["spec", "externalPorts"], ["spec", "runStrategy"], ["spec", "instanceType"], ["spec", "instanceProfile"], ["spec", "disks"], ["spec", "networks"], ["spec", "subnets"], ["spec", "gpus"], ["spec", "cpuModel"], ["spec", "resources"], ["spec", "sshKeys"], ["spec", "cloudInit"], ["spec", "cloudInitSeed"]] secrets: exclude: [] include: [] diff --git a/pkg/apis/apps/validation/validation.go b/pkg/apis/apps/validation/validation.go index d62f2f22..273873a4 100644 --- a/pkg/apis/apps/validation/validation.go +++ b/pkg/apis/apps/validation/validation.go @@ -17,39 +17,16 @@ limitations under the License. package validation import ( - "regexp" - k8svalidation "k8s.io/apimachinery/pkg/util/validation" "k8s.io/apimachinery/pkg/util/validation/field" ) -// TenantKind is the Application.Kind string that gates the tenant-specific -// name rules below. It must stay in sync with the `kind` field of the tenant -// ApplicationDefinition (packages/system/tenant-rd/cozyrds/tenant.yaml) which -// is the upstream source the aggregated API reads at startup via -// config.Application.Kind. -const TenantKind = "Tenant" - -// tenantNameRegex enforces alphanumeric-only tenant names that begin with a -// lowercase letter. This is stricter than DNS-1035 because the tenant Helm -// chart's tenant.name helper (packages/apps/tenant/templates/_helpers.tpl) -// splits Release.Name on "-" and fails unless the result is exactly -// ["tenant", ""]. Any dash inside would break that invariant at -// Helm template time, so the aggregated API must reject such names up-front -// with a specific error. Requiring a leading letter (rather than letting -// leading-digit names fall through to DNS-1035) keeps the error message -// tenant-specific for all invalid inputs. -var tenantNameRegex = regexp.MustCompile(`^[a-z][a-z0-9]*$`) - -// ValidateApplicationName validates that an Application name is acceptable for -// the given kind. All applications must conform to DNS-1035 because their -// names are used to create Kubernetes resources (Services, Namespaces, etc.) -// that require DNS-1035 compliance. Tenant applications additionally must be -// alphanumeric and begin with a lowercase letter because of the Helm chart -// constraint described on tenantNameRegex. -// Note: length validation is handled separately by validateNameLength in the -// REST handler, which computes dynamic limits based on Helm release prefix. -func ValidateApplicationName(name, kindName string, fldPath *field.Path) field.ErrorList { +// ValidateApplicationName validates that an Application name conforms to DNS-1035. +// This is required because Application names are used to create Kubernetes resources +// (Services, Namespaces, etc.) that must have DNS-1035 compliant names. +// Note: length validation is handled separately by validateNameLength in the REST +// handler, which computes dynamic limits based on Helm release prefix. +func ValidateApplicationName(name string, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} if len(name) == 0 { @@ -57,16 +34,6 @@ func ValidateApplicationName(name, kindName string, fldPath *field.Path) field.E return allErrs } - // Tenant names must be alphanumeric starting with a letter — see - // tenantNameRegex comment for the reason. Check before DNS-1035 so the - // error message is specific to the tenant contract, not the generic DNS - // label rules. - if kindName == TenantKind && !tenantNameRegex.MatchString(name) { - allErrs = append(allErrs, field.Invalid(fldPath, name, - "tenant names must start with a lowercase letter and contain only lowercase letters and digits; dashes are not allowed")) - return allErrs - } - for _, msg := range k8svalidation.IsDNS1035Label(name) { allErrs = append(allErrs, field.Invalid(fldPath, name, msg)) } diff --git a/pkg/apis/apps/validation/validation_test.go b/pkg/apis/apps/validation/validation_test.go index 7930f8ae..7bf194d5 100644 --- a/pkg/apis/apps/validation/validation_test.go +++ b/pkg/apis/apps/validation/validation_test.go @@ -27,148 +27,55 @@ func TestValidateApplicationName(t *testing.T) { tests := []struct { name string appName string - kindName string wantError bool }{ - // Valid names (non-tenant kinds permit DNS-1035 including hyphens) - {"valid simple name", "tenant-one", "MySQL", false}, - {"valid single letter", "a", "MySQL", false}, - {"valid with numbers", "abc-123", "MySQL", false}, - {"valid lowercase", "my-tenant", "MySQL", false}, - {"valid long name", "my-very-long-tenant-name", "MySQL", false}, - {"valid double hyphen", "my--tenant", "MySQL", false}, - {"valid at DNS-1035 max (63 chars)", strings.Repeat("a", 63), "MySQL", false}, - {"valid with empty kind", "my-db", "", false}, + // Valid names + {"valid simple name", "tenant-one", false}, + {"valid single letter", "a", false}, + {"valid with numbers", "abc-123", false}, + {"valid lowercase", "my-tenant", false}, + {"valid long name", "my-very-long-tenant-name", false}, + {"valid double hyphen", "my--tenant", false}, + {"valid at DNS-1035 max (63 chars)", strings.Repeat("a", 63), false}, // Invalid: starts with wrong character - {"starts with digit", "1john", "MySQL", true}, - {"only digits", "123", "MySQL", true}, - {"starts with hyphen", "-tenant", "MySQL", true}, + {"starts with digit", "1john", true}, + {"only digits", "123", true}, + {"starts with hyphen", "-tenant", true}, // Invalid: ends with wrong character - {"ends with hyphen", "tenant-", "MySQL", true}, + {"ends with hyphen", "tenant-", true}, // Invalid: wrong characters - {"uppercase letters", "Tenant", "MySQL", true}, - {"mixed case", "myTenant", "MySQL", true}, - {"underscore", "my_tenant", "MySQL", true}, - {"dot", "my.tenant", "MySQL", true}, - {"space", "my tenant", "MySQL", true}, - {"unicode cyrillic", "тенант", "MySQL", true}, - {"unicode emoji", "tenant🚀", "MySQL", true}, - {"special chars", "tenant@home", "MySQL", true}, - {"colon", "tenant:one", "MySQL", true}, - {"slash", "tenant/one", "MySQL", true}, + {"uppercase letters", "Tenant", true}, + {"mixed case", "myTenant", true}, + {"underscore", "my_tenant", true}, + {"dot", "my.tenant", true}, + {"space", "my tenant", true}, + {"unicode cyrillic", "тенант", true}, + {"unicode emoji", "tenant🚀", true}, + {"special chars", "tenant@home", true}, + {"colon", "tenant:one", true}, + {"slash", "tenant/one", true}, // Invalid: empty or whitespace - {"empty string", "", "MySQL", true}, - {"only spaces", " ", "MySQL", true}, - {"leading space", " tenant", "MySQL", true}, - {"trailing space", "tenant ", "MySQL", true}, + {"empty string", "", true}, + {"only spaces", " ", true}, + {"leading space", " tenant", true}, + {"trailing space", "tenant ", true}, // Invalid: exceeds DNS-1035 max length (63) - {"too long (64 chars)", strings.Repeat("a", 64), "MySQL", true}, - {"way too long (100 chars)", strings.Repeat("a", 100), "MySQL", true}, - - // Tenant kind: stricter alphanumeric-only rule. - // The tenant Helm chart's tenant.name helper (packages/apps/tenant/templates/_helpers.tpl) - // splits Release.Name on "-" and fails unless the result is exactly - // ["tenant", ""]. Any dash inside breaks that invariant, so - // the aggregated API must reject tenant names containing dashes up-front - // with a specific error — instead of letting Flux reconciliation fail later. - {"tenant alphanumeric simple", "foo", "Tenant", false}, - {"tenant alphanumeric with digits", "foo123", "Tenant", false}, - {"tenant single char", "a", "Tenant", false}, - {"tenant single hyphen", "foo-bar", "Tenant", true}, - {"tenant leading hyphen", "-foo", "Tenant", true}, - {"tenant trailing hyphen", "foo-", "Tenant", true}, - {"tenant double hyphen", "foo--bar", "Tenant", true}, - {"tenant uppercase", "Foo", "Tenant", true}, - {"tenant underscore", "foo_bar", "Tenant", true}, - {"tenant empty", "", "Tenant", true}, - // Leading digit must be caught by the tenant-specific regex (not by - // falling through to DNS-1035) so the error message reflects the - // tenant contract — see TestValidateApplicationName_TenantErrorMessage. - {"tenant leading digit", "123foo", "Tenant", true}, + {"too long (64 chars)", strings.Repeat("a", 64), true}, + {"way too long (100 chars)", strings.Repeat("a", 100), true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - errs := ValidateApplicationName(tt.appName, tt.kindName, field.NewPath("metadata").Child("name")) + errs := ValidateApplicationName(tt.appName, field.NewPath("metadata").Child("name")) if (len(errs) > 0) != tt.wantError { - t.Errorf("ValidateApplicationName(%q, kind=%q) returned %d errors, wantError = %v, errors = %v", - tt.appName, tt.kindName, len(errs), tt.wantError, errs) + t.Errorf("ValidateApplicationName(%q) returned %d errors, wantError = %v, errors = %v", + tt.appName, len(errs), tt.wantError, errs) } }) } } - -// TestValidateApplicationName_TenantErrorMessage pins the contract that when -// a tenant name is invalid, the returned error message is specific to the -// tenant naming rule — not the generic DNS-1035 message. Otherwise users get -// back "must start with an alphabetic character" or similar and have no way -// to know the constraint is tied to the tenant Helm chart. -func TestValidateApplicationName_TenantErrorMessage(t *testing.T) { - // Every tenant-invalid name below must surface a tenant-specific error - // message. In particular, "123foo" starts with a digit — the original - // implementation let that fall through to DNS-1035 with a generic error; - // the regex is tightened specifically so this case fails up-front. - invalidTenantNames := []string{ - "foo-bar", // dash - "-foo", // leading dash - "foo-", // trailing dash - "foo--bar", // double dash - "Foo", // uppercase - "foo_bar", // underscore - "foo.bar", // dot - "foo bar", // space - "123foo", // leading digit — must not fall through to DNS-1035 - } - - const wantSubstring = "tenant names must" - - for _, name := range invalidTenantNames { - t.Run(name, func(t *testing.T) { - errs := ValidateApplicationName(name, "Tenant", field.NewPath("metadata").Child("name")) - if len(errs) == 0 { - t.Fatalf("expected error for tenant name %q, got none", name) - } - if !strings.Contains(errs[0].Detail, wantSubstring) { - t.Errorf("tenant name %q: error detail = %q, want substring %q (generic DNS-1035 message is not tenant-specific)", - name, errs[0].Detail, wantSubstring) - } - }) - } -} - -// TestValidateApplicationName_TenantLengthFallthrough documents the one -// invalid-tenant case where the error message is intentionally NOT tenant- -// specific: when a name contains only valid tenant characters but exceeds -// the DNS-1035 63-char label limit, the length error comes from DNS-1035 -// because length is not a tenant-specific constraint (every application -// kind is subject to the same Kubernetes label limit). REST.validateNameLength -// further tightens the limit using the Helm release prefix, so tenants cannot -// actually reach 64 characters end-to-end — this test only pins the package- -// level fallthrough so a future refactor does not accidentally promote the -// length error into tenant-specific wording. -// -// NOTE: this is an architectural decision, not a user-facing requirement. -// If tenant length is ever promoted into a tenant-specific rule (e.g. to -// include the Helm release prefix budget in this package's error message), -// this test should be updated or deleted — it is not a backwards-compat -// guarantee, just a checkpoint on the current layering. -func TestValidateApplicationName_TenantLengthFallthrough(t *testing.T) { - name := strings.Repeat("a", 64) // valid tenant pattern, too long for DNS-1035 - - errs := ValidateApplicationName(name, "Tenant", field.NewPath("metadata").Child("name")) - if len(errs) == 0 { - t.Fatalf("expected DNS-1035 length error for 64-char tenant name, got none") - } - // This error is the generic DNS-1035 one, NOT the tenant-specific message. - // We deliberately do not assert against the exact upstream DNS-1035 text - // (that would tie this test to a k8s.io/apimachinery internal string and - // break on unrelated upstream wording changes). - if strings.Contains(errs[0].Detail, "tenant names must") { - t.Errorf("64-char tenant name should surface the generic DNS-1035 error, got tenant-specific: %q", errs[0].Detail) - } -} diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index 005fc640..bf02e642 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -21,7 +21,6 @@ import ( "fmt" "time" - cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" helmv2 "github.com/fluxcd/helm-controller/api/v2" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -80,11 +79,6 @@ func init() { if err := rbacv1.AddToScheme(mgrScheme); err != nil { panic(fmt.Errorf("Failed to add RBAC types to scheme: %w", err)) } - - // Register Cozystack types for WorkloadMonitor queries. - if err := cozyv1alpha1.AddToScheme(mgrScheme); err != nil { - panic(fmt.Errorf("failed to add Cozystack types to scheme: %w", err)) - } // Add unversioned types. metav1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"}) @@ -174,7 +168,6 @@ func (c completedConfig) New() (*CozyServer, error) { &corev1.Namespace{}, &corev1.Service{}, &rbacv1.RoleBinding{}, - &cozyv1alpha1.WorkloadMonitor{}, ); err != nil { return nil, fmt.Errorf("failed to get informers: %w", err) } diff --git a/pkg/cmd/server/start.go b/pkg/cmd/server/start.go index 7ebbbc89..f7791332 100644 --- a/pkg/cmd/server/start.go +++ b/pkg/cmd/server/start.go @@ -160,34 +160,6 @@ func (o *CozyServerOptions) Complete() error { // Convert to ResourceConfig o.ResourceConfig = &config.ResourceConfig{} for _, crd := range crdList.Items { - release := config.ReleaseConfig{ - Prefix: crd.Spec.Release.Prefix, - Labels: crd.Spec.Release.Labels, - ChartRef: config.ChartRefConfig{ - Kind: crd.Spec.Release.ChartRef.Kind, - Name: crd.Spec.Release.ChartRef.Name, - Namespace: crd.Spec.Release.ChartRef.Namespace, - }, - } - // Per-Application HelmRelease Install/Upgrade timeout. Applications - // whose parent chart contains asynchronously-provisioned resources - // the chart itself depends on (for example, the Kamaji-provisioned - // admin-kubeconfig Secret for Kubernetes tenants) need a longer - // wait budget than the Flux default. Consumed by the REST storage - // layer when building the HelmRelease Spec. The parser rejects - // units Flux would reject at webhook time, so a bad annotation - // surfaces as a loud startup failure instead of a silent drop to - // defaults. - d, err := config.ParseHelmInstallTimeoutAnnotation( - crd.Annotations[config.HelmInstallTimeoutAnnotation], - ) - if err != nil { - return fmt.Errorf( - "ApplicationDefinition %q has invalid %s annotation: %w", - crd.Name, config.HelmInstallTimeoutAnnotation, err, - ) - } - release.HelmInstallTimeout = d resource := config.Resource{ Application: config.ApplicationConfig{ Kind: crd.Spec.Application.Kind, @@ -196,7 +168,15 @@ func (o *CozyServerOptions) Complete() error { ShortNames: []string{}, // TODO: implement shortnames OpenAPISchema: crd.Spec.Application.OpenAPISchema, }, - Release: release, + Release: config.ReleaseConfig{ + Prefix: crd.Spec.Release.Prefix, + Labels: crd.Spec.Release.Labels, + ChartRef: config.ChartRefConfig{ + Kind: crd.Spec.Release.ChartRef.Kind, + Name: crd.Spec.Release.ChartRef.Name, + Namespace: crd.Spec.Release.ChartRef.Namespace, + }, + }, } o.ResourceConfig.Resources = append(o.ResourceConfig.Resources, resource) } diff --git a/pkg/config/config.go b/pkg/config/config.go index 21de27b2..1e123e2c 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -16,48 +16,6 @@ limitations under the License. package config -import ( - "fmt" - "regexp" - "time" -) - -// HelmInstallTimeoutAnnotation is the ApplicationDefinition metadata -// annotation key that overrides the Flux HelmRelease Install.Timeout and -// Upgrade.Timeout for a given Application kind. -const HelmInstallTimeoutAnnotation = "release.cozystack.io/helm-install-timeout" - -// helmTimeoutPattern mirrors the CRD validation pattern used by Flux -// helm-controller on HelmReleaseSpec.Install.Timeout (ms/s/m/h units only). -// time.ParseDuration accepts ns/us/µs, but Flux rejects them - parsing here -// with the same shape avoids feeding the controller a value it will later -// reject at webhook time. See -// github.com/fluxcd/helm-controller/api/v2 HelmReleaseSpec.Install.Timeout -// in the go module cache. -var helmTimeoutPattern = regexp.MustCompile(`^([0-9]+(\.[0-9]+)?(ms|s|m|h))+$`) - -// ParseHelmInstallTimeoutAnnotation parses the value of the -// release.cozystack.io/helm-install-timeout annotation. The empty string is -// treated as "unset" and returns (0, nil) so callers can leave -// HelmInstallTimeout zeroed and let flux defaults apply. Values accepted by -// time.ParseDuration but rejected by Flux (ns/us/µs) return a helpful -// error instead of silently parsing and failing later at HelmRelease -// admission. -func ParseHelmInstallTimeoutAnnotation(raw string) (time.Duration, error) { - if raw == "" { - return 0, nil - } - if !helmTimeoutPattern.MatchString(raw) { - return 0, fmt.Errorf("must match %s (Flux accepts ms/s/m/h units only), got %q", - helmTimeoutPattern, raw) - } - d, err := time.ParseDuration(raw) - if err != nil { - return 0, fmt.Errorf("time.ParseDuration(%q): %w", raw, err) - } - return d, nil -} - // ResourceConfig represents the structure of the configuration file. type ResourceConfig struct { Resources []Resource `yaml:"resources"` @@ -83,12 +41,6 @@ type ReleaseConfig struct { Prefix string `yaml:"prefix"` Labels map[string]string `yaml:"labels"` ChartRef ChartRefConfig `yaml:"chartRef"` - // HelmInstallTimeout overrides the Flux HelmRelease Install.Timeout and - // Upgrade.Timeout for this Application kind. When zero, flux defaults - // apply. Populated from the - // release.cozystack.io/helm-install-timeout annotation on the - // ApplicationDefinition at start-up. - HelmInstallTimeout time.Duration `yaml:"helmInstallTimeout,omitempty"` } // ChartRefConfig references a Flux source artifact for the Helm chart. diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go deleted file mode 100644 index eb6990af..00000000 --- a/pkg/config/config_test.go +++ /dev/null @@ -1,119 +0,0 @@ -package config - -import ( - "strings" - "testing" - "time" -) - -// Cover the annotation parser used by cozystack-api at startup. The parser -// is consumed by pkg/cmd/server/start.go on every ApplicationDefinition; a -// typo here silently drops back to flux defaults and the Kubernetes tenant -// race described in cozystack#2412 reappears, so the table must exercise: -// - the unset path (empty string treated as "no override"), -// - every unit Flux accepts (ms, s, m, h), -// - compound forms (the CRD pattern accepts repeats), -// - units time.ParseDuration accepts but Flux rejects (ns, us, µs), -// - outright garbage. -func TestParseHelmInstallTimeoutAnnotation(t *testing.T) { - cases := []struct { - name string - input string - want time.Duration - wantErr bool - errMatch string - }{ - { - name: "empty string leaves flux defaults in place", - input: "", - want: 0, - }, - { - name: "minutes", - input: "15m", - want: 15 * time.Minute, - }, - { - name: "hours", - input: "1h", - want: time.Hour, - }, - { - name: "seconds", - input: "45s", - want: 45 * time.Second, - }, - { - name: "milliseconds", - input: "500ms", - want: 500 * time.Millisecond, - }, - { - name: "compound hour and minutes", - input: "2h30m", - want: 2*time.Hour + 30*time.Minute, - }, - { - name: "decimal minutes", - input: "1.5m", - want: 90 * time.Second, - }, - { - name: "nanoseconds rejected - Flux CRD pattern excludes ns", - input: "500ns", - wantErr: true, - errMatch: "Flux accepts ms/s/m/h units only", - }, - { - name: "microseconds rejected - Flux CRD pattern excludes us", - input: "500us", - wantErr: true, - errMatch: "Flux accepts ms/s/m/h units only", - }, - { - name: "microseconds unicode rejected", - input: "500µs", - wantErr: true, - errMatch: "Flux accepts ms/s/m/h units only", - }, - { - name: "bare digits rejected", - input: "15", - wantErr: true, - errMatch: "Flux accepts ms/s/m/h units only", - }, - { - name: "garbage rejected", - input: "abc", - wantErr: true, - errMatch: "Flux accepts ms/s/m/h units only", - }, - { - name: "negative rejected", - input: "-15m", - wantErr: true, - errMatch: "Flux accepts ms/s/m/h units only", - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - got, err := ParseHelmInstallTimeoutAnnotation(tc.input) - if tc.wantErr { - if err == nil { - t.Fatalf("expected error, got duration=%v", got) - } - if tc.errMatch != "" && !strings.Contains(err.Error(), tc.errMatch) { - t.Errorf("error %q does not contain %q", err.Error(), tc.errMatch) - } - return - } - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if got != tc.want { - t.Errorf("got %v, want %v", got, tc.want) - } - }) - } -} diff --git a/pkg/registry/apps/application/rest.go b/pkg/registry/apps/application/rest.go index d233ecc3..7871bbe7 100644 --- a/pkg/registry/apps/application/rest.go +++ b/pkg/registry/apps/application/rest.go @@ -26,7 +26,6 @@ import ( "sync" "time" - cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" helmv2 "github.com/fluxcd/helm-controller/api/v2" corev1 "k8s.io/api/core/v1" metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion" @@ -155,8 +154,8 @@ func (r *REST) Create(ctx context.Context, obj runtime.Object, createValidation return nil, fmt.Errorf("expected *appsv1alpha1.Application object, got %T", obj) } - // Validate Application name format (DNS-1035 plus any kind-specific rules) - if errs := r.validateNameFormat(app.Name); len(errs) > 0 { + // Validate Application name conforms to DNS-1035 + if errs := validation.ValidateApplicationName(app.Name, field.NewPath("metadata").Child("name")); len(errs) > 0 { return nil, apierrors.NewInvalid(r.gvk.GroupKind(), app.Name, errs) } @@ -165,17 +164,6 @@ func (r *REST) Create(ctx context.Context, obj runtime.Object, createValidation return nil, apierrors.NewInvalid(r.gvk.GroupKind(), app.Name, nameLenErrs) } - // For Tenant applications, also validate that the computed workload - // namespace fits within the DNS-1123 label limit. A deeply-nested tenant - // can exceed the limit even when its own name passes the per-name Helm - // release length check, because the namespace is formed from the full - // ancestor chain. - if r.kindName == "Tenant" { - if nsErrs := r.validateTenantNamespaceLength(app.Namespace, app.Name); len(nsErrs) > 0 { - return nil, apierrors.NewInvalid(r.gvk.GroupKind(), app.Name, nsErrs) - } - } - // Validate that values don't contain reserved keys (starting with "_") if err := validateNoInternalKeys(app.Spec); err != nil { return nil, apierrors.NewBadRequest(err.Error()) @@ -704,49 +692,14 @@ func (r *REST) Watch(ctx context.Context, options *metainternalversion.ListOptio } customW.underlying = helmWatcher - // Start watch on WorkloadMonitor to detect pod readiness changes. - // For Tenant applications the WorkloadMonitor lives in a computed child - // namespace (see computeTenantNamespace), not in the HelmRelease namespace, - // so scoping the watch to `namespace` would miss all events. Use a - // cluster-wide watch in that case — label selectors still restrict the - // stream to the relevant kind/group. - wmLabelSelector := labels.NewSelector().Add(*appKindReq, *appGroupReq) - wmList := &cozyv1alpha1.WorkloadMonitorList{} - wmListOpts := &client.ListOptions{LabelSelector: wmLabelSelector} - if r.kindName != "Tenant" { - wmListOpts.Namespace = namespace - } - wmWatcher, err := r.w.Watch(ctx, wmList, wmListOpts) - if err != nil { - klog.Warningf("Failed to set up WorkloadMonitor watch, workload status changes won't trigger events: %v", err) - // Non-fatal: proceed without WorkloadMonitor watch - wmWatcher = nil - } - go func() { - // Capture wmWatcher for cleanup; the variable may be set to nil - // inside the loop when the channel closes, so defer must use this copy. - wmWatcherForCleanup := wmWatcher defer close(customW.resultChan) defer customW.underlying.Stop() - if wmWatcherForCleanup != nil { - defer wmWatcherForCleanup.Stop() - } // Track whether we've sent the initial-events-end bookmark initialEventsEndSent := !sendInitialEvents // If not sendInitialEvents, consider it already sent var lastResourceVersion string - // Buffer of WorkloadMonitor events that arrived before the initial - // snapshot finished. The watch-list contract requires the stream to - // deliver all ADDED events followed by the initial-events-end bookmark - // before any live updates, so we hold WM-triggered Modified events and - // replay them once the bookmark has been emitted. Without this, a - // workload whose status flips during the snapshot window (after the - // Application ADDED but before the bookmark) and then stops changing - // would leave the client with a stale WorkloadsReady forever. - var pendingWMEvents []watch.Event - // Get the starting resourceVersion from options // If client provides resourceVersion (e.g., from a previous List), we should skip // objects with resourceVersion <= startingRV (client already has them) @@ -757,19 +710,6 @@ func (r *REST) Watch(ctx context.Context, options *metainternalversion.ListOptio } } - drainPendingWMEvents := func() { - for _, ev := range pendingWMEvents { - select { - case customW.resultChan <- ev: - case <-customW.stopChan: - return - case <-ctx.Done(): - return - } - } - pendingWMEvents = nil - } - // Helper function to send initial-events-end bookmark sendInitialEventsEndBookmark := func() { if initialEventsEndSent { @@ -794,11 +734,8 @@ func (r *REST) Watch(ctx context.Context, options *metainternalversion.ListOptio select { case customW.resultChan <- bookmarkEvent: case <-customW.stopChan: - return case <-ctx.Done(): - return } - drainPendingWMEvents() } // Process watch events @@ -826,10 +763,8 @@ func (r *REST) Watch(ctx context.Context, options *metainternalversion.ListOptio APIVersion: appsv1alpha1.SchemeGroupVersion.String(), Kind: r.kindName, } - justFlipped := false if !initialEventsEndSent { initialEventsEndSent = true - justFlipped = true bookmarkApp.SetAnnotations(map[string]string{ "k8s.io/initial-events-end": "true", }) @@ -846,9 +781,6 @@ func (r *REST) Watch(ctx context.Context, options *metainternalversion.ListOptio case <-ctx.Done(): return } - if justFlipped { - drainPendingWMEvents() - } } continue } @@ -936,104 +868,6 @@ func (r *REST) Watch(ctx context.Context, options *metainternalversion.ListOptio return } - case wmEvent, ok := <-wmResultChan(wmWatcher): - if !ok { - klog.V(4).Info("WorkloadMonitor watcher closed") - wmWatcher = nil - continue - } - if wmEvent.Type == watch.Bookmark || wmEvent.Type == watch.Error { - if wmEvent.Type == watch.Error { - klog.V(4).Infof("WorkloadMonitor watch error event: %v", wmEvent.Object) - } - continue - } - wm, ok := wmEvent.Object.(*cozyv1alpha1.WorkloadMonitor) - if !ok { - continue - } - // All WM event types (Added/Modified/Deleted) produce a Modified - // Application event because the Application itself is what changed - // from the client's perspective. - wmAppName, hasLabel := wm.Labels[ApplicationNameLabel] - if !hasLabel { - continue - } - // Filter: skip WorkloadMonitor events for applications not matching - // the watch scope (single-resource or field-selector filtered watches) - hrName := r.releaseConfig.Prefix + wmAppName - if filterByName != "" && hrName != filterByName { - continue - } - if resourceName != "" && wmAppName != resourceName { - continue - } - - // Locate the owning HelmRelease. For most application kinds the - // WorkloadMonitor and its HelmRelease live in the same namespace, - // but Tenant workloads live in a child namespace (see - // computeTenantNamespace) while the HelmRelease remains in the - // parent/requested namespace — so the WM-to-HR namespace mapping - // differs. - hrNS := wm.Namespace - if r.kindName == "Tenant" { - hrNS = namespace - // Filter out WM events whose child namespace does not - // correspond to the Tenant in our watched namespace, since - // the cluster-wide WM watch delivers events for all tenants. - if r.computeTenantNamespace(namespace, wmAppName) != wm.Namespace { - continue - } - if !fieldFilter.MatchesNamespace(hrNS) { - continue - } - } else if !fieldFilter.MatchesNamespace(hrNS) { - continue - } - hr := &helmv2.HelmRelease{} - if err := r.c.Get(ctx, client.ObjectKey{Namespace: hrNS, Name: hrName}, hr); err != nil { - klog.V(4).Infof("Cannot find HelmRelease %s/%s for WorkloadMonitor event: %v", hrNS, hrName, err) - continue - } - // Pass the fresh WorkloadMonitor so conversion uses the latest - // operational status even if the cache (r.c) has not yet - // observed this watch event. - app, err := r.ConvertHelmReleaseToApplicationWithMonitor(ctx, hr, wm) - if err != nil { - klog.V(4).Infof("Error converting HelmRelease for WorkloadMonitor event: %v", err) - continue - } - // Apply label selector filtering (same as HelmRelease event path) - if options.LabelSelector != nil { - sel, err := labels.Parse(options.LabelSelector.String()) - if err != nil { - klog.Errorf("Invalid label selector: %v", err) - continue - } - if !sel.Matches(labels.Set(app.Labels)) { - continue - } - } - // Use the WorkloadMonitor's ResourceVersion for the emitted event - // so clients see a monotonically increasing RV and don't skip this update. - app.SetResourceVersion(wm.GetResourceVersion()) - outEvent := watch.Event{Type: watch.Modified, Object: &app} - // Buffer WM-triggered events that arrive before the - // initial-events-end bookmark. They will be replayed in order - // immediately after the bookmark is emitted. - if !initialEventsEndSent { - pendingWMEvents = append(pendingWMEvents, outEvent) - continue - } - lastResourceVersion = wm.GetResourceVersion() - select { - case customW.resultChan <- outEvent: - case <-customW.stopChan: - return - case <-ctx.Done(): - return - } - case <-customW.stopChan: return case <-ctx.Done(): @@ -1046,15 +880,6 @@ func (r *REST) Watch(ctx context.Context, options *metainternalversion.ListOptio return customW, nil } -// wmResultChan returns the result channel of a WorkloadMonitor watcher, or a nil -// channel (which blocks forever in select) if the watcher is nil. -func wmResultChan(w watch.Interface) <-chan watch.Event { - if w == nil { - return nil - } - return w.ResultChan() -} - // customWatcher wraps the original watcher and filters/converts events type customWatcher struct { resultChan chan watch.Event @@ -1158,21 +983,12 @@ func filterPrefixedMap(original map[string]string, prefix string) map[string]str return processed } -// ConvertHelmReleaseToApplication converts a HelmRelease to an Application. +// ConvertHelmReleaseToApplication converts a HelmRelease to an Application func (r *REST) ConvertHelmReleaseToApplication(ctx context.Context, hr *helmv2.HelmRelease) (appsv1alpha1.Application, error) { - return r.ConvertHelmReleaseToApplicationWithMonitor(ctx, hr, nil) -} - -// ConvertHelmReleaseToApplicationWithMonitor converts a HelmRelease to an -// Application, optionally overriding the cached copy of a WorkloadMonitor with -// a fresher version received from the watch client. This prevents the emitted -// Application object from carrying stale WorkloadsReady data when r.c (cache) -// lags behind r.w (watch). -func (r *REST) ConvertHelmReleaseToApplicationWithMonitor(ctx context.Context, hr *helmv2.HelmRelease, freshMonitor *cozyv1alpha1.WorkloadMonitor) (appsv1alpha1.Application, error) { klog.V(6).Infof("Converting HelmRelease to Application for resource %s", hr.GetName()) // Convert HelmRelease struct to Application struct - app, err := r.convertHelmReleaseToApplication(ctx, hr, freshMonitor) + app, err := r.convertHelmReleaseToApplication(ctx, hr) if err != nil { klog.Errorf("Error converting from HelmRelease to Application: %v", err) return appsv1alpha1.Application{}, err @@ -1233,19 +1049,6 @@ func validateNoInternalKeys(values *apiextv1.JSON) error { // chart-generated resource suffixes within the 63-char DNS-1035 label limit. const maxHelmReleaseName = 53 -// maxNamespaceName is the DNS-1123 label limit for Kubernetes namespace names. -// The tenant Helm chart creates a Namespace whose name is the computed -// workload namespace (parent namespace + "-" + tenant name), so the total -// must fit inside a single 63-char DNS-1123 label. -const maxNamespaceName = 63 - -// validateNameFormat checks an Application name against DNS-1035 and any -// kind-specific format rules (e.g. Tenant names must be alphanumeric — see -// validation.ValidateApplicationName for the reasoning). -func (r *REST) validateNameFormat(name string) field.ErrorList { - return validation.ValidateApplicationName(name, r.kindName, field.NewPath("metadata").Child("name")) -} - // validateNameLength checks that the application name won't exceed Kubernetes limits. // prefix + name must fit within the Helm release name limit (53 chars). func (r *REST) validateNameLength(name string) field.ErrorList { @@ -1267,29 +1070,8 @@ func (r *REST) validateNameLength(name string) field.ErrorList { return allErrs } -// validateTenantNamespaceLength checks that the computed workload namespace -// for a Tenant application fits within the DNS-1123 label limit. The namespace -// is formed by dash-joining the parent namespace with the tenant name, so -// deep nesting can exceed the limit even when each individual name passes the -// per-name Helm release length check. -func (r *REST) validateTenantNamespaceLength(currentNamespace, tenantName string) field.ErrorList { - fldPath := field.NewPath("metadata").Child("name") - allErrs := field.ErrorList{} - - computed := r.computeTenantNamespace(currentNamespace, tenantName) - if len(computed) > maxNamespaceName { - allErrs = append(allErrs, field.Invalid(fldPath, tenantName, - fmt.Sprintf("computed tenant namespace %q would be %d characters, which exceeds the %d-character Kubernetes namespace limit; shorten the tenant name or reduce ancestor nesting depth", - computed, len(computed), maxNamespaceName))) - } - return allErrs -} - -// convertHelmReleaseToApplication implements the actual conversion logic. -// The optional freshMonitor is used to override the cache copy of a -// WorkloadMonitor when a newer version was delivered via the watch client — -// see ConvertHelmReleaseToApplicationWithMonitor for the rationale. -func (r *REST) convertHelmReleaseToApplication(ctx context.Context, hr *helmv2.HelmRelease, freshMonitor *cozyv1alpha1.WorkloadMonitor) (appsv1alpha1.Application, error) { +// convertHelmReleaseToApplication implements the actual conversion logic +func (r *REST) convertHelmReleaseToApplication(ctx context.Context, hr *helmv2.HelmRelease) (appsv1alpha1.Application, error) { // Filter out internal keys (starting with "_") from spec filteredSpec := filterInternalKeys(hr.Spec.Values) @@ -1326,81 +1108,10 @@ func (r *REST) convertHelmReleaseToApplication(ctx context.Context, hr *helmv2.H }) } } - // Enrich conditions with WorkloadMonitor operational status. - // Tenant workloads live in a child namespace (computed from the Tenant name), - // not in the same namespace as the owning HelmRelease — look there instead. - workloadsNS := hr.Namespace - if r.kindName == "Tenant" { - workloadsNS = r.computeTenantNamespace(hr.Namespace, app.Name) - } - ws, wsErr := r.getWorkloadsOperational(ctx, workloadsNS, app.Name, freshMonitor) - // Derive a stable LastTransitionTime: use the owning HelmRelease's own - // condition update time (or CreationTimestamp as a floor) so that repeated - // conversions of the same underlying state produce identical timestamps. - wrTransition := hr.CreationTimestamp - for _, c := range hr.GetConditions() { - if c.LastTransitionTime.After(wrTransition.Time) { - wrTransition = c.LastTransitionTime - } - } - if ws.transitionTime.After(wrTransition.Time) { - wrTransition = ws.transitionTime - } - if wrTransition.IsZero() { - // Fallback for objects that somehow have no timestamps at all - // (e.g. hand-crafted test fixtures). In production HelmReleases - // always carry a CreationTimestamp, so the stable branch above - // is used. - wrTransition = metav1.Now() - } - if wsErr != nil { - // Fail-open: if we can't query WorkloadMonitors (e.g., informer cache not ready), - // don't override Ready. Prefer operational availability over safety. - // The WorkloadsReady=Unknown condition still signals the issue to the user. - klog.Warningf("Failed to check workload monitors for %s/%s: %v", workloadsNS, app.Name, wsErr) - conditions = append(conditions, metav1.Condition{ - Type: "WorkloadsReady", - Status: metav1.ConditionUnknown, - LastTransitionTime: wrTransition, - Reason: "Error", - Message: fmt.Sprintf("Failed to check workload status: %v", wsErr), - }) - } else if ws.found { - workloadsCondition := metav1.Condition{ - Type: "WorkloadsReady", - LastTransitionTime: wrTransition, - Reason: "WorkloadMonitorCheck", - } - switch { - case !ws.operational: - // Concrete failure takes priority over unknown/pending state - workloadsCondition.Status = metav1.ConditionFalse - workloadsCondition.Message = "One or more workloads are not operational" - case ws.unknown: - workloadsCondition.Status = metav1.ConditionUnknown - workloadsCondition.Reason = "Pending" - workloadsCondition.Message = "One or more workloads have not been reconciled yet" - default: - workloadsCondition.Status = metav1.ConditionTrue - workloadsCondition.Message = "All workloads are operational" - } - conditions = append(conditions, workloadsCondition) - - // Intentionally do NOT override the Ready condition based on WorkloadsReady. - // Ready continues to reflect HelmRelease state only, which: - // - preserves backward compatibility with existing tooling (kubectl wait, - // GitOps health checks) that expect Ready to match HelmRelease - // - avoids false-negative Ready=False during normal startup windows where - // pods are still coming up but WorkloadMonitor has already reported - // Operational=false due to availableReplicas < MinReplicas - // WorkloadsReady is a separate condition that surfaces workload health - // independently — users and dashboards can observe it for operational visibility. - } - app.SetConditions(conditions) // Add namespace field for Tenant applications - if r.kindName == validation.TenantKind { + if r.kindName == "Tenant" { app.Status.Namespace = r.computeTenantNamespace(hr.Namespace, app.Name) externalIPsCount, err := r.countTenantExternalIPs(ctx, app.Status.Namespace) if err != nil { @@ -1413,79 +1124,6 @@ func (r *REST) convertHelmReleaseToApplication(ctx context.Context, hr *helmv2.H return app, nil } -// workloadsStatus holds the aggregated operational status of WorkloadMonitors. -type workloadsStatus struct { - operational bool - found bool - unknown bool // true when at least one monitor has nil Operational (not yet reconciled) - // transitionTime is the most recent metadata update time across the - // matching monitors. Used as WorkloadsReady.LastTransitionTime so that - // repeated conversions for the same underlying state produce stable - // timestamps (preserving the Kubernetes contract that identical - // resource versions represent identical content). - transitionTime metav1.Time -} - -// getWorkloadsOperational checks WorkloadMonitor resources for an application and returns -// aggregated operational status. If no monitors exist, returns found=false. -// When freshOverride is non-nil, its status replaces the cached copy for the -// corresponding monitor — this keeps the result consistent with watch events -// when the cache (r.c) lags behind the watch client (r.w). -func (r *REST) getWorkloadsOperational(ctx context.Context, namespace, appName string, freshOverride *cozyv1alpha1.WorkloadMonitor) (workloadsStatus, error) { - monitors := &cozyv1alpha1.WorkloadMonitorList{} - if err := r.c.List(ctx, monitors, - client.InNamespace(namespace), - client.MatchingLabels{ - appsv1alpha1.ApplicationKindLabel: r.kindName, - appsv1alpha1.ApplicationGroupLabel: r.gvk.Group, - appsv1alpha1.ApplicationNameLabel: appName, - }, - ); err != nil { - return workloadsStatus{}, err - } - // Ensure the freshOverride is represented in the aggregation even when - // the cache has not yet observed it (brand-new resource) or is behind. - replaced := false - if freshOverride != nil { - for i := range monitors.Items { - if monitors.Items[i].UID == freshOverride.UID || - (monitors.Items[i].Name == freshOverride.Name && monitors.Items[i].Namespace == freshOverride.Namespace) { - monitors.Items[i] = *freshOverride - replaced = true - break - } - } - if !replaced { - monitors.Items = append(monitors.Items, *freshOverride) - } - } - if len(monitors.Items) == 0 { - return workloadsStatus{operational: true, found: false}, nil - } - operational := true - unknown := false - var latest metav1.Time - for _, m := range monitors.Items { - if m.Status.Operational == nil { - unknown = true - } else if !*m.Status.Operational { - operational = false - } - // Pick the most recent monitor mtime as a stable transition time. - if t := latestMonitorTime(&m); t.After(latest.Time) { - latest = t - } - } - return workloadsStatus{operational: operational, found: true, unknown: unknown, transitionTime: latest}, nil -} - -// latestMonitorTime returns the most recent timestamp associated with a -// WorkloadMonitor — currently only the object creation time is guaranteed. -// Status does not carry a transition time, so we fall back to CreationTimestamp. -func latestMonitorTime(m *cozyv1alpha1.WorkloadMonitor) metav1.Time { - return m.CreationTimestamp -} - // convertApplicationToHelmRelease implements the actual conversion logic func (r *REST) convertApplicationToHelmRelease(app *appsv1alpha1.Application) (*helmv2.HelmRelease, error) { helmRelease := &helmv2.HelmRelease{ @@ -1528,24 +1166,6 @@ func (r *REST) convertApplicationToHelmRelease(app *appsv1alpha1.Application) (* }, } - // Per-Application HelmRelease wait budget. The mechanism is generic: - // an ApplicationDefinition that sets - // release.cozystack.io/helm-install-timeout gets Install.Timeout and - // Upgrade.Timeout populated from ReleaseConfig.HelmInstallTimeout - // (parsed at startup). Applications that leave it unset keep flux - // defaults so their failed installs remediate on the normal cadence. - // Today only kubernetes-rd carries the annotation because the - // Kubernetes Application's parent chart contains CAPI/Kamaji - // resources whose admin-kubeconfig Secret is provisioned - // asynchronously and Kamaji cold-start routinely exceeds flux's - // default wait budget; any future kind with the same shape can opt - // in by setting the same annotation. - if r.releaseConfig.HelmInstallTimeout > 0 { - timeout := metav1.Duration{Duration: r.releaseConfig.HelmInstallTimeout} - helmRelease.Spec.Install.Timeout = &timeout - helmRelease.Spec.Upgrade.Timeout = &timeout - } - return helmRelease, nil } diff --git a/pkg/registry/apps/application/rest_conditions_test.go b/pkg/registry/apps/application/rest_conditions_test.go deleted file mode 100644 index 185f59b2..00000000 --- a/pkg/registry/apps/application/rest_conditions_test.go +++ /dev/null @@ -1,331 +0,0 @@ -package application - -import ( - "context" - "testing" - - cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" - helmv2 "github.com/fluxcd/helm-controller/api/v2" - appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1" - "github.com/cozystack/cozystack/pkg/config" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/utils/ptr" - "sigs.k8s.io/controller-runtime/pkg/client/fake" -) - -func findCondition(conditions []metav1.Condition, condType string) *metav1.Condition { - for i := range conditions { - if conditions[i].Type == condType { - return &conditions[i] - } - } - return nil -} - -func makeHelmRelease(name, namespace string) *helmv2.HelmRelease { - hr := &helmv2.HelmRelease{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - Labels: map[string]string{ - ApplicationKindLabel: "PostgreSQL", - ApplicationGroupLabel: "apps.cozystack.io", - ApplicationNameLabel: "mydb", - }, - }, - } - return hr -} - -func TestConvertConditions_WorkloadsReadyAdded(t *testing.T) { - monitor := &cozyv1alpha1.WorkloadMonitor{ - ObjectMeta: metav1.ObjectMeta{ - Name: "mon-1", - Namespace: "default", - Labels: map[string]string{ - appsv1alpha1.ApplicationKindLabel: "PostgreSQL", - appsv1alpha1.ApplicationGroupLabel: "apps.cozystack.io", - appsv1alpha1.ApplicationNameLabel: "mydb", - }, - }, - Status: cozyv1alpha1.WorkloadMonitorStatus{ - Operational: ptr.To(true), - }, - } - - r := newTestRESTWithSchemes(monitor) - hr := makeHelmRelease("postgresql-mydb", "default") - hr.Status.Conditions = []metav1.Condition{ - {Type: "Ready", Status: metav1.ConditionTrue, Reason: "Succeeded", Message: "Release applied"}, - {Type: "Released", Status: metav1.ConditionTrue, Reason: "Succeeded", Message: "Released"}, - } - - app, err := r.convertHelmReleaseToApplication(context.TODO(), hr, nil) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - wc := findCondition(app.GetConditions(), "WorkloadsReady") - if wc == nil { - t.Fatal("expected WorkloadsReady condition to be present") - } - if wc.Status != metav1.ConditionTrue { - t.Errorf("expected WorkloadsReady=True, got %s", wc.Status) - } - - rc := findCondition(app.GetConditions(), "Ready") - if rc == nil { - t.Fatal("expected Ready condition to be present") - } - if rc.Status != metav1.ConditionTrue { - t.Errorf("expected Ready=True, got %s", rc.Status) - } -} - -func TestConvertConditions_ReadyNotOverriddenWhenWorkloadsNotReady(t *testing.T) { - // Ready must reflect HelmRelease state only. WorkloadsReady is a separate - // signal that users/dashboards can observe independently. - monitor := &cozyv1alpha1.WorkloadMonitor{ - ObjectMeta: metav1.ObjectMeta{ - Name: "mon-1", - Namespace: "default", - Labels: map[string]string{ - appsv1alpha1.ApplicationKindLabel: "PostgreSQL", - appsv1alpha1.ApplicationGroupLabel: "apps.cozystack.io", - appsv1alpha1.ApplicationNameLabel: "mydb", - }, - }, - Status: cozyv1alpha1.WorkloadMonitorStatus{ - Operational: ptr.To(false), - }, - } - - r := newTestRESTWithSchemes(monitor) - hr := makeHelmRelease("postgresql-mydb", "default") - hr.Status.Conditions = []metav1.Condition{ - {Type: "Ready", Status: metav1.ConditionTrue, Reason: "Succeeded", Message: "Release applied"}, - } - - app, err := r.convertHelmReleaseToApplication(context.TODO(), hr, nil) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - wc := findCondition(app.GetConditions(), "WorkloadsReady") - if wc == nil { - t.Fatal("expected WorkloadsReady condition to be present") - } - if wc.Status != metav1.ConditionFalse { - t.Errorf("expected WorkloadsReady=False, got %s", wc.Status) - } - - rc := findCondition(app.GetConditions(), "Ready") - if rc == nil { - t.Fatal("expected Ready condition to be present") - } - if rc.Status != metav1.ConditionTrue { - t.Errorf("expected Ready=True (reflects HelmRelease only), got %s", rc.Status) - } - if rc.Reason != "Succeeded" { - t.Errorf("expected Ready.Reason=Succeeded (unchanged), got %s", rc.Reason) - } -} - -func TestConvertConditions_NoOverrideWhenNoMonitors(t *testing.T) { - r := newTestRESTWithSchemes() - hr := makeHelmRelease("postgresql-mydb", "default") - hr.Status.Conditions = []metav1.Condition{ - {Type: "Ready", Status: metav1.ConditionTrue, Reason: "Succeeded", Message: "Release applied"}, - } - - app, err := r.convertHelmReleaseToApplication(context.TODO(), hr, nil) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - wc := findCondition(app.GetConditions(), "WorkloadsReady") - if wc != nil { - t.Error("expected no WorkloadsReady condition when no monitors exist") - } - - rc := findCondition(app.GetConditions(), "Ready") - if rc == nil { - t.Fatal("expected Ready condition to be present") - } - if rc.Status != metav1.ConditionTrue { - t.Errorf("expected Ready=True unchanged, got %s", rc.Status) - } -} - -func TestConvertConditions_ReadyStaysTrue_WhenAllOperational(t *testing.T) { - monitor := &cozyv1alpha1.WorkloadMonitor{ - ObjectMeta: metav1.ObjectMeta{ - Name: "mon-1", - Namespace: "default", - Labels: map[string]string{ - appsv1alpha1.ApplicationKindLabel: "PostgreSQL", - appsv1alpha1.ApplicationGroupLabel: "apps.cozystack.io", - appsv1alpha1.ApplicationNameLabel: "mydb", - }, - }, - Status: cozyv1alpha1.WorkloadMonitorStatus{ - Operational: ptr.To(true), - }, - } - - r := newTestRESTWithSchemes(monitor) - hr := makeHelmRelease("postgresql-mydb", "default") - hr.Status.Conditions = []metav1.Condition{ - {Type: "Ready", Status: metav1.ConditionTrue, Reason: "Succeeded", Message: "Release applied"}, - {Type: "Released", Status: metav1.ConditionTrue, Reason: "Succeeded", Message: "Released"}, - } - - app, err := r.convertHelmReleaseToApplication(context.TODO(), hr, nil) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - rc := findCondition(app.GetConditions(), "Ready") - if rc == nil { - t.Fatal("expected Ready condition") - } - if rc.Status != metav1.ConditionTrue { - t.Errorf("expected Ready=True when all workloads operational, got %s", rc.Status) - } - if rc.Reason != "Succeeded" { - t.Errorf("expected Ready.Reason=Succeeded (unchanged), got %s", rc.Reason) - } -} - -func TestConvertConditions_WorkloadsReadyTimestampIsNonZero(t *testing.T) { - monitor := &cozyv1alpha1.WorkloadMonitor{ - ObjectMeta: metav1.ObjectMeta{ - Name: "mon-1", - Namespace: "default", - Labels: map[string]string{ - appsv1alpha1.ApplicationKindLabel: "PostgreSQL", - appsv1alpha1.ApplicationGroupLabel: "apps.cozystack.io", - appsv1alpha1.ApplicationNameLabel: "mydb", - }, - }, - Status: cozyv1alpha1.WorkloadMonitorStatus{ - Operational: ptr.To(true), - }, - } - - r := newTestRESTWithSchemes(monitor) - hr := makeHelmRelease("postgresql-mydb", "default") - // No Ready condition — HR still being reconciled - hr.Status.Conditions = []metav1.Condition{} - - app, err := r.convertHelmReleaseToApplication(context.TODO(), hr, nil) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - wc := findCondition(app.GetConditions(), "WorkloadsReady") - if wc == nil { - t.Fatal("expected WorkloadsReady condition") - } - if wc.LastTransitionTime.IsZero() { - t.Error("expected non-zero LastTransitionTime") - } -} - -func TestConvertConditions_WorkloadsReadyUnknownWhenNilOperational(t *testing.T) { - monitor := &cozyv1alpha1.WorkloadMonitor{ - ObjectMeta: metav1.ObjectMeta{ - Name: "mon-1", - Namespace: "default", - Labels: map[string]string{ - appsv1alpha1.ApplicationKindLabel: "PostgreSQL", - appsv1alpha1.ApplicationGroupLabel: "apps.cozystack.io", - appsv1alpha1.ApplicationNameLabel: "mydb", - }, - }, - Status: cozyv1alpha1.WorkloadMonitorStatus{ - Operational: nil, // Not yet reconciled - }, - } - - r := newTestRESTWithSchemes(monitor) - hr := makeHelmRelease("postgresql-mydb", "default") - hr.Status.Conditions = []metav1.Condition{ - {Type: "Ready", Status: metav1.ConditionTrue, Reason: "Succeeded", Message: "ok"}, - } - - app, err := r.convertHelmReleaseToApplication(context.TODO(), hr, nil) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - wc := findCondition(app.GetConditions(), "WorkloadsReady") - if wc == nil { - t.Fatal("expected WorkloadsReady condition") - } - if wc.Status != metav1.ConditionUnknown { - t.Errorf("expected WorkloadsReady=Unknown for nil Operational, got %s", wc.Status) - } - - // Ready should NOT be overridden for unknown — prefer availability during startup - rc := findCondition(app.GetConditions(), "Ready") - if rc == nil { - t.Fatal("expected Ready condition") - } - if rc.Status != metav1.ConditionTrue { - t.Errorf("expected Ready=True when workloads unknown (pending), got %s", rc.Status) - } -} - -func TestConvertConditions_WorkloadsReadyUnknownOnError(t *testing.T) { - // Create a client with only HelmRelease scheme — WorkloadMonitor List will fail - scheme := runtime.NewScheme() - _ = helmv2.AddToScheme(scheme) - // Deliberately NOT registering cozyv1alpha1 so that List returns an error - - c := fake.NewClientBuilder().WithScheme(scheme).Build() - r := NewREST(c, nil, &config.Resource{ - Application: config.ApplicationConfig{ - Kind: "PostgreSQL", - Plural: "postgresqls", - Singular: "postgresql", - }, - Release: config.ReleaseConfig{ - Prefix: "postgresql-", - }, - }) - - hr := makeHelmRelease("postgresql-mydb", "default") - hr.CreationTimestamp = metav1.Now() - hr.Status.Conditions = []metav1.Condition{ - {Type: "Ready", Status: metav1.ConditionTrue, Reason: "Succeeded", Message: "ok"}, - } - - app, err := r.convertHelmReleaseToApplication(context.TODO(), hr, nil) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - wc := findCondition(app.GetConditions(), "WorkloadsReady") - if wc == nil { - t.Fatal("expected WorkloadsReady condition with Unknown status on error") - } - if wc.Status != metav1.ConditionUnknown { - t.Errorf("expected WorkloadsReady=Unknown, got %s", wc.Status) - } - if wc.Reason != "Error" { - t.Errorf("expected reason=Error, got %s", wc.Reason) - } - - // Ready should NOT be overridden on error (fail-open: prefer availability) - rc := findCondition(app.GetConditions(), "Ready") - if rc == nil { - t.Fatal("expected Ready condition") - } - if rc.Status != metav1.ConditionTrue { - t.Errorf("expected Ready=True (fail-open on error), got %s", rc.Status) - } -} - - diff --git a/pkg/registry/apps/application/rest_timeout_test.go b/pkg/registry/apps/application/rest_timeout_test.go deleted file mode 100644 index 85ac0b97..00000000 --- a/pkg/registry/apps/application/rest_timeout_test.go +++ /dev/null @@ -1,116 +0,0 @@ -package application - -import ( - "testing" - "time" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1" - "github.com/cozystack/cozystack/pkg/config" -) - -func newRESTForTimeout(kind, prefix string, helmInstallTimeout time.Duration) *REST { - return &REST{ - kindName: kind, - releaseConfig: config.ReleaseConfig{ - Prefix: prefix, - ChartRef: config.ChartRefConfig{ - Kind: "HelmChart", - Name: "x", - Namespace: "cozy-system", - }, - HelmInstallTimeout: helmInstallTimeout, - }, - } -} - -// Table-driven: every Application kind carries a per-CRD HelmRelease wait -// budget. The Kubernetes kind's parent chart contains CAPI/Kamaji resources -// whose admin-kubeconfig Secret is provisioned asynchronously, so its -// ApplicationDefinition sets release.cozystack.io/helm-install-timeout=15m -// (or longer). Other kinds leave the annotation unset and keep flux defaults -// so their failed installs remediate on the normal cadence. The test must -// cover both paths: a kind with the timeout set and one without. -func TestConvertApplicationToHelmRelease_AppliesReleaseConfigTimeout(t *testing.T) { - cases := []struct { - name string - kind string - prefix string - configured time.Duration - wantSet bool - }{ - { - name: "Kubernetes kind with 15m configured gets Install and Upgrade Timeout", - kind: "Kubernetes", - prefix: "kubernetes-", - configured: 15 * time.Minute, - wantSet: true, - }, - { - // Fictional kind on purpose: the test is about the unset path - // regardless of which real Application kind ends up needing a - // timeout override. Using a real kind name would create false - // coupling to that Application's ApplicationDefinition. - name: "unrelated kind without configured timeout keeps flux defaults", - kind: "PlaceholderKindForDefaults", - prefix: "placeholder-", - configured: 0, - wantSet: false, - }, - { - name: "arbitrary future kind with 20m configured gets 20m", - kind: "TalosCluster", - prefix: "talos-", - configured: 20 * time.Minute, - wantSet: true, - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - r := newRESTForTimeout(tc.kind, tc.prefix, tc.configured) - app := &appsv1alpha1.Application{ - ObjectMeta: metav1.ObjectMeta{Name: "example", Namespace: "tenant-root"}, - } - - hr, err := r.convertApplicationToHelmRelease(app) - if err != nil { - t.Fatalf("convertApplicationToHelmRelease returned error: %v", err) - } - - if hr.Spec.Install == nil || hr.Spec.Upgrade == nil { - t.Fatalf("Spec.Install/Upgrade must be non-nil") - } - - if tc.wantSet { - if hr.Spec.Install.Timeout == nil { - t.Fatalf("Spec.Install.Timeout must be set when HelmInstallTimeout=%v", tc.configured) - } - if hr.Spec.Install.Timeout.Duration != tc.configured { - t.Errorf("Spec.Install.Timeout = %v, want %v", hr.Spec.Install.Timeout.Duration, tc.configured) - } - if hr.Spec.Upgrade.Timeout == nil { - t.Fatalf("Spec.Upgrade.Timeout must be set when HelmInstallTimeout=%v", tc.configured) - } - if hr.Spec.Upgrade.Timeout.Duration != tc.configured { - t.Errorf("Spec.Upgrade.Timeout = %v, want %v", hr.Spec.Upgrade.Timeout.Duration, tc.configured) - } - } else { - if hr.Spec.Install.Timeout != nil { - t.Errorf("Spec.Install.Timeout must be nil when HelmInstallTimeout=0, got %v", hr.Spec.Install.Timeout.Duration) - } - if hr.Spec.Upgrade.Timeout != nil { - t.Errorf("Spec.Upgrade.Timeout must be nil when HelmInstallTimeout=0, got %v", hr.Spec.Upgrade.Timeout.Duration) - } - } - - if hr.Spec.Install.Remediation == nil || hr.Spec.Install.Remediation.Retries != -1 { - t.Errorf("Spec.Install.Remediation.Retries must remain -1, got %+v", hr.Spec.Install.Remediation) - } - if hr.Spec.Upgrade.Remediation == nil || hr.Spec.Upgrade.Remediation.Retries != -1 { - t.Errorf("Spec.Upgrade.Remediation.Retries must remain -1, got %+v", hr.Spec.Upgrade.Remediation) - } - }) - } -} diff --git a/pkg/registry/apps/application/rest_validation_test.go b/pkg/registry/apps/application/rest_validation_test.go index b806161c..c5d331b2 100644 --- a/pkg/registry/apps/application/rest_validation_test.go +++ b/pkg/registry/apps/application/rest_validation_test.go @@ -17,207 +17,12 @@ limitations under the License. package application import ( - "context" - "fmt" "strings" "testing" - helmv2 "github.com/fluxcd/helm-controller/api/v2" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apiserver/pkg/endpoints/request" - "k8s.io/apiserver/pkg/registry/rest" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - - appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1" - "github.com/cozystack/cozystack/pkg/apis/apps/validation" "github.com/cozystack/cozystack/pkg/config" ) -func TestValidateNameFormat(t *testing.T) { - tests := []struct { - name string - kindName string - appName string - wantError bool - }{ - // Non-tenant kinds follow DNS-1035 only — hyphens are allowed. - {"non-tenant accepts hyphen", "MySQL", "my-db", false}, - {"non-tenant accepts double hyphen", "MySQL", "my--db", false}, - {"non-tenant rejects uppercase", "MySQL", "MyDB", true}, - - // Tenant kind enforces alphanumeric-only — see - // packages/apps/tenant/templates/_helpers.tpl for the reason. - {"tenant accepts alphanumeric", validation.TenantKind, "foo", false}, - {"tenant accepts digits", validation.TenantKind, "foo123", false}, - {"tenant rejects single hyphen", validation.TenantKind, "foo-bar", true}, - {"tenant rejects leading hyphen", validation.TenantKind, "-foo", true}, - {"tenant rejects trailing hyphen", validation.TenantKind, "foo-", true}, - {"tenant rejects uppercase", validation.TenantKind, "Foo", true}, - {"tenant rejects underscore", validation.TenantKind, "foo_bar", true}, - {"tenant rejects empty", validation.TenantKind, "", true}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - r := &REST{kindName: tt.kindName} - - errs := r.validateNameFormat(tt.appName) - - if tt.wantError && len(errs) == 0 { - t.Errorf("expected error for name %q (kind=%q), got none", tt.appName, tt.kindName) - } - if !tt.wantError && len(errs) > 0 { - t.Errorf("unexpected error for name %q (kind=%q): %v", tt.appName, tt.kindName, errs) - } - }) - } -} - -// TestUpdate_ForceAllowCreate_RejectsTenantDashName pins the wiring from the -// Update → Create fall-through path. When a user runs `kubectl apply` and -// the object does not yet exist, Kubernetes routes the request through -// REST.Update with forceAllowCreate=true, which delegates to REST.Create -// (rest.go:452). Without this test, a future refactor could quietly reroute -// that delegation and bypass the tenant name check — unit tests of the -// pure r.validateNameFormat method would still pass while upsert-style -// kubectl apply regressed back to accepting tenant names with dashes. -func TestUpdate_ForceAllowCreate_RejectsTenantDashName(t *testing.T) { - scheme := runtime.NewScheme() - if err := helmv2.AddToScheme(scheme); err != nil { - t.Fatalf("register helmv2 scheme: %v", err) - } - // Register the dynamic Tenant kind so the Application type round-trips - // through the scheme the same way the real aggregated API server wires - // it at startup. - resourceCfg := &config.ResourceConfig{ - Resources: []config.Resource{ - { - Application: config.ApplicationConfig{Kind: validation.TenantKind}, - }, - }, - } - if err := appsv1alpha1.RegisterDynamicTypes(scheme, resourceCfg); err != nil { - t.Fatalf("register dynamic Tenant type: %v", err) - } - - fakeClient := fake.NewClientBuilder().WithScheme(scheme).Build() - - r := &REST{ - c: fakeClient, - gvr: schema.GroupVersionResource{ - Group: appsv1alpha1.GroupName, - Version: "v1alpha1", - Resource: "tenants", - }, - gvk: schema.GroupVersionKind{ - Group: appsv1alpha1.GroupName, - Version: "v1alpha1", - Kind: validation.TenantKind, - }, - kindName: validation.TenantKind, - releaseConfig: config.ReleaseConfig{ - Prefix: "tenant-", - }, - } - - newApp := &appsv1alpha1.Application{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "apps.cozystack.io/v1alpha1", - Kind: validation.TenantKind, - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "foo-bar", - Namespace: "tenant-root", - }, - } - - ctx := request.WithNamespace(context.Background(), "tenant-root") - - _, _, err := r.Update( - ctx, - "foo-bar", - rest.DefaultUpdatedObjectInfo(newApp), - nil, // createValidation - nil, // updateValidation - true, // forceAllowCreate → routes through Create on NotFound - &metav1.UpdateOptions{}, - ) - if err == nil { - t.Fatalf("expected Update to reject tenant name with dashes, got no error") - } - if !apierrors.IsInvalid(err) { - t.Errorf("expected Invalid status error, got %T: %v", err, err) - } - if !strings.Contains(err.Error(), "tenant names must") { - t.Errorf("expected tenant-specific error in %q", err.Error()) - } -} - -// TestConvertHelmReleaseToApplication_TenantNamespaceKindGate pins the -// behavior that convertHelmReleaseToApplication fills Status.Namespace only -// when the kind is Tenant. This path is gated on r.kindName matching a -// specific literal — the test uses the validation.TenantKind constant so -// that if the source of truth for the tenant kind string is ever renamed, -// the gate and the constant drift together (or the test fails). -func TestConvertHelmReleaseToApplication_TenantNamespaceKindGate(t *testing.T) { - scheme := runtime.NewScheme() - if err := helmv2.AddToScheme(scheme); err != nil { - t.Fatalf("register helmv2 scheme: %v", err) - } - if err := corev1.AddToScheme(scheme); err != nil { - t.Fatalf("register corev1 scheme: %v", err) - } - - fakeClient := fake.NewClientBuilder().WithScheme(scheme).Build() - - hr := &helmv2.HelmRelease{ - ObjectMeta: metav1.ObjectMeta{ - Name: "tenant-foo", - Namespace: "tenant-root", - }, - } - - t.Run("tenant kind fills Status.Namespace", func(t *testing.T) { - r := &REST{ - c: fakeClient, - kindName: validation.TenantKind, - releaseConfig: config.ReleaseConfig{ - Prefix: "tenant-", - }, - } - - app, err := r.convertHelmReleaseToApplication(context.Background(), hr, nil) - if err != nil { - t.Fatalf("convertHelmReleaseToApplication: %v", err) - } - if app.Status.Namespace == "" { - t.Errorf("expected Status.Namespace to be populated for tenant kind, got empty") - } - }) - - t.Run("non-tenant kind leaves Status.Namespace empty", func(t *testing.T) { - r := &REST{ - c: fakeClient, - kindName: "MySQL", - releaseConfig: config.ReleaseConfig{ - Prefix: "mysql-", - }, - } - - app, err := r.convertHelmReleaseToApplication(context.Background(), hr, nil) - if err != nil { - t.Fatalf("convertHelmReleaseToApplication: %v", err) - } - if app.Status.Namespace != "" { - t.Errorf("expected Status.Namespace to be empty for non-tenant kind, got %q", app.Status.Namespace) - } - }) -} - func TestValidateNameLength(t *testing.T) { tests := []struct { name string @@ -297,101 +102,3 @@ func TestValidateNameLength(t *testing.T) { }) } } - -// TestValidateTenantNamespaceLength covers the check that the computed -// workload namespace for a Tenant application fits inside the 63-char -// DNS-1123 label limit. The namespace is formed by dash-joining the -// parent namespace with the tenant name, so deep nesting can exceed the -// limit even when each individual name passes the per-name Helm release -// length check. -// -// The "tenant-root" branches of computeTenantNamespace do not get a -// dedicated overflow case: that branch produces "tenant-" (7 + -// len(name)), so for the computed result to exceed 63 chars the name -// would need to exceed 56 chars, which is already blocked by -// validateNameLength (max 46 for the "tenant-" prefix). The invariant -// holds by arithmetic. -func TestValidateTenantNamespaceLength(t *testing.T) { - tests := []struct { - name string - currentNamespace string - tenantName string - wantError bool - }{ - { - name: "tenant-root with short name passes", - currentNamespace: "tenant-root", - tenantName: "alpha", - wantError: false, - }, - { - name: "root tenant inside root namespace passes", - currentNamespace: "tenant-root", - tenantName: "root", - wantError: false, - }, - { - name: "short parent and short name passes", - currentNamespace: "tenant-foo", - tenantName: "bar", - wantError: false, - }, - { - name: "exactly at 63-char limit passes", - currentNamespace: "tenant-" + strings.Repeat("a", 45), // 52 chars - tenantName: strings.Repeat("b", 10), // 10 chars -> 52+1+10 = 63 - wantError: false, - }, - { - name: "one char over the limit fails", - currentNamespace: "tenant-" + strings.Repeat("a", 45), // 52 chars - tenantName: strings.Repeat("b", 11), // 11 chars -> 52+1+11 = 64 - wantError: true, - }, - { - name: "deeply nested failure with issue-style names", - currentNamespace: "tenant-" + strings.Repeat("a", 35), // 42 chars - tenantName: strings.Repeat("b", 25), // 25 chars -> 42+1+25 = 68 - wantError: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - r := &REST{ - kindName: "Tenant", - releaseConfig: config.ReleaseConfig{ - Prefix: "tenant-", - }, - } - - errs := r.validateTenantNamespaceLength(tt.currentNamespace, tt.tenantName) - - if tt.wantError && len(errs) == 0 { - computed := r.computeTenantNamespace(tt.currentNamespace, tt.tenantName) - t.Errorf("expected error for parent=%q name=%q (computed=%q, len=%d), got none", - tt.currentNamespace, tt.tenantName, computed, len(computed)) - return - } - if !tt.wantError && len(errs) > 0 { - t.Errorf("unexpected error for parent=%q name=%q: %v", - tt.currentNamespace, tt.tenantName, errs) - return - } - - // For failing cases, verify the error message surfaces the - // computed namespace string and its actual length so a - // regression in the message format is caught. - if tt.wantError { - computed := r.computeTenantNamespace(tt.currentNamespace, tt.tenantName) - msg := errs.ToAggregate().Error() - if !strings.Contains(msg, computed) { - t.Errorf("error message must contain computed namespace %q, got: %s", computed, msg) - } - if !strings.Contains(msg, fmt.Sprintf("%d characters", len(computed))) { - t.Errorf("error message must contain the computed length %d, got: %s", len(computed), msg) - } - } - }) - } -} diff --git a/pkg/registry/apps/application/rest_watch_test.go b/pkg/registry/apps/application/rest_watch_test.go deleted file mode 100644 index 93a51f3c..00000000 --- a/pkg/registry/apps/application/rest_watch_test.go +++ /dev/null @@ -1,223 +0,0 @@ -package application - -import ( - "context" - "testing" - "time" - - cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" - appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1" - "github.com/cozystack/cozystack/pkg/config" - helmv2 "github.com/fluxcd/helm-controller/api/v2" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/watch" - "k8s.io/utils/ptr" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" -) - -func TestWmResultChan_NilWatcher(t *testing.T) { - ch := wmResultChan(nil) - if ch != nil { - t.Error("expected nil channel for nil watcher") - } -} - -func TestWmResultChan_ValidWatcher(t *testing.T) { - fw := watch.NewFake() - ch := wmResultChan(fw) - if ch == nil { - t.Error("expected non-nil channel for valid watcher") - } - fw.Stop() -} - -// TestWatchIntegration_WorkloadMonitorTriggersModifiedEvent verifies the -// full path: WM event → label lookup → HelmRelease Get → Application conversion. -func TestWatchIntegration_WorkloadMonitorTriggersModifiedEvent(t *testing.T) { - scheme := runtime.NewScheme() - _ = cozyv1alpha1.AddToScheme(scheme) - _ = helmv2.AddToScheme(scheme) - - hr := &helmv2.HelmRelease{ - ObjectMeta: metav1.ObjectMeta{ - Name: "postgresql-mydb", - Namespace: "default", - Labels: map[string]string{ - ApplicationKindLabel: "PostgreSQL", - ApplicationGroupLabel: "apps.cozystack.io", - ApplicationNameLabel: "mydb", - }, - }, - } - hr.Status.Conditions = []metav1.Condition{ - {Type: "Ready", Status: metav1.ConditionTrue, Reason: "Succeeded", Message: "ok"}, - } - - wm := &cozyv1alpha1.WorkloadMonitor{ - ObjectMeta: metav1.ObjectMeta{ - Name: "mon-1", - Namespace: "default", - Labels: map[string]string{ - appsv1alpha1.ApplicationKindLabel: "PostgreSQL", - appsv1alpha1.ApplicationGroupLabel: "apps.cozystack.io", - appsv1alpha1.ApplicationNameLabel: "mydb", - }, - }, - Status: cozyv1alpha1.WorkloadMonitorStatus{ - Operational: ptr.To(false), - }, - } - - c := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(hr, wm).Build() - - r := newTestRESTWithSchemesFromClient(c) - - // Simulate the Watch goroutine path: extract app name from WM labels, - // construct HelmRelease name, look up HR, convert to Application - wmAppName := wm.Labels[ApplicationNameLabel] - hrName := r.releaseConfig.Prefix + wmAppName - - foundHR := &helmv2.HelmRelease{} - if err := c.Get(context.TODO(), types.NamespacedName{Namespace: "default", Name: hrName}, foundHR); err != nil { - t.Fatalf("failed to get HelmRelease: %v", err) - } - app, err := r.ConvertHelmReleaseToApplication(context.TODO(), foundHR) - if err != nil { - t.Fatalf("failed to convert: %v", err) - } - if app.Name != "mydb" { - t.Errorf("expected app name 'mydb', got %q", app.Name) - } - - // Verify WorkloadsReady is False due to non-operational WM - wc := findCondition(app.GetConditions(), "WorkloadsReady") - if wc == nil { - t.Fatal("expected WorkloadsReady condition") - } - if wc.Status != metav1.ConditionFalse { - t.Errorf("expected WorkloadsReady=False, got %s", wc.Status) - } -} - -// TestWatchIntegration_MonitorDeletionDropsWorkloadsReady verifies that when a -// WorkloadMonitor is deleted, the Application's WorkloadsReady condition -// disappears. Ready condition always reflects HelmRelease state regardless. -func TestWatchIntegration_MonitorDeletionDropsWorkloadsReady(t *testing.T) { - scheme := runtime.NewScheme() - _ = cozyv1alpha1.AddToScheme(scheme) - _ = helmv2.AddToScheme(scheme) - - hr := &helmv2.HelmRelease{ - ObjectMeta: metav1.ObjectMeta{ - Name: "postgresql-mydb", - Namespace: "default", - Labels: map[string]string{ - ApplicationKindLabel: "PostgreSQL", - ApplicationGroupLabel: "apps.cozystack.io", - ApplicationNameLabel: "mydb", - }, - }, - } - hr.Status.Conditions = []metav1.Condition{ - {Type: "Ready", Status: metav1.ConditionTrue, Reason: "Succeeded", Message: "ok"}, - } - - nonOpMonitor := &cozyv1alpha1.WorkloadMonitor{ - ObjectMeta: metav1.ObjectMeta{ - Name: "mon-1", - Namespace: "default", - Labels: map[string]string{ - appsv1alpha1.ApplicationKindLabel: "PostgreSQL", - appsv1alpha1.ApplicationGroupLabel: "apps.cozystack.io", - appsv1alpha1.ApplicationNameLabel: "mydb", - }, - }, - Status: cozyv1alpha1.WorkloadMonitorStatus{ - Operational: ptr.To(false), - }, - } - - c := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(hr, nonOpMonitor).Build() - r := newTestRESTWithSchemesFromClient(c) - - // Step 1: With non-operational monitor, WorkloadsReady=False, Ready=True - app1, err := r.convertHelmReleaseToApplication(context.TODO(), hr, nil) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - wc1 := findCondition(app1.GetConditions(), "WorkloadsReady") - if wc1 == nil || wc1.Status != metav1.ConditionFalse { - t.Fatalf("expected WorkloadsReady=False with non-operational monitor, got %v", wc1) - } - rc1 := findCondition(app1.GetConditions(), "Ready") - if rc1 == nil || rc1.Status != metav1.ConditionTrue { - t.Fatalf("expected Ready=True (reflects HelmRelease), got %v", rc1) - } - - // Step 2: Delete the monitor - if err := c.Delete(context.TODO(), nonOpMonitor); err != nil { - t.Fatalf("failed to delete monitor: %v", err) - } - - // Step 3: WorkloadsReady should disappear, Ready stays True - app2, err := r.convertHelmReleaseToApplication(context.TODO(), hr, nil) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - wc2 := findCondition(app2.GetConditions(), "WorkloadsReady") - if wc2 != nil { - t.Error("expected no WorkloadsReady condition after monitor deletion") - } - rc2 := findCondition(app2.GetConditions(), "Ready") - if rc2 == nil { - t.Fatal("expected Ready condition") - } - if rc2.Status != metav1.ConditionTrue { - t.Errorf("expected Ready=True after monitor deletion, got %s", rc2.Status) - } -} - -// TestWatchIntegration_WMWatcherCloseProducesNilChannel verifies that -// after wmWatcher is set to nil, wmResultChan returns nil channel. -func TestWatchIntegration_WMWatcherCloseProducesNilChannel(t *testing.T) { - fw := watch.NewFake() - ch := wmResultChan(fw) - if ch == nil { - t.Fatal("expected non-nil channel before close") - } - - fw.Stop() - - timeout := time.After(time.Second) - for { - select { - case _, ok := <-ch: - if !ok { - var nilWatcher watch.Interface - nilCh := wmResultChan(nilWatcher) - if nilCh != nil { - t.Error("expected nil channel after watcher set to nil") - } - return - } - case <-timeout: - t.Fatal("timeout waiting for watcher channel to close") - } - } -} - -func newTestRESTWithSchemesFromClient(c client.Client) *REST { - return NewREST(c, nil, &config.Resource{ - Application: config.ApplicationConfig{ - Kind: "PostgreSQL", - Plural: "postgresqls", - Singular: "postgresql", - }, - Release: config.ReleaseConfig{ - Prefix: "postgresql-", - }, - }) -} diff --git a/pkg/registry/apps/application/rest_workloads_test.go b/pkg/registry/apps/application/rest_workloads_test.go deleted file mode 100644 index b706c323..00000000 --- a/pkg/registry/apps/application/rest_workloads_test.go +++ /dev/null @@ -1,342 +0,0 @@ -package application - -import ( - "context" - "testing" - - cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" - appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1" - "github.com/cozystack/cozystack/pkg/config" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/utils/ptr" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - - helmv2 "github.com/fluxcd/helm-controller/api/v2" -) - -func newTestRESTWithSchemes(objs ...runtime.Object) *REST { - scheme := runtime.NewScheme() - _ = cozyv1alpha1.AddToScheme(scheme) - _ = helmv2.AddToScheme(scheme) - - builder := fake.NewClientBuilder().WithScheme(scheme) - for _, obj := range objs { - builder = builder.WithRuntimeObjects(obj) - } - c := builder.Build() - - return NewREST(c, nil, &config.Resource{ - Application: config.ApplicationConfig{ - Kind: "PostgreSQL", - Plural: "postgresqls", - Singular: "postgresql", - }, - Release: config.ReleaseConfig{ - Prefix: "postgresql-", - }, - }) -} - -func TestGetWorkloadsOperational_NoMonitors(t *testing.T) { - r := newTestRESTWithSchemes() - ws, err := r.getWorkloadsOperational(context.TODO(), "default", "mydb", nil) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if ws.found { - t.Error("expected found=false when no monitors exist") - } - if !ws.operational { - t.Error("expected operational=true when no monitors exist") - } -} - -func TestGetWorkloadsOperational_AllOperational(t *testing.T) { - m1 := &cozyv1alpha1.WorkloadMonitor{ - ObjectMeta: metav1.ObjectMeta{ - Name: "mon-1", - Namespace: "default", - Labels: map[string]string{ - appsv1alpha1.ApplicationKindLabel: "PostgreSQL", - appsv1alpha1.ApplicationGroupLabel: "apps.cozystack.io", - appsv1alpha1.ApplicationNameLabel: "mydb", - }, - }, - Status: cozyv1alpha1.WorkloadMonitorStatus{ - Operational: ptr.To(true), - }, - } - m2 := &cozyv1alpha1.WorkloadMonitor{ - ObjectMeta: metav1.ObjectMeta{ - Name: "mon-2", - Namespace: "default", - Labels: map[string]string{ - appsv1alpha1.ApplicationKindLabel: "PostgreSQL", - appsv1alpha1.ApplicationGroupLabel: "apps.cozystack.io", - appsv1alpha1.ApplicationNameLabel: "mydb", - }, - }, - Status: cozyv1alpha1.WorkloadMonitorStatus{ - Operational: ptr.To(true), - }, - } - - r := newTestRESTWithSchemes(m1, m2) - ws, err := r.getWorkloadsOperational(context.TODO(), "default", "mydb", nil) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if !ws.found { - t.Error("expected found=true") - } - if !ws.operational { - t.Error("expected operational=true when all monitors are operational") - } -} - -func TestGetWorkloadsOperational_SomeNotOperational(t *testing.T) { - m1 := &cozyv1alpha1.WorkloadMonitor{ - ObjectMeta: metav1.ObjectMeta{ - Name: "mon-1", - Namespace: "default", - Labels: map[string]string{ - appsv1alpha1.ApplicationKindLabel: "PostgreSQL", - appsv1alpha1.ApplicationGroupLabel: "apps.cozystack.io", - appsv1alpha1.ApplicationNameLabel: "mydb", - }, - }, - Status: cozyv1alpha1.WorkloadMonitorStatus{ - Operational: ptr.To(true), - }, - } - m2 := &cozyv1alpha1.WorkloadMonitor{ - ObjectMeta: metav1.ObjectMeta{ - Name: "mon-2", - Namespace: "default", - Labels: map[string]string{ - appsv1alpha1.ApplicationKindLabel: "PostgreSQL", - appsv1alpha1.ApplicationGroupLabel: "apps.cozystack.io", - appsv1alpha1.ApplicationNameLabel: "mydb", - }, - }, - Status: cozyv1alpha1.WorkloadMonitorStatus{ - Operational: ptr.To(false), - }, - } - - r := newTestRESTWithSchemes(m1, m2) - ws, err := r.getWorkloadsOperational(context.TODO(), "default", "mydb", nil) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if !ws.found { - t.Error("expected found=true") - } - if ws.operational { - t.Error("expected operational=false when at least one monitor is not operational") - } -} - -func TestGetWorkloadsOperational_OperationalNil(t *testing.T) { - m := &cozyv1alpha1.WorkloadMonitor{ - ObjectMeta: metav1.ObjectMeta{ - Name: "mon-1", - Namespace: "default", - Labels: map[string]string{ - appsv1alpha1.ApplicationKindLabel: "PostgreSQL", - appsv1alpha1.ApplicationGroupLabel: "apps.cozystack.io", - appsv1alpha1.ApplicationNameLabel: "mydb", - }, - }, - Status: cozyv1alpha1.WorkloadMonitorStatus{ - Operational: nil, // Not yet reconciled - }, - } - - r := newTestRESTWithSchemes(m) - ws, err := r.getWorkloadsOperational(context.TODO(), "default", "mydb", nil) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if !ws.found { - t.Error("expected found=true") - } - if !ws.unknown { - t.Error("expected unknown=true when Operational is nil") - } -} - -func TestGetWorkloadsOperational_MixedNilAndOperational(t *testing.T) { - m1 := &cozyv1alpha1.WorkloadMonitor{ - ObjectMeta: metav1.ObjectMeta{ - Name: "mon-1", - Namespace: "default", - Labels: map[string]string{ - appsv1alpha1.ApplicationKindLabel: "PostgreSQL", - appsv1alpha1.ApplicationGroupLabel: "apps.cozystack.io", - appsv1alpha1.ApplicationNameLabel: "mydb", - }, - }, - Status: cozyv1alpha1.WorkloadMonitorStatus{ - Operational: ptr.To(true), - }, - } - m2 := &cozyv1alpha1.WorkloadMonitor{ - ObjectMeta: metav1.ObjectMeta{ - Name: "mon-2", - Namespace: "default", - Labels: map[string]string{ - appsv1alpha1.ApplicationKindLabel: "PostgreSQL", - appsv1alpha1.ApplicationGroupLabel: "apps.cozystack.io", - appsv1alpha1.ApplicationNameLabel: "mydb", - }, - }, - Status: cozyv1alpha1.WorkloadMonitorStatus{ - Operational: nil, - }, - } - - r := newTestRESTWithSchemes(m1, m2) - ws, err := r.getWorkloadsOperational(context.TODO(), "default", "mydb", nil) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if !ws.unknown { - t.Error("expected unknown=true when at least one monitor has nil Operational") - } -} - -func TestGetWorkloadsOperational_MixedFailedAndPending(t *testing.T) { - m1 := &cozyv1alpha1.WorkloadMonitor{ - ObjectMeta: metav1.ObjectMeta{ - Name: "mon-1", - Namespace: "default", - Labels: map[string]string{ - appsv1alpha1.ApplicationKindLabel: "PostgreSQL", - appsv1alpha1.ApplicationGroupLabel: "apps.cozystack.io", - appsv1alpha1.ApplicationNameLabel: "mydb", - }, - }, - Status: cozyv1alpha1.WorkloadMonitorStatus{ - Operational: ptr.To(false), // Confirmed failure - }, - } - m2 := &cozyv1alpha1.WorkloadMonitor{ - ObjectMeta: metav1.ObjectMeta{ - Name: "mon-2", - Namespace: "default", - Labels: map[string]string{ - appsv1alpha1.ApplicationKindLabel: "PostgreSQL", - appsv1alpha1.ApplicationGroupLabel: "apps.cozystack.io", - appsv1alpha1.ApplicationNameLabel: "mydb", - }, - }, - Status: cozyv1alpha1.WorkloadMonitorStatus{ - Operational: nil, // Not yet reconciled - }, - } - - r := newTestRESTWithSchemes(m1, m2) - ws, err := r.getWorkloadsOperational(context.TODO(), "default", "mydb", nil) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if ws.operational { - t.Error("expected operational=false when at least one monitor is explicitly failed") - } - if !ws.unknown { - t.Error("expected unknown=true when at least one monitor has nil Operational") - } -} - -func TestConvertConditions_MixedFailedAndPendingShowsFalse(t *testing.T) { - mFailed := &cozyv1alpha1.WorkloadMonitor{ - ObjectMeta: metav1.ObjectMeta{ - Name: "mon-failed", - Namespace: "default", - Labels: map[string]string{ - appsv1alpha1.ApplicationKindLabel: "PostgreSQL", - appsv1alpha1.ApplicationGroupLabel: "apps.cozystack.io", - appsv1alpha1.ApplicationNameLabel: "mydb", - }, - }, - Status: cozyv1alpha1.WorkloadMonitorStatus{ - Operational: ptr.To(false), - }, - } - mPending := &cozyv1alpha1.WorkloadMonitor{ - ObjectMeta: metav1.ObjectMeta{ - Name: "mon-pending", - Namespace: "default", - Labels: map[string]string{ - appsv1alpha1.ApplicationKindLabel: "PostgreSQL", - appsv1alpha1.ApplicationGroupLabel: "apps.cozystack.io", - appsv1alpha1.ApplicationNameLabel: "mydb", - }, - }, - Status: cozyv1alpha1.WorkloadMonitorStatus{ - Operational: nil, - }, - } - - r := newTestRESTWithSchemes(mFailed, mPending) - hr := &helmv2.HelmRelease{ - ObjectMeta: metav1.ObjectMeta{ - Name: "postgresql-mydb", - Namespace: "default", - Labels: map[string]string{ - ApplicationKindLabel: "PostgreSQL", - ApplicationGroupLabel: "apps.cozystack.io", - ApplicationNameLabel: "mydb", - }, - }, - } - hr.Status.Conditions = []metav1.Condition{ - {Type: "Ready", Status: metav1.ConditionTrue, Reason: "Succeeded", Message: "ok"}, - } - - app, err := r.convertHelmReleaseToApplication(context.TODO(), hr, nil) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - // Concrete failure should take priority over unknown - wc := findCondition(app.GetConditions(), "WorkloadsReady") - if wc == nil { - t.Fatal("expected WorkloadsReady condition") - } - if wc.Status != metav1.ConditionFalse { - t.Errorf("expected WorkloadsReady=False (failure takes priority over unknown), got %s", wc.Status) - } -} - -func TestGetWorkloadsOperational_DifferentApp_NotFound(t *testing.T) { - m := &cozyv1alpha1.WorkloadMonitor{ - ObjectMeta: metav1.ObjectMeta{ - Name: "mon-1", - Namespace: "default", - Labels: map[string]string{ - appsv1alpha1.ApplicationKindLabel: "PostgreSQL", - appsv1alpha1.ApplicationGroupLabel: "apps.cozystack.io", - appsv1alpha1.ApplicationNameLabel: "other-db", - }, - }, - Status: cozyv1alpha1.WorkloadMonitorStatus{ - Operational: ptr.To(false), - }, - } - - r := newTestRESTWithSchemes(m) - ws, err := r.getWorkloadsOperational(context.TODO(), "default", "mydb", nil) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if ws.found { - t.Error("expected found=false for different app name") - } - if !ws.operational { - t.Error("expected operational=true when no matching monitors found") - } -} - diff --git a/pkg/registry/core/tenantnamespace/rest.go b/pkg/registry/core/tenantnamespace/rest.go index 4ee97eb0..f1ed3fab 100644 --- a/pkg/registry/core/tenantnamespace/rest.go +++ b/pkg/registry/core/tenantnamespace/rest.go @@ -17,8 +17,6 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metainternal "k8s.io/apimachinery/pkg/apis/meta/internalversion" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" @@ -26,7 +24,6 @@ import ( "k8s.io/apimachinery/pkg/watch" "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/registry/rest" - "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" corev1alpha1 "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1" @@ -126,18 +123,8 @@ func (r *REST) Get( return nil, apierrors.NewNotFound(r.gvr.GroupResource(), name) } - // Check if user has access to this namespace - hasAccess, err := r.hasAccessToNamespace(ctx, name) - if err != nil { - return nil, err - } - if !hasAccess { - // Return Forbidden to follow standard K8s RBAC behavior - return nil, apierrors.NewForbidden(r.gvr.GroupResource(), name, fmt.Errorf("access denied")) - } - ns := &corev1.Namespace{} - err = r.c.Get(ctx, types.NamespacedName{Namespace: "", Name: name}, ns, &client.GetOptions{Raw: opts}) + err := r.c.Get(ctx, types.NamespacedName{Namespace: "", Name: name}, ns, &client.GetOptions{Raw: opts}) if err != nil { return nil, err } @@ -156,33 +143,11 @@ func (r *REST) Get( // ----------------------------------------------------------------------------- func (r *REST) Watch(ctx context.Context, opts *metainternal.ListOptions) (watch.Interface, error) { - // Extract user identity once for the lifetime of the watch — it does not - // change between events and rebuilding it per event is wasteful. - u, ok := request.UserFrom(ctx) - if !ok { - return nil, apierrors.NewUnauthorized("user missing in context") - } - username := u.GetName() - groups := make(map[string]struct{}) - for _, group := range u.GetGroups() { - groups[group] = struct{}{} - } - nsList := &corev1.NamespaceList{} - - // Build upstream watch options with field and label selectors - rawOpts := &metav1.ListOptions{ + nsWatch, err := r.w.Watch(ctx, nsList, &client.ListOptions{Raw: &metav1.ListOptions{ Watch: true, ResourceVersion: opts.ResourceVersion, - } - if opts.FieldSelector != nil { - rawOpts.FieldSelector = opts.FieldSelector.String() - } - if opts.LabelSelector != nil { - rawOpts.LabelSelector = opts.LabelSelector.String() - } - - nsWatch, err := r.w.Watch(ctx, nsList, &client.ListOptions{Raw: rawOpts}) + }}) if err != nil { return nil, err } @@ -224,30 +189,6 @@ func (r *REST) Watch(ctx context.Context, opts *metainternal.ListOptions) (watch continue } - // Apply defensive filtering for field and label selectors - if opts.FieldSelector != nil { - if !opts.FieldSelector.Matches(fields.Set{"metadata.name": ns.Name}) { - continue - } - } - if opts.LabelSelector != nil { - if !opts.LabelSelector.Matches(labels.Set(ns.Labels)) { - continue - } - } - - // Check if user has access to this namespace using the cached - // identity — avoids re-extracting user/groups on every event. - hasAccess, err := r.hasAccessToNamespaceForUser(ctx, ns.Name, username, groups) - if err != nil { - klog.ErrorS(err, "Failed to check access for namespace in watch", "namespace", ns.Name) - continue - } - if !hasAccess { - // User doesn't have access, skip this event - continue - } - out := &corev1alpha1.TenantNamespace{ TypeMeta: metav1.TypeMeta{ APIVersion: corev1alpha1.SchemeGroupVersion.String(), @@ -368,26 +309,6 @@ func (r *REST) makeList(src *corev1.NamespaceList, allowed []string) *corev1alph return out } -// matchesSubject checks if a RoleBinding subject matches the user's identity. -// It handles Group, User, and ServiceAccount subjects with proper namespace fallback. -func matchesSubject(subj rbacv1.Subject, bindingNamespace, username string, groups map[string]struct{}) bool { - switch subj.Kind { - case "Group": - _, ok := groups[subj.Name] - return ok - case "User": - return subj.Name == username - case "ServiceAccount": - saNamespace := subj.Namespace - if saNamespace == "" { - saNamespace = bindingNamespace - } - return username == fmt.Sprintf("system:serviceaccount:%s:%s", saNamespace, subj.Name) - default: - return false - } -} - func (r *REST) filterAccessible( ctx context.Context, names []string, @@ -426,9 +347,22 @@ func (r *REST) filterAccessible( subjectLoop: for j := range rbs.Items[i].Subjects { subj := rbs.Items[i].Subjects[j] - if matchesSubject(subj, rbs.Items[i].Namespace, u.GetName(), groups) { - allowedNameSet[rbs.Items[i].Namespace] = struct{}{} - break subjectLoop + switch subj.Kind { + case "Group": + if _, ok = groups[subj.Name]; ok { + allowedNameSet[rbs.Items[i].Namespace] = struct{}{} + break subjectLoop + } + case "User": + if subj.Name == u.GetName() { + allowedNameSet[rbs.Items[i].Namespace] = struct{}{} + break subjectLoop + } + case "ServiceAccount": + if u.GetName() == fmt.Sprintf("system:serviceaccount:%s:%s", subj.Namespace, subj.Name) { + allowedNameSet[rbs.Items[i].Namespace] = struct{}{} + break subjectLoop + } } } } @@ -439,60 +373,6 @@ func (r *REST) filterAccessible( return allowed, nil } -// hasAccessToNamespace checks if the user has access to a single namespace. -// This is optimized for Get/Watch operations where we check one namespace at a time. -// It lists RoleBindings only in the target namespace instead of all cluster RoleBindings. -func (r *REST) hasAccessToNamespace( - ctx context.Context, - namespace string, -) (bool, error) { - u, ok := request.UserFrom(ctx) - if !ok { - return false, fmt.Errorf("user missing in context") - } - groups := make(map[string]struct{}) - for _, group := range u.GetGroups() { - groups[group] = struct{}{} - } - return r.hasAccessToNamespaceForUser(ctx, namespace, u.GetName(), groups) -} - -// hasAccessToNamespaceForUser is the inner check that does not re-extract user -// identity from context. Use this in hot paths (e.g. the Watch loop) where the -// caller has already cached the user name and groups. -func (r *REST) hasAccessToNamespaceForUser( - ctx context.Context, - namespace, username string, - groups map[string]struct{}, -) (bool, error) { - // Check privileged groups - if _, ok := groups["system:masters"]; ok { - return true, nil - } - if _, ok := groups["cozystack-cluster-admin"]; ok { - return true, nil - } - - // List RoleBindings only in the target namespace - rbs := &rbacv1.RoleBindingList{} - err := r.c.List(ctx, rbs, client.InNamespace(namespace)) - if err != nil { - return false, fmt.Errorf("failed to list rolebindings in %s: %w", namespace, err) - } - - // Check if user is in any RoleBinding subjects - for i := range rbs.Items { - for j := range rbs.Items[i].Subjects { - subj := rbs.Items[i].Subjects[j] - if matchesSubject(subj, rbs.Items[i].Namespace, username, groups) { - return true, nil - } - } - } - - return false, nil -} - // ----------------------------------------------------------------------------- // Boiler-plate // ----------------------------------------------------------------------------- diff --git a/pkg/registry/core/tenantnamespace/rest_test.go b/pkg/registry/core/tenantnamespace/rest_test.go index 52d2a075..7f2979bc 100644 --- a/pkg/registry/core/tenantnamespace/rest_test.go +++ b/pkg/registry/core/tenantnamespace/rest_test.go @@ -3,20 +3,10 @@ package tenantnamespace import ( - "context" "testing" corev1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apiserver/pkg/authentication/user" - "k8s.io/apiserver/pkg/endpoints/request" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - - corev1alpha1 "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1" ) func TestMakeListSortsAlphabetically(t *testing.T) { @@ -48,466 +38,3 @@ func TestMakeListSortsAlphabetically(t *testing.T) { } } } - -// Security tests for IDOR fix - -func TestHasAccessToNamespace_WithUserAccess(t *testing.T) { - scheme := runtime.NewScheme() - _ = corev1.AddToScheme(scheme) - _ = rbacv1.AddToScheme(scheme) - - ns := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"}, - } - - rb := &rbacv1.RoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-binding", - Namespace: "tenant-test", - }, - Subjects: []rbacv1.Subject{ - {Kind: "User", Name: "test-user", APIGroup: "rbac.authorization.k8s.io"}, - }, - RoleRef: rbacv1.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "Role", - Name: "test-role", - }, - } - - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(ns, rb). - Build() - - r := &REST{ - c: client, - gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"}, - } - - u := &user.DefaultInfo{ - Name: "test-user", - Groups: []string{"system:authenticated"}, - } - ctx := request.WithUser(context.Background(), u) - - hasAccess, err := r.hasAccessToNamespace(ctx, "tenant-test") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if !hasAccess { - t.Error("expected user to have access, but got false") - } -} - -func TestHasAccessToNamespace_WithoutAccess(t *testing.T) { - scheme := runtime.NewScheme() - _ = corev1.AddToScheme(scheme) - _ = rbacv1.AddToScheme(scheme) - - ns := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"}, - } - - // RoleBinding for different user - rb := &rbacv1.RoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-binding", - Namespace: "tenant-test", - }, - Subjects: []rbacv1.Subject{ - {Kind: "User", Name: "other-user", APIGroup: "rbac.authorization.k8s.io"}, - }, - RoleRef: rbacv1.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "Role", - Name: "test-role", - }, - } - - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(ns, rb). - Build() - - r := &REST{ - c: client, - gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"}, - } - - u := &user.DefaultInfo{ - Name: "test-user", - Groups: []string{"system:authenticated"}, - } - ctx := request.WithUser(context.Background(), u) - - hasAccess, err := r.hasAccessToNamespace(ctx, "tenant-test") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if hasAccess { - t.Error("expected user to NOT have access, but got true") - } -} - -func TestHasAccessToNamespace_WithGroupAccess(t *testing.T) { - scheme := runtime.NewScheme() - _ = corev1.AddToScheme(scheme) - _ = rbacv1.AddToScheme(scheme) - - ns := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"}, - } - - rb := &rbacv1.RoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-binding", - Namespace: "tenant-test", - }, - Subjects: []rbacv1.Subject{ - {Kind: "Group", Name: "test-group", APIGroup: "rbac.authorization.k8s.io"}, - }, - RoleRef: rbacv1.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "Role", - Name: "test-role", - }, - } - - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(ns, rb). - Build() - - r := &REST{ - c: client, - gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"}, - } - - u := &user.DefaultInfo{ - Name: "test-user", - Groups: []string{"system:authenticated", "test-group"}, - } - ctx := request.WithUser(context.Background(), u) - - hasAccess, err := r.hasAccessToNamespace(ctx, "tenant-test") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if !hasAccess { - t.Error("expected user to have access via group, but got false") - } -} - -func TestHasAccessToNamespace_SystemMasters(t *testing.T) { - scheme := runtime.NewScheme() - _ = corev1.AddToScheme(scheme) - _ = rbacv1.AddToScheme(scheme) - - ns := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"}, - } - - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(ns). - Build() - - r := &REST{ - c: client, - gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"}, - } - - u := &user.DefaultInfo{ - Name: "admin", - Groups: []string{"system:masters"}, - } - ctx := request.WithUser(context.Background(), u) - - hasAccess, err := r.hasAccessToNamespace(ctx, "tenant-test") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if !hasAccess { - t.Error("expected system:masters to have access, but got false") - } -} - -func TestHasAccessToNamespace_CozyAdminGroup(t *testing.T) { - scheme := runtime.NewScheme() - _ = corev1.AddToScheme(scheme) - _ = rbacv1.AddToScheme(scheme) - - ns := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"}, - } - - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(ns). - Build() - - r := &REST{ - c: client, - gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"}, - } - - u := &user.DefaultInfo{ - Name: "cozy-admin", - Groups: []string{"cozystack-cluster-admin"}, - } - ctx := request.WithUser(context.Background(), u) - - hasAccess, err := r.hasAccessToNamespace(ctx, "tenant-test") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if !hasAccess { - t.Error("expected cozystack-cluster-admin to have access, but got false") - } -} - -func TestHasAccessToNamespace_ServiceAccount(t *testing.T) { - scheme := runtime.NewScheme() - _ = corev1.AddToScheme(scheme) - _ = rbacv1.AddToScheme(scheme) - - ns := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"}, - } - - rb := &rbacv1.RoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-binding", - Namespace: "tenant-test", - }, - Subjects: []rbacv1.Subject{ - {Kind: "ServiceAccount", Name: "test-sa", Namespace: "tenant-test"}, - }, - RoleRef: rbacv1.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "Role", - Name: "test-role", - }, - } - - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(ns, rb). - Build() - - r := &REST{ - c: client, - gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"}, - } - - u := &user.DefaultInfo{ - Name: "system:serviceaccount:tenant-test:test-sa", - Groups: []string{"system:authenticated"}, - } - ctx := request.WithUser(context.Background(), u) - - hasAccess, err := r.hasAccessToNamespace(ctx, "tenant-test") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if !hasAccess { - t.Error("expected service account to have access, but got false") - } -} - -func TestHasAccessToNamespace_ServiceAccountEmptyNamespace(t *testing.T) { - scheme := runtime.NewScheme() - _ = corev1.AddToScheme(scheme) - _ = rbacv1.AddToScheme(scheme) - - ns := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"}, - } - - // ServiceAccount subject with empty namespace should default to RoleBinding namespace - rb := &rbacv1.RoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-binding", - Namespace: "tenant-test", - }, - Subjects: []rbacv1.Subject{ - {Kind: "ServiceAccount", Name: "test-sa", Namespace: ""}, // Empty namespace - }, - RoleRef: rbacv1.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "Role", - Name: "test-role", - }, - } - - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(ns, rb). - Build() - - r := &REST{ - c: client, - gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"}, - } - - u := &user.DefaultInfo{ - Name: "system:serviceaccount:tenant-test:test-sa", - Groups: []string{"system:authenticated"}, - } - ctx := request.WithUser(context.Background(), u) - - hasAccess, err := r.hasAccessToNamespace(ctx, "tenant-test") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if !hasAccess { - t.Error("expected service account with empty namespace to have access, but got false") - } -} - -func TestGet_WithAccess(t *testing.T) { - scheme := runtime.NewScheme() - _ = corev1.AddToScheme(scheme) - _ = rbacv1.AddToScheme(scheme) - - ns := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"}, - } - - rb := &rbacv1.RoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-binding", - Namespace: "tenant-test", - }, - Subjects: []rbacv1.Subject{ - {Kind: "User", Name: "test-user", APIGroup: "rbac.authorization.k8s.io"}, - }, - RoleRef: rbacv1.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "Role", - Name: "test-role", - }, - } - - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(ns, rb). - Build() - - r := &REST{ - c: client, - gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"}, - } - - u := &user.DefaultInfo{ - Name: "test-user", - Groups: []string{"system:authenticated"}, - } - ctx := request.WithUser(context.Background(), u) - - obj, err := r.Get(ctx, "tenant-test", &metav1.GetOptions{}) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - tn, ok := obj.(*corev1alpha1.TenantNamespace) - if !ok { - t.Fatalf("expected *TenantNamespace, got %T", obj) - } - if tn.Name != "tenant-test" { - t.Errorf("expected name %q, got %q", "tenant-test", tn.Name) - } - if tn.Kind != "TenantNamespace" { - t.Errorf("expected Kind=TenantNamespace, got %q", tn.Kind) - } - if tn.APIVersion != corev1alpha1.SchemeGroupVersion.String() { - t.Errorf("expected APIVersion=%q, got %q", corev1alpha1.SchemeGroupVersion.String(), tn.APIVersion) - } -} - -func TestGet_WithoutAccess(t *testing.T) { - scheme := runtime.NewScheme() - _ = corev1.AddToScheme(scheme) - _ = rbacv1.AddToScheme(scheme) - - ns := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{Name: "tenant-test"}, - } - - // RoleBinding for different user - rb := &rbacv1.RoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-binding", - Namespace: "tenant-test", - }, - Subjects: []rbacv1.Subject{ - {Kind: "User", Name: "other-user", APIGroup: "rbac.authorization.k8s.io"}, - }, - RoleRef: rbacv1.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "Role", - Name: "test-role", - }, - } - - client := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(ns, rb). - Build() - - r := &REST{ - c: client, - gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"}, - } - - u := &user.DefaultInfo{ - Name: "test-user", - Groups: []string{"system:authenticated"}, - } - ctx := request.WithUser(context.Background(), u) - - obj, err := r.Get(ctx, "tenant-test", &metav1.GetOptions{}) - if err == nil { - t.Fatal("expected error, got nil") - } - if obj != nil { - t.Errorf("expected nil object, got %v", obj) - } - - // Verify it returns Forbidden to follow standard K8s RBAC behavior - if !apierrors.IsForbidden(err) { - t.Errorf("expected Forbidden error, got %v", err) - } -} - -func TestGet_NonTenantNamespace(t *testing.T) { - scheme := runtime.NewScheme() - _ = corev1.AddToScheme(scheme) - _ = rbacv1.AddToScheme(scheme) - - client := fake.NewClientBuilder(). - WithScheme(scheme). - Build() - - r := &REST{ - c: client, - gvr: schema.GroupVersionResource{Group: "core.cozystack.io", Version: "v1alpha1", Resource: "tenantnamespaces"}, - } - - u := &user.DefaultInfo{ - Name: "test-user", - Groups: []string{"system:masters"}, - } - ctx := request.WithUser(context.Background(), u) - - obj, err := r.Get(ctx, "default", &metav1.GetOptions{}) - if err == nil { - t.Fatal("expected error for non-tenant namespace, got nil") - } - if obj != nil { - t.Errorf("expected nil object, got %v", obj) - } - if !apierrors.IsNotFound(err) { - t.Errorf("expected NotFound error, got %v", err) - } -}