diff --git a/.github/workflows/build-release-candidate.yml b/.github/workflows/build-release-candidate.yml index 17826cf7b..129249ebf 100644 --- a/.github/workflows/build-release-candidate.yml +++ b/.github/workflows/build-release-candidate.yml @@ -708,3 +708,143 @@ jobs: if-no-files-found: error retention-days: 1 overwrite: true + + qualify-release-containers: + name: Exact-Candidate Container and Helm Smoke + needs: build + runs-on: [self-hosted, Linux, X64, pulse-pve-build] + timeout-minutes: 15 + steps: + - name: Checkout repository + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + + - name: Download immutable release candidate + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ${{ needs.build.outputs.artifact_name }} + path: release + + - name: Download release candidate manifest + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ${{ needs.build.outputs.manifest_artifact_name }} + path: release-candidate-manifest + + - name: Verify immutable release candidate + run: | + python3 scripts/release_candidate_manifest.py verify-local \ + --release-dir release \ + --manifest release-candidate-manifest/release-candidate.json \ + --version "${{ inputs.version }}" \ + --source-sha "${GITHUB_SHA}" + + - name: Prepare exact-candidate container context + run: ./scripts/prepare-release-container-context.sh release "${{ inputs.version }}" "$RUNNER_TEMP/release-container-context" + + - name: Assemble exact-candidate runtime and agent images + run: | + set -euo pipefail + docker buildx build \ + --target runtime_prebuilt \ + --platform linux/amd64 \ + --build-context "release_payload=${RUNNER_TEMP}/release-container-context" \ + --load \ + --tag "pulse-helm-smoke:${{ inputs.version }}" \ + . + docker buildx build \ + --target agent_runtime_prebuilt \ + --platform linux/amd64 \ + --build-context "release_payload=${RUNNER_TEMP}/release-container-context" \ + --load \ + --tag "pulse-agent-candidate:${{ inputs.version }}" \ + . + + - name: Verify container binaries match immutable candidate + run: | + set -euo pipefail + expected_server="$(sha256sum "$RUNNER_TEMP/release-container-context/amd64/bin/pulse" | awk '{print $1}')" + actual_server="$(docker run --rm --entrypoint /bin/sh "pulse-helm-smoke:${{ inputs.version }}" -c 'sha256sum /app/pulse' | awk '{print $1}')" + expected_agent="$(sha256sum "$RUNNER_TEMP/release-container-context/amd64/bin/pulse-agent-linux-amd64" | awk '{print $1}')" + actual_agent="$(docker run --rm --entrypoint /bin/sh "pulse-agent-candidate:${{ inputs.version }}" -c 'sha256sum /usr/local/bin/pulse-agent' | awk '{print $1}')" + test "${actual_server}" = "${expected_server}" + test "${actual_agent}" = "${expected_agent}" + + - name: Set up Helm + uses: azure/setup-helm@dda3372f752e03dde6b3237bc9431cdc2f7a02a2 # v5.0.0 + with: + version: v3.15.2 + + - name: Helm smoke test with local release-line image + env: + SMOKE_IMAGE_REPOSITORY: pulse-helm-smoke + SMOKE_IMAGE_TAG: ${{ inputs.version }} + run: | + set -euo pipefail + + cleanup() { + kind delete cluster --name pulse-test >/dev/null 2>&1 || true + } + + diagnose() { + echo "::group::helm status" + helm status pulse || true + echo "::endgroup::" + + echo "::group::kubectl get all" + kubectl get all -A || true + echo "::endgroup::" + + echo "::group::kubectl describe pods" + kubectl describe pods -A || true + echo "::endgroup::" + + echo "::group::pod logs" + pods=$(kubectl get pods -A -o name 2>/dev/null || true) + for pod in $pods; do + echo "### ${pod}" + kubectl logs --all-containers=true --tail=200 "$pod" || true + done + echo "::endgroup::" + + echo "::group::events" + kubectl get events -A --sort-by=.lastTimestamp || kubectl get events -A || true + echo "::endgroup::" + + cleanup + } + + trap 'diagnose' ERR + + curl -fsSL -o "$RUNNER_TEMP/kind" https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64 + chmod +x "$RUNNER_TEMP/kind" + export PATH="$RUNNER_TEMP:$PATH" + + cleanup + kind create cluster --name pulse-test --wait 5m + kind load docker-image "${SMOKE_IMAGE_REPOSITORY}:${SMOKE_IMAGE_TAG}" --name pulse-test + + helm install pulse deploy/helm/pulse \ + --set persistence.enabled=false \ + --set server.secretEnv.create=true \ + --set server.secretEnv.data.API_TOKENS=test-token \ + --set image.repository="${SMOKE_IMAGE_REPOSITORY}" \ + --set image.tag="${SMOKE_IMAGE_TAG}" \ + --set image.pullPolicy=Never \ + --wait --timeout 5m --debug + + kubectl wait --for=condition=ready pod -l app.kubernetes.io/name=pulse --timeout=180s || (kubectl describe pods -l app.kubernetes.io/name=pulse && exit 1) + kubectl get pods -l app.kubernetes.io/name=pulse + + helm upgrade pulse deploy/helm/pulse \ + --set persistence.enabled=false \ + --set server.secretEnv.create=true \ + --set server.secretEnv.data.API_TOKENS=test-token \ + --set image.repository="${SMOKE_IMAGE_REPOSITORY}" \ + --set image.tag="${SMOKE_IMAGE_TAG}" \ + --set image.pullPolicy=Never \ + --wait --timeout 5m --debug + + trap - ERR + cleanup + + echo "✓ Helm smoke test passed" diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 783602fe6..39acb0722 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -377,149 +377,6 @@ jobs: - name: Run backend tests run: ./scripts/run-release-backend-tests.sh --data-root "$RUNNER_TEMP/pulse-test-data" - # Assemble the immutable candidate into both release container targets, - # verify their embedded binaries, and exercise that runtime through Helm. - docker_build: - name: Exact-Candidate Container and Helm Smoke - needs: - - prepare - - build_release_candidate - if: ${{ needs.prepare.outputs.historical_asset_backfill_only != 'true' }} - runs-on: [self-hosted, Linux, X64, pulse-pve-build] - timeout-minutes: 15 - steps: - - name: Checkout repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - - name: Download immutable release candidate - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: ${{ needs.build_release_candidate.outputs.artifact_name }} - path: release - - - name: Download release candidate manifest - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: ${{ needs.build_release_candidate.outputs.manifest_artifact_name }} - path: release-candidate-manifest - - - name: Verify immutable release candidate - run: | - python3 scripts/release_candidate_manifest.py verify-local \ - --release-dir release \ - --manifest release-candidate-manifest/release-candidate.json \ - --version "${{ needs.prepare.outputs.version }}" \ - --source-sha "${GITHUB_SHA}" - - - name: Prepare exact-candidate container context - run: ./scripts/prepare-release-container-context.sh release "${{ needs.prepare.outputs.version }}" "$RUNNER_TEMP/release-container-context" - - - name: Assemble exact-candidate runtime and agent images - run: | - set -euo pipefail - docker buildx build \ - --target runtime_prebuilt \ - --platform linux/amd64 \ - --build-context "release_payload=${RUNNER_TEMP}/release-container-context" \ - --load \ - --tag "pulse-helm-smoke:${{ needs.prepare.outputs.version }}" \ - . - docker buildx build \ - --target agent_runtime_prebuilt \ - --platform linux/amd64 \ - --build-context "release_payload=${RUNNER_TEMP}/release-container-context" \ - --load \ - --tag "pulse-agent-candidate:${{ needs.prepare.outputs.version }}" \ - . - - - name: Verify container binaries match immutable candidate - run: | - set -euo pipefail - expected_server="$(sha256sum "$RUNNER_TEMP/release-container-context/amd64/bin/pulse" | awk '{print $1}')" - actual_server="$(docker run --rm --entrypoint /bin/sh "pulse-helm-smoke:${{ needs.prepare.outputs.version }}" -c 'sha256sum /app/pulse' | awk '{print $1}')" - expected_agent="$(sha256sum "$RUNNER_TEMP/release-container-context/amd64/bin/pulse-agent-linux-amd64" | awk '{print $1}')" - actual_agent="$(docker run --rm --entrypoint /bin/sh "pulse-agent-candidate:${{ needs.prepare.outputs.version }}" -c 'sha256sum /usr/local/bin/pulse-agent' | awk '{print $1}')" - test "${actual_server}" = "${expected_server}" - test "${actual_agent}" = "${expected_agent}" - - - name: Set up Helm - uses: azure/setup-helm@dda3372f752e03dde6b3237bc9431cdc2f7a02a2 # v5.0.0 - with: - version: v3.15.2 - - - name: Helm smoke test with local release-line image - env: - SMOKE_IMAGE_REPOSITORY: pulse-helm-smoke - SMOKE_IMAGE_TAG: ${{ needs.prepare.outputs.version }} - run: | - set -euo pipefail - - cleanup() { - kind delete cluster --name pulse-test >/dev/null 2>&1 || true - } - - diagnose() { - echo "::group::helm status" - helm status pulse || true - echo "::endgroup::" - - echo "::group::kubectl get all" - kubectl get all -A || true - echo "::endgroup::" - - echo "::group::kubectl describe pods" - kubectl describe pods -A || true - echo "::endgroup::" - - echo "::group::pod logs" - pods=$(kubectl get pods -A -o name 2>/dev/null || true) - for pod in $pods; do - echo "### ${pod}" - kubectl logs --all-containers=true --tail=200 "$pod" || true - done - echo "::endgroup::" - - echo "::group::events" - kubectl get events -A --sort-by=.lastTimestamp || kubectl get events -A || true - echo "::endgroup::" - - cleanup - } - - trap 'diagnose' ERR - - curl -fsSL -o "$RUNNER_TEMP/kind" https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64 - chmod +x "$RUNNER_TEMP/kind" - export PATH="$RUNNER_TEMP:$PATH" - - kind create cluster --name pulse-test --wait 5m - kind load docker-image "${SMOKE_IMAGE_REPOSITORY}:${SMOKE_IMAGE_TAG}" --name pulse-test - - helm install pulse deploy/helm/pulse \ - --set persistence.enabled=false \ - --set server.secretEnv.create=true \ - --set server.secretEnv.data.API_TOKENS=test-token \ - --set image.repository="${SMOKE_IMAGE_REPOSITORY}" \ - --set image.tag="${SMOKE_IMAGE_TAG}" \ - --set image.pullPolicy=Never \ - --wait --timeout 5m --debug - - kubectl wait --for=condition=ready pod -l app.kubernetes.io/name=pulse --timeout=180s || (kubectl describe pods -l app.kubernetes.io/name=pulse && exit 1) - kubectl get pods -l app.kubernetes.io/name=pulse - - helm upgrade pulse deploy/helm/pulse \ - --set persistence.enabled=false \ - --set server.secretEnv.create=true \ - --set server.secretEnv.data.API_TOKENS=test-token \ - --set image.repository="${SMOKE_IMAGE_REPOSITORY}" \ - --set image.tag="${SMOKE_IMAGE_TAG}" \ - --set image.pullPolicy=Never \ - --wait --timeout 5m --debug - - trap - ERR - cleanup - - echo "✓ Helm smoke test passed" # Integration tests - skipped for prereleases (they've been tested in CI) integration_tests: @@ -790,12 +647,11 @@ jobs: - frontend_checks - windows_install_command_smoke - backend_tests - - docker_build - integration_tests - release_smoke # Run if integration_tests passed OR was skipped (prereleases). The # release smoke has no skipped escape: it runs for prereleases too. - if: ${{ needs.prepare.outputs.historical_asset_backfill_only != 'true' && always() && needs.build_release_candidate.result == 'success' && needs.frontend_bundle.result == 'success' && needs.frontend_checks.result == 'success' && needs.windows_install_command_smoke.result == 'success' && needs.backend_tests.result == 'success' && needs.docker_build.result == 'success' && needs.release_smoke.result == 'success' && (needs.integration_tests.result == 'success' || needs.integration_tests.result == 'skipped') }} + if: ${{ needs.prepare.outputs.historical_asset_backfill_only != 'true' && always() && needs.build_release_candidate.result == 'success' && needs.frontend_bundle.result == 'success' && needs.frontend_checks.result == 'success' && needs.windows_install_command_smoke.result == 'success' && needs.backend_tests.result == 'success' && needs.release_smoke.result == 'success' && (needs.integration_tests.result == 'success' || needs.integration_tests.result == 'skipped') }} runs-on: ubuntu-24.04 timeout-minutes: 30 permissions: diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index 29334eb73..0fa55c415 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -554,7 +554,10 @@ upgrade, update, release, or artifact-selection behavior. the prebuilt runtime and agent targets without recompiling source, and compare the embedded server and agent digests with the candidate bytes before exercising the same local runtime through the Helm install/upgrade - smoke. This credential-free lane may run against the isolated rootless + smoke. The reusable build-release-candidate workflow owns this proof so a + standalone candidate dispatch, a release dry run, and a publishing release + all cross the same container boundary before the candidate can succeed. + This credential-free lane may run against the isolated rootless Docker daemon owned by the low-priority PVE build identity; it must not gain host-Docker access, signing keys, registry login, or package-write authority. The public orchestrator may stage the exact private packet and private image diff --git a/scripts/installtests/build_release_assets_test.go b/scripts/installtests/build_release_assets_test.go index a58941e15..881fa1d59 100644 --- a/scripts/installtests/build_release_assets_test.go +++ b/scripts/installtests/build_release_assets_test.go @@ -166,7 +166,7 @@ func TestReleaseContainerTargetsConsumeImmutableCandidate(t *testing.T) { for _, needle := range []string{ `pulse-v${version}-linux-${arch}.tar.gz`, `validate_archive_entries "${archive}"`, - `tar --no-same-owner -xzf`, + `tar --no-same-owner --no-same-permissions -xzf`, `diff -qr --exclude=pulse`, } { if !strings.Contains(prepareScript, needle) { @@ -333,13 +333,13 @@ func TestAgentBuildCacheDoesNotResurrectPulseAgentPackage(t *testing.T) { } } - release, err := os.ReadFile(repoFile(".github", "workflows", "create-release.yml")) + release, err := os.ReadFile(repoFile(".github", "workflows", "build-release-candidate.yml")) if err != nil { - t.Fatalf("read create-release.yml: %v", err) + t.Fatalf("read build-release-candidate.yml: %v", err) } releaseText := string(release) if !strings.Contains(releaseText, `--target agent_runtime_prebuilt`) { - t.Fatal("create-release.yml must assemble the candidate agent image without targeting an unpublished package") + t.Fatal("build-release-candidate.yml must assemble the candidate agent image without targeting an unpublished package") } if strings.Contains(releaseText, "agent-buildcache") { t.Fatal("exact-candidate release qualification must not create a remote agent image cache") @@ -1207,7 +1207,7 @@ func TestReleaseWorkflowsUseSecretSafeAttestedImageBuilds(t *testing.T) { `attestations: write`, `uses: actions/attest@59d89421af93a897026c735860bf21b6eb4f7b26 # v4`, } - containerJob := workflowJobBlock(t, string(createReleaseBytes), "docker_build") + containerJob := workflowJobBlock(t, string(candidateWorkflowBytes), "qualify-release-containers") for _, forbidden := range []string{ "PULSE_UPDATE_SIGNING_KEY", "PULSE_LICENSE_PUBLIC_KEY", diff --git a/scripts/release_control/release_promotion_policy_test.py b/scripts/release_control/release_promotion_policy_test.py index 338c79ead..18bd6288c 100644 --- a/scripts/release_control/release_promotion_policy_test.py +++ b/scripts/release_control/release_promotion_policy_test.py @@ -1298,13 +1298,13 @@ class ReleasePromotionPolicyTest(unittest.TestCase): def test_release_workflow_enforces_rc_lineage_soak_and_v5_notice(self) -> None: content = read(".github/workflows/create-release.yml") - docker_build = workflow_job_block(content, "docker_build") update_demo_workflow = read(".github/workflows/update-demo-server.yml") deploy_demo_workflow = read(".github/workflows/deploy-demo-server.yml") demo_ssh_helper = read(".github/scripts/setup-demo-ssh.sh") demo_reachability_helper = read(".github/scripts/check-demo-reachability.sh") validation_workflow = read(".github/workflows/validate-release-assets.yml") candidate_workflow = read(".github/workflows/build-release-candidate.yml") + docker_build = workflow_job_block(candidate_workflow, "qualify-release-containers") release_validator = read("scripts/validate-release.sh") helper = read("scripts/trigger-release.sh") stable_patch_helper = read("scripts/trigger-stable-patch.sh") @@ -1691,6 +1691,7 @@ class ReleasePromotionPolicyTest(unittest.TestCase): demo = read(".github/workflows/update-demo-server.yml") preview_deploy = read(".github/workflows/deploy-demo-server.yml") release_workflow = read(".github/workflows/create-release.yml") + candidate_workflow = read(".github/workflows/build-release-candidate.yml") dry_run_workflow = read(".github/workflows/release-dry-run.yml") helm = read(".github/workflows/publish-helm-chart.yml") helm_pages = read(".github/workflows/helm-pages.yml") @@ -1787,11 +1788,11 @@ class ReleasePromotionPolicyTest(unittest.TestCase): self.assertIn("previous stable tag", artifact_validator) self.assertIn("stable_patch", artifact_validator) self.assertIn("Refusing {purpose}", artifact_validator) - self.assertIn("Assemble exact-candidate runtime and agent images", release_workflow) - self.assertIn('kind load docker-image "${SMOKE_IMAGE_REPOSITORY}:${SMOKE_IMAGE_TAG}" --name pulse-test', release_workflow) - self.assertIn('--set image.repository="${SMOKE_IMAGE_REPOSITORY}"', release_workflow) - self.assertIn('--set image.pullPolicy=Never', release_workflow) - self.assertIn("needs.docker_build.result == 'success'", release_workflow) + self.assertIn("Assemble exact-candidate runtime and agent images", candidate_workflow) + self.assertIn('kind load docker-image "${SMOKE_IMAGE_REPOSITORY}:${SMOKE_IMAGE_TAG}" --name pulse-test', candidate_workflow) + self.assertIn('--set image.repository="${SMOKE_IMAGE_REPOSITORY}"', candidate_workflow) + self.assertIn('--set image.pullPolicy=Never', candidate_workflow) + self.assertNotIn("needs.docker_build.result", release_workflow) self.assertNotIn("needs.helm_smoke.result", release_workflow) self.assertIn('--github-output "$GITHUB_OUTPUT"', helm_pages) self.assertIn("workflow_call:", helm_pages)