From dcc67662b6c339e75d67eee8d72975a046607c0c Mon Sep 17 00:00:00 2001 From: rcourtman Date: Fri, 21 Aug 2026 13:23:44 +0100 Subject: [PATCH] Stage release archives directly in parallel --- .../subsystems/deployment-installability.md | 9 +- scripts/build-release.sh | 88 ++++++++++--------- .../installtests/build_release_assets_test.go | 12 +++ scripts/release_asset_common.sh | 55 ++++++++++++ scripts/release_candidate_manifest.py | 12 ++- .../release_candidate_manifest_test.py | 34 +++++++ 6 files changed, 164 insertions(+), 46 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index b7851cbeda..5bc8854142 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -526,7 +526,9 @@ upgrade, update, release, or artifact-selection behavior. Cross-platform release compilation may run concurrently on a separate, low-priority PVE identity using only public embedding keys. It must produce one exact-version, exact-source-SHA manifest covering the complete frontend - and binary payload. The hosted candidate job must verify that manifest + and binary payload. That manifest may cover canonical relative paths in the + payload tree but must reject absolute, traversing, or noncanonical names. + The hosted candidate job must verify that manifest before applying required native binaries, packaging, update-signing, SBOM generation, validation, or upload. Private signing material and publication credentials must never enter the PVE compilation job. @@ -539,6 +541,11 @@ upgrade, update, release, or artifact-selection behavior. may compile the public payload and Pro server matrix, but the hosted private job must verify both SHAs before release signing, R2 upload, private-registry publication, or paid-runtime staging. + Public and Pro server archives must use the shared canonical staging helper + and may assemble independent target archives concurrently with a bounded + worker count. The verified dual-SHA Pro path must stage its five archives + directly from the precompiled public-agent and Pro-server payloads; it must + not build, sign, and then discard a complete public release packet first. The backend runner must compile the race-enabled `internal/api` test binary once, enumerate every top-level test from that exact binary, and generate a deterministic manifest proving a complete, disjoint partition. Each diff --git a/scripts/build-release.sh b/scripts/build-release.sh index de90a89831..c74cf5effc 100755 --- a/scripts/build-release.sh +++ b/scripts/build-release.sh @@ -238,51 +238,53 @@ for target in "${build_order[@]}"; do } done -# Create platform-specific tarballs that include all unified agent binaries for download endpoints +# Create platform-specific tarballs that include all unified agent binaries for +# download endpoints. The archives are independent, so stage them concurrently +# while keeping the default bounded for hosted runners. +package_server_target() { + local build_name="$1" + local archive_path="${PULSE_REPO_ROOT}/${RELEASE_DIR}/pulse-v${VERSION}-${build_name}.tar.gz" + local staging_dir="${PULSE_REPO_ROOT}/${BUILD_DIR}/staging-${build_name}" + + echo "Packaging release for ${build_name}..." + pulse_release_stage_server_archive \ + "${archive_path}" \ + "${staging_dir}" \ + "${PULSE_REPO_ROOT}/${BUILD_DIR}/pulse-${build_name}" \ + "${VERSION}" \ + "${PULSE_REPO_ROOT}/${BUILD_DIR}" \ + "${PULSE_REPO_ROOT}/${RENDERED_INSTALLERS_DIR}" + rm -rf "${staging_dir}" + echo "Created ${RELEASE_DIR}/pulse-v${VERSION}-${build_name}.tar.gz" +} + +package_workers="${PULSE_RELEASE_PACKAGE_WORKERS:-4}" +if [[ ! "${package_workers}" =~ ^[1-9][0-9]*$ ]]; then + echo "Error: PULSE_RELEASE_PACKAGE_WORKERS must be a positive integer." >&2 + exit 1 +fi +package_pids=() for build_name in "${build_order[@]}"; do - echo "Packaging release for $build_name..." - - tar_name="pulse-v${VERSION}-${build_name}.tar.gz" - staging_dir="$BUILD_DIR/staging-$build_name" - rm -rf "$staging_dir" - mkdir -p "$staging_dir/bin" - mkdir -p "$staging_dir/scripts" - - # Copy architecture-specific runtime binaries - cp "$BUILD_DIR/pulse-$build_name" "$staging_dir/bin/pulse" - - # Copy unified agent binaries for every supported platform/architecture - for target in "${agent_build_order[@]}"; do - src="$BUILD_DIR/pulse-agent-$target" - dest="$staging_dir/bin/pulse-agent-$target" - if [[ "$target" == windows-* ]]; then - src="${src}.exe" - dest="${dest}.exe" - fi - cp "$src" "$dest" - done - ( cd "$staging_dir/bin" && ln -sf pulse-agent-windows-amd64.exe pulse-agent-windows-amd64 && ln -sf pulse-agent-windows-arm64.exe pulse-agent-windows-arm64 && ln -sf pulse-agent-windows-386.exe pulse-agent-windows-386 ) - - # Copy scripts and VERSION metadata - cp "scripts/install-container-agent.sh" "$staging_dir/scripts/install-container-agent.sh" - cp "scripts/install-docker.sh" "$staging_dir/scripts/install-docker.sh" - cp "${RENDERED_INSTALLERS_DIR}/install.sh" "$staging_dir/scripts/install.sh" - [ -f "${RENDERED_INSTALLERS_DIR}/install.ps1" ] && cp "${RENDERED_INSTALLERS_DIR}/install.ps1" "$staging_dir/scripts/install.ps1" - chmod 755 "$staging_dir/scripts/"*.sh - chmod 755 "$staging_dir/scripts/"*.ps1 2>/dev/null || true - echo "$VERSION" > "$staging_dir/VERSION" - pulse_release_sign_directory_assets "$staging_dir/bin" - pulse_release_sign_directory_assets "$staging_dir/scripts" - pulse_release_sign_file "$staging_dir/VERSION" - - # Create tarball from staging directory - cd "$staging_dir" - tar -czf "../../$RELEASE_DIR/$tar_name" . - cd ../.. - - rm -rf "$staging_dir" - echo "Created $RELEASE_DIR/$tar_name" + package_server_target "${build_name}" & + package_pids+=("$!") + if [[ ${#package_pids[@]} -ge ${package_workers} ]]; then + package_failed=0 + for package_pid in "${package_pids[@]}"; do + if ! wait "${package_pid}"; then + package_failed=1 + fi + done + [[ ${package_failed} -eq 0 ]] || exit 1 + package_pids=() + fi done +package_failed=0 +for package_pid in "${package_pids[@]}"; do + if ! wait "${package_pid}"; then + package_failed=1 + fi +done +[[ ${package_failed} -eq 0 ]] || exit 1 # Create universal tarball with all binaries echo "Creating universal tarball..." diff --git a/scripts/installtests/build_release_assets_test.go b/scripts/installtests/build_release_assets_test.go index fec27fe9f7..167f4dab5d 100644 --- a/scripts/installtests/build_release_assets_test.go +++ b/scripts/installtests/build_release_assets_test.go @@ -118,12 +118,24 @@ func TestBuildReleaseUsesV6InstallScripts(t *testing.T) { `"${resolved_tool}" "dir:${release_dir}" -o "spdx-json=${tmp_sbom}"`, `if compgen -G "pulse-*.sbom.spdx.json" > /dev/null; then`, `find . -maxdepth 1 -type f \( -name '*.sig' -o -name '*.sshsig' \) -delete`, + `pulse_release_stage_server_archive()`, + `for target in "${PULSE_RELEASE_AGENT_TARGETS[@]}"; do`, + `install -m 0755 "${server_binary}" "${staging_dir}/bin/pulse"`, } for _, needle := range helperRequired { if !strings.Contains(helper, needle) { t.Fatalf("release_asset_common.sh missing canonical release asset wiring: %s", needle) } } + for _, needle := range []string{ + `package_workers="${PULSE_RELEASE_PACKAGE_WORKERS:-4}"`, + `package_server_target "${build_name}" &`, + `pulse_release_stage_server_archive \`, + } { + if !strings.Contains(script, needle) { + t.Fatalf("build-release.sh missing bounded parallel archive assembly: %s", needle) + } + } } func TestProviderMSPReleaseBundleIsRequiredAndValidated(t *testing.T) { diff --git a/scripts/release_asset_common.sh b/scripts/release_asset_common.sh index 421004051f..13431151a0 100644 --- a/scripts/release_asset_common.sh +++ b/scripts/release_asset_common.sh @@ -124,6 +124,61 @@ pulse_release_sign_directory_assets() { done < <(find "${dir}" -maxdepth 1 -type f ! -name '*.sig' ! -name '*.sshsig' -print0) } +pulse_release_stage_server_archive() { + local archive_path="$1" + local staging_dir="$2" + local server_binary="$3" + local version="$4" + local agent_binary_dir="$5" + local rendered_installers_dir="$6" + local target="" + local src="" + local dest="" + + if [[ ${#PULSE_RELEASE_AGENT_TARGETS[@]} -eq 0 ]]; then + echo "Error: PULSE_RELEASE_AGENT_TARGETS is empty." >&2 + return 1 + fi + + rm -rf "${staging_dir}" + mkdir -p "${staging_dir}/bin" "${staging_dir}/scripts" + install -m 0755 "${server_binary}" "${staging_dir}/bin/pulse" + + for target in "${PULSE_RELEASE_AGENT_TARGETS[@]}"; do + src="${agent_binary_dir}/pulse-agent-${target}" + dest="${staging_dir}/bin/pulse-agent-${target}" + if [[ "${target}" == windows-* ]]; then + src="${src}.exe" + dest="${dest}.exe" + fi + install -m 0755 "${src}" "${dest}" + done + ( + cd "${staging_dir}/bin" + ln -sf pulse-agent-windows-amd64.exe pulse-agent-windows-amd64 + ln -sf pulse-agent-windows-arm64.exe pulse-agent-windows-arm64 + ln -sf pulse-agent-windows-386.exe pulse-agent-windows-386 + ) + + install -m 0755 "${PULSE_SCRIPTS_DIR}/install-container-agent.sh" "${staging_dir}/scripts/install-container-agent.sh" + install -m 0755 "${PULSE_SCRIPTS_DIR}/install-docker.sh" "${staging_dir}/scripts/install-docker.sh" + install -m 0755 "${rendered_installers_dir}/install.sh" "${staging_dir}/scripts/install.sh" + if [[ -f "${rendered_installers_dir}/install.ps1" ]]; then + install -m 0755 "${rendered_installers_dir}/install.ps1" "${staging_dir}/scripts/install.ps1" + fi + printf '%s\n' "${version}" > "${staging_dir}/VERSION" + + pulse_release_sign_directory_assets "${staging_dir}/bin" + pulse_release_sign_directory_assets "${staging_dir}/scripts" + pulse_release_sign_file "${staging_dir}/VERSION" + + mkdir -p "$(dirname "${archive_path}")" + ( + cd "${staging_dir}" + tar -czf "${archive_path}" . + ) +} + pulse_release_generate_packet_sbom() { local release_dir="$1" local output_name="$2" diff --git a/scripts/release_candidate_manifest.py b/scripts/release_candidate_manifest.py index 47153a0bf6..89298be6b4 100644 --- a/scripts/release_candidate_manifest.py +++ b/scripts/release_candidate_manifest.py @@ -8,7 +8,7 @@ import hashlib import json import re import sys -from pathlib import Path +from pathlib import Path, PurePosixPath from typing import Any @@ -92,7 +92,15 @@ def manifest_assets_by_name(manifest: dict[str, Any]) -> dict[str, dict[str, Any name = asset.get("name") size = asset.get("size") digest = asset.get("sha256") - if not isinstance(name, str) or not name or Path(name).name != name: + if not isinstance(name, str) or not name: + raise ValueError(f"manifest asset {index} has invalid name: {name!r}") + relative_name = PurePosixPath(name) + if ( + relative_name.is_absolute() + or relative_name.as_posix() != name + or "\\" in name + or any(part in {"", ".", ".."} for part in relative_name.parts) + ): raise ValueError(f"manifest asset {index} has invalid name: {name!r}") if not isinstance(size, int) or size < 0: raise ValueError(f"manifest asset {name!r} has invalid size: {size!r}") diff --git a/scripts/release_control/release_candidate_manifest_test.py b/scripts/release_control/release_candidate_manifest_test.py index a8ff0128e7..16b3b1ad0c 100644 --- a/scripts/release_control/release_candidate_manifest_test.py +++ b/scripts/release_control/release_candidate_manifest_test.py @@ -9,6 +9,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from release_candidate_manifest import ( create_manifest, load_release_assets, + manifest_assets_by_name, verify_local, verify_release, ) @@ -46,6 +47,39 @@ class ReleaseCandidateManifestTest(unittest.TestCase): with self.assertRaisesRegex(ValueError, "digest mismatch"): verify_local(release_dir, manifest, "6.1.0-rc.1", SOURCE_SHA) + def test_verify_local_accepts_manifest_covered_payload_tree(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + release_dir = self.create_release_dir(Path(temp_dir)) + binary_dir = release_dir / "binaries" + binary_dir.mkdir() + (binary_dir / "pulse-agent-linux-amd64").write_bytes(b"agent") + manifest = create_manifest(release_dir, "6.1.0-rc.1", SOURCE_SHA) + + verify_local(release_dir, manifest, "6.1.0-rc.1", SOURCE_SHA) + self.assertIn( + "binaries/pulse-agent-linux-amd64", + [asset["name"] for asset in manifest["assets"]], + ) + + def test_manifest_rejects_noncanonical_or_traversing_asset_names(self) -> None: + for name in ( + "../outside", + "binaries/../outside", + "/absolute", + "binaries\\outside", + "./binaries/agent", + ): + with self.subTest(name=name), self.assertRaisesRegex( + ValueError, "invalid name" + ): + manifest_assets_by_name( + { + "assets": [ + {"name": name, "size": 1, "sha256": "0" * 64} + ] + } + ) + def test_verify_release_uses_server_side_digests(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: release_dir = self.create_release_dir(Path(temp_dir))