mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-11-16 13:39:36 +00:00
Preserve working server units when replacement writes fail
Check every unit write in the existing conditional callers, stage beside the destination, and require mode and atomic publication success. Preserve masks and redirected or non-regular destinations. Exercise old-unit preservation, all rendered phases and the real missing-unit update callers with confined failure fixtures. Contract-Neutral: Restores the existing complete-unit installation and failure-propagation contract without changing rendered settings, service authority, startup verification or release policy. Change-source: pulse-maintainer
This commit is contained in:
parent
952d2855ce
commit
ccb0ef54a7
4 changed files with 372 additions and 9 deletions
53
install.sh
53
install.sh
|
|
@ -5265,7 +5265,23 @@ install_systemd_service() {
|
|||
print_info "Using existing service name: pulse-backend"
|
||||
fi
|
||||
|
||||
cat > /etc/systemd/system/$SERVICE_NAME.service << EOF
|
||||
local service_path="/etc/systemd/system/$SERVICE_NAME.service"
|
||||
local staged_service="" render_failed=false
|
||||
# Atomic publication must not replace an operator mask or redirected unit.
|
||||
# Keep non-regular destinations for deliberate manual reconciliation.
|
||||
if [[ -L "$service_path" || ( -e "$service_path" && ! -f "$service_path" ) ]]; then
|
||||
print_error "Cannot replace a redirected or non-regular service definition ($service_path); installation is incomplete. Inspect the unit before retrying."
|
||||
return 1
|
||||
fi
|
||||
if ! staged_service=$(mktemp "${service_path}.staged.XXXXXX"); then
|
||||
print_error "Cannot stage the systemd service ($service_path); installation is incomplete. The existing service definition was not changed."
|
||||
return 1
|
||||
fi
|
||||
|
||||
# The main callers check this function with || return 1, which suppresses
|
||||
# errexit throughout it. Check every write, including both append phases;
|
||||
# best-effort daemon-reload must never hide a truncated unit publication.
|
||||
if ! cat > "$staged_service" << EOF
|
||||
[Unit]
|
||||
Description=Pulse Monitoring Server
|
||||
After=network.target
|
||||
|
|
@ -5290,15 +5306,21 @@ Environment="PULSE_DEPLOYMENT_METHOD=systemd"
|
|||
Environment="PULSE_LOG_JOURNAL_LEVEL_PREFIX=true"
|
||||
EnvironmentFile=-$CONFIG_DIR/.env
|
||||
EOF
|
||||
|
||||
# Add port configuration if not default
|
||||
if [[ "${FRONTEND_PORT:-7655}" != "7655" ]]; then
|
||||
cat >> /etc/systemd/system/$SERVICE_NAME.service << EOF
|
||||
Environment="FRONTEND_PORT=$FRONTEND_PORT"
|
||||
EOF
|
||||
then
|
||||
render_failed=true
|
||||
fi
|
||||
|
||||
cat >> /etc/systemd/system/$SERVICE_NAME.service << EOF
|
||||
# Add port configuration if not default
|
||||
if [[ "$render_failed" == false && "${FRONTEND_PORT:-7655}" != "7655" ]]; then
|
||||
if ! cat >> "$staged_service" << EOF
|
||||
Environment="FRONTEND_PORT=$FRONTEND_PORT"
|
||||
EOF
|
||||
then
|
||||
render_failed=true
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$render_failed" == false ]] && ! cat >> "$staged_service" << EOF
|
||||
|
||||
# Security hardening
|
||||
NoNewPrivileges=true
|
||||
|
|
@ -5314,6 +5336,21 @@ CapabilityBoundingSet=CAP_NET_RAW
|
|||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
then
|
||||
render_failed=true
|
||||
fi
|
||||
|
||||
# Recheck the destination before the one same-directory atomic rename.
|
||||
# A failed staging, mode or publication operation stops the real caller
|
||||
# before startup/completion; no rollback or service recovery is implied.
|
||||
if [[ "$render_failed" == true || -L "$service_path" || ( -e "$service_path" && ! -f "$service_path" ) ]] ||
|
||||
! chmod 0644 "$staged_service" || ! mv -fT -- "$staged_service" "$service_path"; then
|
||||
if ! rm -f -- "$staged_service"; then
|
||||
print_warn "Could not remove staged service definition ($staged_service); inspect the retained staging file before retrying."
|
||||
fi
|
||||
print_error "Could not publish the complete systemd service ($service_path); installation is incomplete. Inspect the service definition and state before retrying; no automatic restart was attempted."
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Reload systemd daemon
|
||||
safe_systemctl daemon-reload
|
||||
|
|
|
|||
25
scripts/installtests/server_unit_publication_test.go
Normal file
25
scripts/installtests/server_unit_publication_test.go
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
package installtests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"runtime"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestRootInstallSystemdUnitPublication(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("Unix installer and confined filesystem controls")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
|
||||
defer cancel()
|
||||
cmd := exec.CommandContext(ctx, "python3", repoFile("scripts", "tests", "test_server_unit_publication.py"), "-v")
|
||||
cmd.Env = append(os.Environ(), "PULSE_INSTALLER_UNDER_TEST="+repoFile("install.sh"))
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("complete server-unit publication and preservation controls: %v\n%s", err, out)
|
||||
}
|
||||
t.Logf("confined unit publication controls:\n%s", out)
|
||||
}
|
||||
250
scripts/tests/test_server_unit_publication.py
Normal file
250
scripts/tests/test_server_unit_publication.py
Normal file
|
|
@ -0,0 +1,250 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Real server-unit rendering/publication, confined to ordinary-user fixtures.
|
||||
|
||||
Only the fixed unit directory is relocated. No system manager or host path is
|
||||
mutated; passing these controls is not installed-service acceptance.
|
||||
"""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import stat
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
INSTALLER = Path(os.environ.get("PULSE_INSTALLER_UNDER_TEST", ROOT / "install.sh")).resolve()
|
||||
|
||||
HARNESS = r'''
|
||||
set -euo pipefail
|
||||
SERVICE_NAME="$SELECTED_SERVICE"
|
||||
SERVICE_NAME_EXPLICIT="$EXPLICIT"
|
||||
INSTALL_DIR="$FIXTURE/install folder"
|
||||
CONFIG_DIR="$FIXTURE/saved config"
|
||||
FRONTEND_PORT="$PORT"
|
||||
unit_cat_count=0
|
||||
record() { printf '%s\n' "$*" >> "$FIXTURE/calls"; }
|
||||
print_info() { printf '%s\n' "$*"; }
|
||||
print_warn() { printf 'WARN: %s\n' "$*"; }
|
||||
print_error() { printf 'ERROR: %s\n' "$*"; }
|
||||
detect_service_name() { printf '%s\n' "$DETECTED_SERVICE"; }
|
||||
safe_systemctl() { record "$*"; }
|
||||
mktemp() {
|
||||
record allocate
|
||||
[[ "$FAULT" != allocation ]] || { echo INJECTED_ALLOCATION >&2; return 71; }
|
||||
command mktemp "$@"
|
||||
}
|
||||
cat() {
|
||||
unit_cat_count=$((unit_cat_count + 1))
|
||||
record "write $unit_cat_count"
|
||||
if [[ "$FAULT" == "write-$unit_cat_count" || ( "$FAULT" == cleanup && "$unit_cat_count" == 1 ) ]]; then
|
||||
printf 'partial unit\n'
|
||||
echo INJECTED_WRITE >&2
|
||||
return 72
|
||||
fi
|
||||
if [[ "$FAULT" == redirected-during-render && "$unit_cat_count" == 1 ]]; then
|
||||
ln -s "$FIXTURE/redirect-target" "$FIXTURE/units/$SERVICE_NAME.service"
|
||||
fi
|
||||
command cat "$@"
|
||||
}
|
||||
chmod() {
|
||||
record mode
|
||||
[[ "$FAULT" != mode ]] || { echo INJECTED_MODE >&2; return 73; }
|
||||
command chmod "$@"
|
||||
}
|
||||
mv() {
|
||||
record publish
|
||||
[[ "$FAULT" != rename ]] || { echo INJECTED_RENAME >&2; return 74; }
|
||||
[[ "$#" == 4 && "$1" == -fT && "$2" == -- && "$3" == "$4".staged.* ]] || return 95
|
||||
# The destination is still old/absent, and the staged bytes are complete,
|
||||
# with the intended mode, at the only publication boundary.
|
||||
if [[ "$EXISTING" == true ]]; then
|
||||
cmp "$FIXTURE/original" "$4" || return 96
|
||||
else
|
||||
[[ ! -e "$4" && ! -L "$4" ]] || return 96
|
||||
fi
|
||||
cmp "$FIXTURE/expected" "$3" || return 97
|
||||
[[ "$(stat -c %a "$3")" == 644 ]] || return 98
|
||||
command mv "$@"
|
||||
}
|
||||
rm() {
|
||||
record cleanup
|
||||
[[ "$FAULT" != cleanup ]] || { echo INJECTED_CLEANUP >&2; return 75; }
|
||||
command rm "$@"
|
||||
}
|
||||
source "$FIXTURE/unit-function.sh"
|
||||
status=0
|
||||
# Match main's OR-list context: failures must not rely on implicit errexit.
|
||||
install_systemd_service || status=$?
|
||||
exit "$status"
|
||||
'''
|
||||
|
||||
|
||||
def expected_unit(install, config, port):
|
||||
header = f'''[Unit]
|
||||
Description=Pulse Monitoring Server
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=pulse
|
||||
Group=pulse
|
||||
WorkingDirectory={install}
|
||||
ExecStart={install}/bin/pulse
|
||||
Restart=always
|
||||
RestartSec=3
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogLevelPrefix=true
|
||||
Environment="HOME={install}"
|
||||
Environment="PATH={install}/.local/bin:{install}/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
||||
Environment="PULSE_DATA_DIR={config}"
|
||||
Environment="PULSE_DEPLOYMENT_METHOD=systemd"
|
||||
# Pulse prefixes each JSON line with its zerolog priority. systemd consumes
|
||||
# the prefix into PRIORITY and retains the original JSON as MESSAGE.
|
||||
Environment="PULSE_LOG_JOURNAL_LEVEL_PREFIX=true"
|
||||
EnvironmentFile=-{config}/.env
|
||||
'''
|
||||
if port != "7655":
|
||||
header += f'Environment="FRONTEND_PORT={port}"\n'
|
||||
return header + f'''
|
||||
# Security hardening
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=true
|
||||
ReadWritePaths={install} {config}
|
||||
# ICMP availability probes exec the system ping binary; NoNewPrivileges strips
|
||||
# its setuid/file capabilities, so grant CAP_NET_RAW ambiently instead (#1554).
|
||||
AmbientCapabilities=CAP_NET_RAW
|
||||
CapabilityBoundingSet=CAP_NET_RAW
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
'''
|
||||
|
||||
|
||||
class ServerUnitPublication(unittest.TestCase):
|
||||
def exercise(self, *, existing=False, port="7655", fault="none", destination="file",
|
||||
selected="pulse", explicit=True, detected="pulse"):
|
||||
with tempfile.TemporaryDirectory(prefix="pulse-unit-publication-") as directory:
|
||||
f = Path(directory)
|
||||
(f / "units").mkdir()
|
||||
# The legacy detection branch checks that its selected unit exists.
|
||||
service = "pulse-backend" if not explicit and detected == "pulse-backend" else selected
|
||||
unit = f / "units" / (service + ".service")
|
||||
original = "previous complete service definition\n"
|
||||
(f / "original").write_text(original)
|
||||
target = f / "redirect-target"
|
||||
target.write_text("redirected unit must survive\n")
|
||||
if destination == "mask":
|
||||
unit.symlink_to("/dev/null")
|
||||
elif destination == "symlink":
|
||||
unit.symlink_to(target)
|
||||
elif destination == "dangling":
|
||||
unit.symlink_to(f / "not-created")
|
||||
elif destination == "directory":
|
||||
unit.mkdir()
|
||||
elif destination == "fifo":
|
||||
os.mkfifo(unit)
|
||||
elif existing:
|
||||
unit.write_text(original)
|
||||
unit.chmod(0o640)
|
||||
expected = expected_unit(f / "install folder", f / "saved config", port)
|
||||
(f / "expected").write_text(expected)
|
||||
match = re.search(r"(?ms)^install_systemd_service\(\) \{\n.*?^\}", INSTALLER.read_text())
|
||||
self.assertIsNotNone(match)
|
||||
(f / "unit-function.sh").write_text(match[0].replace("/etc/systemd/system/", str(f / "units") + "/"))
|
||||
env = dict(os.environ, FIXTURE=str(f), FAULT=fault, PORT=port,
|
||||
SELECTED_SERVICE=selected, EXPLICIT=str(explicit).lower(),
|
||||
DETECTED_SERVICE=detected, EXISTING=str(existing).lower())
|
||||
result = subprocess.run(["bash", "-c", HARNESS], env=env, cwd=f,
|
||||
capture_output=True, text=True, timeout=10)
|
||||
stages = list((f / "units").glob("*.staged.*"))
|
||||
return dict(result=result, output=result.stdout + result.stderr,
|
||||
calls=(f / "calls").read_text().splitlines() if (f / "calls").exists() else [],
|
||||
contents=unit.read_text() if unit.is_file() and not unit.is_symlink() else None,
|
||||
mode=stat.S_IMODE(unit.stat().st_mode) if unit.is_file() and not unit.is_symlink() else None,
|
||||
link=os.readlink(unit) if unit.is_symlink() else None,
|
||||
original=original, expected=expected, target=target.read_text(),
|
||||
stages=[(p.read_text(), stat.S_IMODE(p.stat().st_mode)) for p in stages],
|
||||
is_directory=unit.is_dir(), is_fifo=unit.exists() and stat.S_ISFIFO(unit.lstat().st_mode))
|
||||
|
||||
def test_complete_publication_preserves_rendering_and_service_selection(self):
|
||||
for port in ("7655", "8765"):
|
||||
for selected, explicit, detected in (("pulse", True, "pulse"),
|
||||
("pulse-site-b", True, "pulse-backend"),
|
||||
("pulse", False, "pulse-backend")):
|
||||
with self.subTest(port=port, selected=selected, explicit=explicit):
|
||||
e = self.exercise(existing=True, port=port, selected=selected,
|
||||
explicit=explicit, detected=detected)
|
||||
self.assertEqual(e["result"].returncode, 0, e["output"])
|
||||
self.assertEqual(e["contents"], e["expected"])
|
||||
self.assertEqual(e["mode"], 0o644)
|
||||
self.assertEqual(e["calls"].count("publish"), 1)
|
||||
self.assertEqual(e["calls"][-1], "daemon-reload")
|
||||
self.assertEqual(e["stages"], [])
|
||||
|
||||
def test_failures_never_publish_partial_units_or_reload(self):
|
||||
for existing in (False, True):
|
||||
for port in ("7655", "8765"):
|
||||
faults = ["allocation", "write-1", "write-2", "mode", "rename", "cleanup"]
|
||||
if port != "7655":
|
||||
faults.append("write-3")
|
||||
for fault in faults:
|
||||
with self.subTest(existing=existing, port=port, fault=fault):
|
||||
e = self.exercise(existing=existing, port=port, fault=fault)
|
||||
self.assertEqual(e["result"].returncode, 1, e["output"])
|
||||
self.assertIn("INJECTED_", e["output"])
|
||||
self.assertIn("installation is incomplete", e["output"])
|
||||
self.assertNotIn("daemon-reload", e["calls"])
|
||||
self.assertEqual(e["contents"], e["original"] if existing else None)
|
||||
self.assertEqual(e["mode"], 0o640 if existing else None)
|
||||
if fault == "cleanup":
|
||||
self.assertIn("Could not remove staged service definition", e["output"])
|
||||
self.assertEqual(e["stages"], [("partial unit\n", 0o600)])
|
||||
else:
|
||||
self.assertEqual(e["stages"], [])
|
||||
|
||||
def test_new_units_are_complete_before_becoming_visible(self):
|
||||
for port in ("7655", "8765"):
|
||||
with self.subTest(port=port):
|
||||
e = self.exercise(port=port)
|
||||
self.assertEqual(e["result"].returncode, 0, e["output"])
|
||||
self.assertEqual(e["contents"], e["expected"])
|
||||
self.assertEqual(e["mode"], 0o644)
|
||||
self.assertEqual(e["stages"], [])
|
||||
|
||||
def test_redirects_masks_and_nonregular_destinations_are_not_replaced(self):
|
||||
for destination in ("mask", "symlink", "dangling", "directory", "fifo"):
|
||||
with self.subTest(destination=destination):
|
||||
e = self.exercise(destination=destination)
|
||||
self.assertEqual(e["result"].returncode, 1, e["output"])
|
||||
self.assertIn("redirected or non-regular", e["output"])
|
||||
self.assertEqual(e["calls"], [])
|
||||
self.assertEqual(e["target"], "redirected unit must survive\n")
|
||||
self.assertEqual(e["stages"], [])
|
||||
if destination == "mask":
|
||||
self.assertEqual(e["link"], "/dev/null")
|
||||
elif destination in ("symlink", "dangling"):
|
||||
self.assertIsNotNone(e["link"])
|
||||
elif destination == "directory":
|
||||
self.assertTrue(e["is_directory"])
|
||||
else:
|
||||
self.assertTrue(e["is_fifo"])
|
||||
|
||||
def test_redirect_appearing_during_render_is_not_overwritten(self):
|
||||
e = self.exercise(fault="redirected-during-render")
|
||||
self.assertEqual(e["result"].returncode, 1, e["output"])
|
||||
self.assertIn("Could not publish", e["output"])
|
||||
self.assertNotIn("publish", e["calls"])
|
||||
self.assertNotIn("daemon-reload", e["calls"])
|
||||
self.assertIsNotNone(e["link"])
|
||||
self.assertEqual(e["target"], "redirected unit must survive\n")
|
||||
self.assertEqual(e["stages"], [])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
@ -89,6 +89,39 @@ print_completion() { record complete; }
|
|||
if [[ "$UNIT_ERROR" == true ]]; then
|
||||
install_systemd_service() { record refused-unit; return 23; }
|
||||
fi
|
||||
unit_cat_count=0
|
||||
cat() {
|
||||
if [[ "${FUNCNAME[1]:-}" == install_systemd_service ]]; then
|
||||
unit_cat_count=$((unit_cat_count + 1))
|
||||
if [[ "$UNIT_FAULT" == "write-$unit_cat_count" ]]; then
|
||||
printf 'partial unit\n'
|
||||
echo INJECTED_UNIT_WRITE >&2
|
||||
return 72
|
||||
fi
|
||||
fi
|
||||
command cat "$@"
|
||||
}
|
||||
mktemp() {
|
||||
if [[ "$UNIT_FAULT" == allocation && "$1" == "$FIXTURE/units/"* ]]; then
|
||||
echo INJECTED_UNIT_ALLOCATION >&2
|
||||
return 71
|
||||
fi
|
||||
command mktemp "$@"
|
||||
}
|
||||
chmod() {
|
||||
if [[ "$UNIT_FAULT" == mode && "${@: -1}" == "$FIXTURE/units/"*.staged.* ]]; then
|
||||
echo INJECTED_UNIT_MODE >&2
|
||||
return 73
|
||||
fi
|
||||
command chmod "$@"
|
||||
}
|
||||
mv() {
|
||||
if [[ "$UNIT_FAULT" == rename && "${@: -1}" == "$FIXTURE/units/pulse.service" ]]; then
|
||||
echo INJECTED_UNIT_RENAME >&2
|
||||
return 74
|
||||
fi
|
||||
command mv "$@"
|
||||
}
|
||||
# Failure propagation must survive an OR-list, not depend on implicit errexit.
|
||||
status=0
|
||||
main || status=$?
|
||||
|
|
@ -97,7 +130,7 @@ exit "$status"
|
|||
|
||||
|
||||
class ServerUpdateRepair(unittest.TestCase):
|
||||
def exercise(self, *, pinned, automatic, unit_error=False):
|
||||
def exercise(self, *, pinned, automatic, unit_error=False, unit_fault="none"):
|
||||
with tempfile.TemporaryDirectory(prefix="pulse-update-repair-") as directory:
|
||||
f = Path(directory)
|
||||
for name in ["tools", "install/bin", "units"]:
|
||||
|
|
@ -118,6 +151,7 @@ class ServerUpdateRepair(unittest.TestCase):
|
|||
env.update(FIXTURE=str(f), INSTALLER=str(INSTALLER),
|
||||
PIN="v6.6.0" if pinned else "",
|
||||
AUTO_UPDATES=str(automatic).lower(), UNIT_ERROR=str(unit_error).lower(),
|
||||
UNIT_FAULT=unit_fault,
|
||||
PATH=str(f / "tools") + os.pathsep + os.environ["PATH"],
|
||||
PULSE_INSTALL_DIR=str(f / "install"), PULSE_CONFIG_DIR=str(f / "config"))
|
||||
# Config and unit are genuinely absent on entry, not pre-created.
|
||||
|
|
@ -135,6 +169,23 @@ class ServerUpdateRepair(unittest.TestCase):
|
|||
env=env_file.read_text() if env_file.exists() else "",
|
||||
unit=(f / "units/pulse.service").read_text() if (f / "units/pulse.service").exists() else "")
|
||||
|
||||
def test_real_unit_publication_failures_stop_both_main_update_paths(self):
|
||||
for pinned in (True, False):
|
||||
for automatic in (True, False):
|
||||
for fault in ("allocation", "write-1", "write-2", "mode", "rename"):
|
||||
with self.subTest(pinned=pinned, automatic=automatic, fault=fault):
|
||||
e = self.exercise(pinned=pinned, automatic=automatic, unit_fault=fault)
|
||||
r = e["result"]
|
||||
self.assertEqual(r.returncode, 1, r.stdout + r.stderr)
|
||||
self.assertIn("INJECTED_UNIT_", r.stderr)
|
||||
self.assertIn("installation is incomplete", r.stdout + r.stderr)
|
||||
self.assertNotIn("start pulse", e["events"])
|
||||
self.assertNotIn("marker", e["events"])
|
||||
self.assertNotIn("complete", e["events"])
|
||||
self.assertNotIn(["daemon-reload"], e["calls"])
|
||||
self.assertEqual(e["unit"], "")
|
||||
self.assertIn("PULSE_MOCK_MODE=false", e["env"])
|
||||
|
||||
def test_both_update_paths_repair_config_and_unit_before_activation(self):
|
||||
for pinned in (True, False):
|
||||
for automatic in (True, False):
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue