Pulse/scripts/installtests/server_reset_safety_test.go
pulse-triage[bot] ccd3e4b05b Overlap isolated server reset and removal safety checks
The installer package still serialises these independent temporary-state matrices. Admit their roots and cases through Go test's existing parallel budget, with each watchdog starting inside its admitted case. Preserve every sourced installer action, preservation assertion, case, timeout and cleanup.

Contract-Neutral: Only independent test scheduling changes; no production installer, safety contract, assertion, deadline, selected case or frontend change.
Change-source: pulse-maintainer
2026-10-30 02:48:17 +00:00

381 lines
16 KiB
Go

package installtests
import (
"context"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
)
// This exercises reset_pulse from the actual sourced installer. All systemctl
// calls pass through the real timeout binary to a stateful executable double.
// Deletion is confined to fixture configuration; no host installation is used.
// Each case owns its files and child environment. Use Go's existing parallel
// budget, starting the unchanged action/fixture deadlines only after admission.
func TestRootInstallResetRejectsUnusableObservationsBeforeMutation(t *testing.T) {
t.Parallel()
for _, unit := range []string{"pulse.service", "pulse-update.timer", "pulse-update.service", "pulse-backend.service", "pulse-backend-update.timer", "pulse-backend-update.service"} {
for _, mode := range []string{"query-fails", "empty-load", "unknown-load", "empty-active", "unsettled", "empty-enabled", "unknown-enabled", "enabled-query-fails"} {
t.Run(unit+"/"+mode, func(t *testing.T) {
t.Parallel()
r := runServerResetFixture(t, mode, unit, "active", "enabled", false)
assertResetIncomplete(t, r, true)
for _, action := range []string{"stop ", "disable ", "start ", "enable "} {
if strings.Contains(r.calls, action) {
t.Fatalf("bad initial observation allowed mutation %s:\n%s", action, r.calls)
}
}
})
}
}
}
func TestRootInstallResetPreservesDataOnUnconfirmedQuiescence(t *testing.T) {
t.Parallel()
for _, unit := range []string{"pulse.service", "pulse-update.timer", "pulse-update.service", "pulse-backend.service", "pulse-backend-update.timer", "pulse-backend-update.service"} {
for _, mode := range []string{"stop-fails", "stop-ignored", "stop-readback-fails", "still-failed", "reactivated"} {
t.Run(unit+"/"+mode, func(t *testing.T) {
t.Parallel()
assertResetIncomplete(t, runServerResetFixture(t, mode, unit, "active", "enabled", false), true)
})
}
}
for _, unit := range []string{"pulse-update.timer", "pulse-backend-update.timer"} {
for _, mode := range []string{"disable-fails", "disable-ignored", "reenabled"} {
t.Run(unit+"/"+mode, func(t *testing.T) {
t.Parallel()
assertResetIncomplete(t, runServerResetFixture(t, mode, unit, "active", "enabled", false), true)
})
}
}
}
func TestRootInstallResetRestoresOnlyPriorActiveStates(t *testing.T) {
t.Parallel()
for _, active := range []string{"active", "inactive", "failed"} {
for _, enabled := range []string{"enabled", "enabled-runtime", "disabled", "static", "indirect", "masked", "masked-runtime", "linked", "linked-runtime"} {
t.Run(active+"/"+enabled, func(t *testing.T) {
t.Parallel()
r := runServerResetFixture(t, "normal", "", active, enabled, false)
if r.err != nil || !strings.Contains(r.output, "Pulse configuration has been reset") || r.configPresent {
t.Fatalf("reset failed: %v\n%s\n%s", r.err, r.output, r.calls)
}
deleteAt := strings.Index(r.calls, "DELETE ")
if deleteAt < 0 {
t.Fatal("reset did not delete configuration")
}
for _, unit := range resetFixtureUnits(false) {
if strings.Count(r.calls[:deleteAt], "show "+unit+" --property=ActiveState") < 3 {
t.Fatalf("%s lacked snapshot, stop readback and final pre-delete reconciliation:\n%s", unit, r.calls)
}
expected := active
if active == "failed" || strings.HasSuffix(unit, "-update.service") {
expected = "inactive"
}
if r.states[unit+".active"] != expected || r.states[unit+".enabled"] != enabled {
t.Fatalf("%s not restored: %v", unit, r.states)
}
if strings.HasSuffix(unit, "-update.service") && strings.Contains(r.calls, "start "+unit) {
t.Fatalf("interrupted updater was replayed:\n%s", r.calls)
}
}
if active != "active" && strings.Contains(r.calls, "start ") {
t.Fatalf("originally inactive/failed unit was started:\n%s", r.calls)
}
if active == "active" {
firstTimer := strings.Index(r.calls, "stop pulse-update.timer")
lastTimer := strings.Index(r.calls, "stop pulse-backend-update.timer")
firstUpdate := strings.Index(r.calls, "stop pulse-update.service")
lastUpdate := strings.Index(r.calls, "stop pulse-backend-update.service")
server := strings.Index(r.calls, "stop pulse.service")
if !(firstTimer < lastTimer && lastTimer < firstUpdate && firstUpdate < lastUpdate && lastUpdate < server && server < deleteAt) {
t.Fatalf("unsafe writer stop ordering:\n%s", r.calls)
}
if !strings.Contains(r.output, "remains stopped and was not replayed") {
t.Fatalf("interrupted updater disposition is hidden:\n%s", r.output)
}
}
if enabled == "enabled-runtime" && (!strings.Contains(r.calls, "disable --runtime pulse-update.timer") || !strings.Contains(r.calls, "enable --runtime pulse-update.timer")) {
t.Fatalf("runtime enablement was not preserved:\n%s", r.calls)
}
for _, unit := range []string{"pulse.service", "pulse-backend.service", "pulse-update.service", "pulse-backend-update.service"} {
if strings.Contains(r.calls, "enable "+unit) || strings.Contains(r.calls, "disable "+unit) {
t.Fatalf("non-timer enablement was changed:\n%s", r.calls)
}
}
})
}
}
}
func TestRootInstallResetReportsIncompleteDeletionOrRestoration(t *testing.T) {
t.Parallel()
for _, tc := range []struct{ mode, target string }{
{"delete-fails", ""},
{"start-fails", "pulse.service"},
{"start-ignored", "pulse.service"},
{"start-readback-fails", "pulse.service"},
{"enable-fails", "pulse-update.timer"},
{"enable-ignored", "pulse-update.timer"},
{"start-fails", "pulse-update.timer"},
{"start-ignored", "pulse-update.timer"},
{"start-readback-fails", "pulse-update.timer"},
{"final-updater-active", "pulse-update.service"},
} {
t.Run(tc.mode+"/"+tc.target, func(t *testing.T) {
t.Parallel()
r := runServerResetFixture(t, tc.mode, tc.target, "active", "enabled", false)
assertResetIncomplete(t, r, tc.mode == "delete-fails")
if tc.mode == "delete-fails" || tc.target == "pulse.service" {
if strings.Contains(r.calls, "start pulse-update.timer") || strings.Contains(r.calls, "enable pulse-update.timer") {
t.Fatalf("timer restored despite uncertain deletion/server recovery:\n%s", r.calls)
}
}
})
}
}
func TestRootInstallResetEnforcesSystemdDeadlines(t *testing.T) {
t.Parallel()
for _, mode := range []string{"query-timeout", "stop-timeout"} {
t.Run(mode, func(t *testing.T) {
t.Parallel()
start := time.Now()
r := runServerResetFixture(t, mode, "pulse-update.service", "active", "enabled", false)
assertResetIncomplete(t, r, true)
if elapsed := time.Since(start); elapsed >= 15*time.Second {
t.Fatalf("bounded systemctl action took %s", elapsed)
}
})
}
}
func TestRootInstallResetDoesNotStartAnIdleServerWhenTimerWasActive(t *testing.T) {
t.Parallel()
r := runServerResetFixture(t, "mixed", "", "active", "enabled", false)
if r.err != nil || r.configPresent || strings.Contains(r.calls, "start pulse.service") || strings.Contains(r.calls, "start pulse-backend.service") {
t.Fatalf("idle/failed server was restarted: %v\n%s\n%s", r.err, r.output, r.calls)
}
if !strings.Contains(r.calls, "start pulse-update.timer") || r.states["pulse.service.active"] != "inactive" {
t.Fatalf("timer/server states not independently restored: %v\n%s", r.states, r.calls)
}
}
func TestRootInstallResetKeepsCustomInstanceIsolated(t *testing.T) {
t.Parallel()
r := runServerResetFixture(t, "normal", "", "active", "enabled", true)
if r.err != nil || r.configPresent {
t.Fatalf("custom reset failed: %v\n%s\n%s", r.err, r.output, r.calls)
}
for _, unit := range resetFixtureUnits(false) {
if strings.Contains(r.calls, unit) {
t.Fatalf("custom reset touched other installation %s:\n%s", unit, r.calls)
}
}
}
func TestRootInstallResetKeepsUpdateSuffixedServerRole(t *testing.T) {
t.Parallel()
r := runServerResetFixture(t, "update-named-server", "", "active", "enabled", true)
if r.err != nil || r.states["pulse-other-update.service.active"] != "active" || !strings.Contains(r.calls, "start pulse-other-update.service") || strings.Contains(r.calls, "start pulse-other-update-update.service") {
t.Fatalf("server name was confused with updater role: %v\n%s\n%s", r.err, r.output, r.calls)
}
}
func TestRootInstallResetHandlesAbsentOptionalUnits(t *testing.T) {
t.Parallel()
r := runServerResetFixture(t, "optional-absent", "", "inactive", "disabled", false)
if r.err != nil || r.configPresent || strings.Contains(r.calls, "stop ") || strings.Contains(r.calls, "start ") || strings.Contains(r.calls, "disable ") {
t.Fatalf("absent optional units should not cause mutation: %v\n%s\n%s", r.err, r.output, r.calls)
}
assertResetIncomplete(t, runServerResetFixture(t, "server-absent", "", "inactive", "disabled", false), true)
}
type serverResetResult struct {
output, calls string
err error
configPresent bool
states map[string]string
}
func assertResetIncomplete(t *testing.T, r serverResetResult, preserve bool) {
t.Helper()
if r.err == nil || strings.Contains(r.output, "Pulse configuration has been reset") || strings.Contains(r.output, "Pulse has been reset") {
t.Fatalf("incomplete reset claimed completion: %v\n%s\n%s", r.err, r.output, r.calls)
}
if preserve && (!r.configPresent || strings.Contains(r.calls, "start ") || (strings.Contains(r.calls, "DELETE ") && !strings.Contains(r.output, "Configuration deletion failed"))) {
t.Fatalf("uncertain pre-deletion state allowed mutation/restart:\n%s\n%s", r.output, r.calls)
}
}
func resetFixtureUnits(custom bool) []string {
if custom {
return []string{"pulse-other-update.timer", "pulse-other-update.service", "pulse-other.service"}
}
return []string{"pulse-update.timer", "pulse-backend-update.timer", "pulse-update.service", "pulse-backend-update.service", "pulse.service", "pulse-backend.service"}
}
func runServerResetFixture(t *testing.T, mode, target, active, enabled string, custom bool) serverResetResult {
t.Helper()
dir := t.TempDir()
binDir := filepath.Join(dir, "tools")
for _, d := range []string{binDir, filepath.Join(dir, "config"), filepath.Join(dir, "install"), filepath.Join(dir, "other-config")} {
if err := os.Mkdir(d, 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(d, "preserve"), []byte("original"), 0600); err != nil {
t.Fatal(err)
}
}
service := "pulse"
if custom {
service = "pulse-other"
}
if mode == "update-named-server" {
service = "pulse-other-update"
}
units := resetFixtureUnits(custom)
if mode == "update-named-server" {
for i := range units {
units[i] = strings.Replace(units[i], "pulse-other", service, 1)
}
}
for _, unit := range units {
load, a, e := "loaded", active, enabled
if mode == "mixed" {
if unit == "pulse.service" {
a = "inactive"
}
if unit == "pulse-backend.service" {
a = "failed"
}
}
if mode == "server-absent" || (mode == "optional-absent" && unit != "pulse.service") {
load, a, e = "not-found", "inactive", ""
}
for suffix, value := range map[string]string{"load": load, "active": a, "enabled": e} {
if err := os.WriteFile(filepath.Join(dir, unit+"."+suffix), []byte(value+"\n"), 0600); err != nil {
t.Fatal(err)
}
}
}
mock := `#!/bin/bash
set -eu
printf '%s\n' "$*" >> "$FIXTURE_DIR/calls"
action=$1; shift
runtime=false
if [[ "${1:-}" == --runtime ]]; then runtime=true; shift; fi
if [[ "${1:-}" == --quiet ]]; then shift; fi
unit=${1:-}; shift || true
[[ "$unit" == *.service || "$unit" == *.timer ]] || unit="$unit.service"
if [[ "$action" == show ]]; then
prop=$1
if [[ "$unit" == "$TARGET" ]]; then
case "$MODE:$prop" in
query-fails:*) exit 1 ;;
query-timeout:*) sleep 7; exit 1 ;;
empty-load:--property=LoadState|empty-active:--property=ActiveState|empty-enabled:--property=UnitFileState) exit 0 ;;
unknown-load:--property=LoadState|unknown-enabled:--property=UnitFileState) echo unexpected; exit 0 ;;
unsettled:--property=ActiveState) echo activating; exit 0 ;;
enabled-query-fails:--property=UnitFileState) exit 1 ;;
stop-readback-fails:--property=ActiveState) [[ ! -e "$FIXTURE_DIR/$unit.stopped" ]] || exit 1 ;;
start-readback-fails:--property=ActiveState) [[ ! -e "$FIXTURE_DIR/$unit.started" ]] || exit 1 ;;
esac
if [[ -e "$FIXTURE_DIR/pulse-backend.service.stopped" && "$prop" == --property=ActiveState && "$MODE" == reactivated ]]; then echo active; exit 0; fi
if [[ -e "$FIXTURE_DIR/pulse-backend.service.stopped" && "$prop" == --property=UnitFileState && "$MODE" == reenabled ]]; then echo enabled; exit 0; fi
if [[ -e "$FIXTURE_DIR/pulse-update.timer.started" && "$prop" == --property=ActiveState && "$MODE" == final-updater-active ]]; then echo active; exit 0; fi
fi
case "$prop" in
--property=LoadState) cat "$FIXTURE_DIR/$unit.load" ;;
--property=ActiveState) cat "$FIXTURE_DIR/$unit.active" ;;
--property=UnitFileState) cat "$FIXTURE_DIR/$unit.enabled" ;;
*) exit 2 ;;
esac
elif [[ "$action" == is-active ]]; then
[[ "$(cat "$FIXTURE_DIR/$unit.active")" == active ]]
elif [[ "$action" == stop ]]; then
[[ "$unit:$MODE" != "$TARGET:stop-timeout" ]] || { sleep 7; exit 1; }
[[ "$unit:$MODE" != "$TARGET:stop-fails" ]] || exit 1
touch "$FIXTURE_DIR/$unit.stopped"
case "$unit:$MODE" in
"$TARGET:stop-ignored") ;;
"$TARGET:still-failed") echo failed > "$FIXTURE_DIR/$unit.active" ;;
*) echo inactive > "$FIXTURE_DIR/$unit.active" ;;
esac
elif [[ "$action" == disable ]]; then
[[ "$unit:$MODE" != "$TARGET:disable-fails" ]] || exit 1
[[ "$unit:$MODE" == "$TARGET:disable-ignored" ]] || echo disabled > "$FIXTURE_DIR/$unit.enabled"
elif [[ "$action" == enable ]]; then
[[ "$unit:$MODE" != "$TARGET:enable-fails" ]] || exit 1
if [[ "$unit:$MODE" != "$TARGET:enable-ignored" ]]; then
if [[ "$runtime" == true ]]; then echo enabled-runtime; else echo enabled; fi > "$FIXTURE_DIR/$unit.enabled"
fi
elif [[ "$action" == start ]]; then
[[ "$unit:$MODE" != "$TARGET:start-fails" ]] || exit 1
touch "$FIXTURE_DIR/$unit.started"
[[ "$unit:$MODE" == "$TARGET:start-ignored" ]] || echo active > "$FIXTURE_DIR/$unit.active"
else exit 2
fi
`
if err := os.WriteFile(filepath.Join(binDir, "systemctl"), []byte(mock), 0700); err != nil {
t.Fatal(err)
}
installer := os.Getenv("PULSE_TEST_ROOT_INSTALLER")
if installer == "" {
installer = filepath.Join("..", "..", "install.sh")
}
installer, err := filepath.Abs(installer)
if err != nil {
t.Fatal(err)
}
script := `
source "$INSTALLER_UNDER_TEST"
CONFIG_DIR="$FIXTURE_DIR/config"
SERVICE_NAME="$SERVICE"
SERVICE_NAME_EXPLICIT="$CUSTOM"
UPDATE_SERVICE_UNIT="$SERVICE-update.service"
UPDATE_TIMER_UNIT="$SERVICE-update.timer"
check_root() { :; }
print_header() { :; }
detect_service_name() { echo "$SERVICE"; }
rm() {
echo "DELETE $*" >> "$FIXTURE_DIR/calls"
[[ "$MODE" != delete-fails ]] || return 1
local arg
for arg in "$@"; do
case "$arg" in -rf|--) ;; "$FIXTURE_DIR/config/"*) /bin/rm -rf -- "$arg" ;; *) return 91 ;; esac
done
}
reset_pulse
`
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
cmd := exec.CommandContext(ctx, "bash", "-c", script)
cmd.Env = append(os.Environ(), "PATH="+binDir+":"+os.Getenv("PATH"), "INSTALLER_UNDER_TEST="+installer, "FIXTURE_DIR="+dir, "MODE="+mode, "TARGET="+target, "SERVICE="+service, "CUSTOM="+boolString(custom))
out, runErr := cmd.CombinedOutput()
if ctx.Err() != nil {
t.Fatalf("fixture deadline: %v\n%s", ctx.Err(), out)
}
calls, _ := os.ReadFile(filepath.Join(dir, "calls"))
config, configErr := os.ReadFile(filepath.Join(dir, "config", "preserve"))
r := serverResetResult{output: string(out), calls: string(calls), err: runErr, configPresent: configErr == nil && string(config) == "original", states: map[string]string{}}
for _, d := range []string{"install", "other-config"} {
b, e := os.ReadFile(filepath.Join(dir, d, "preserve"))
if e != nil || string(b) != "original" {
t.Fatalf("reset changed %s: %v\n%s", d, e, out)
}
}
for _, unit := range units {
for _, suffix := range []string{"active", "enabled"} {
b, e := os.ReadFile(filepath.Join(dir, unit+"."+suffix))
if e != nil {
t.Fatal(e)
}
r.states[unit+"."+suffix] = strings.TrimSpace(string(b))
}
}
return r
}