Add --agent-id-file so containerized agents keep a stable identity

Pulse agents derive their identity from /etc/machine-id by default. In
Docker containers (especially nested in LXCs), /etc/machine-id is not
guaranteed stable across container recreation: a fresh image instance
gets a new machine-id, and the resulting AgentID drift causes the
server to reject reports with 401 because the API token is bound to
the original AgentID via the bound_agent_id token-metadata check
(internal/api/router.go:1448-1458). Refs #1447.

Add a --agent-id-file (and PULSE_AGENT_ID_FILE env var) flag that:

  1. Reads the persisted AgentID from the file on start, when present,
     and short-circuits machine-id detection. The user mounts the file
     as a Docker volume (e.g. -v pulse-agent-id:/var/lib/pulse-agent)
     so it survives container recreation.
  2. On first start (or when the file is missing/empty), the existing
     machine-id derivation runs and the resolved ID is written to the
     file atomically (tmp + rename, 0600 perms, parent dir created).

Subsequent restarts of the container — even after `docker rm -f` and
a fresh `docker run` — read the same ID from the volume and the
server keeps recognising the agent.

Default is no flag set, which preserves the current
/etc/machine-id-derived behaviour for non-containerized installs.

cmd/pulse-agent/main.go

@@ -2,12 +2,15 @@ package main
 
 import (
 	"context"
+	"errors"
 	"flag"
 	"fmt"
+	"io/fs"
 	"net/http"
 	"os"
 	"os/exec"
 	"os/signal"
+	"path/filepath"
 	"reflect"
 	"strconv"
 	"strings"
@@ -123,6 +126,25 @@ func run(ctx context.Context, args []string, getenv func(string) string) error {
 	// 2b. Compute Agent ID if missing (needed for remote config)
 	// We replicate the logic from hostagent.New to ensure we get the same ID
 	lookupHostname := strings.TrimSpace(cfg.HostnameOverride)
+	// Prefer a persisted ID from --agent-id-file if the file is configured
+	// and exists. This is what prevents Docker-container restarts from
+	// breaking the server's bound-token agent association (#1447): the user
+	// mounts the file as a volume, the agent reads its prior ID on each
+	// start, and the server keeps recognising it.
+	if cfg.AgentID == "" && cfg.AgentIDFile != "" {
+		if persisted, err := readAgentIDFile(cfg.AgentIDFile); err == nil && persisted != "" {
+			cfg.AgentID = persisted
+			logger.Info().
+				Str("path", cfg.AgentIDFile).
+				Str("agentID", cfg.AgentID).
+				Msg("Loaded persisted agent ID")
+		} else if err != nil && !errors.Is(err, fs.ErrNotExist) {
+			logger.Warn().
+				Err(err).
+				Str("path", cfg.AgentIDFile).
+				Msg("Failed to read agent-id-file; will fall back to machine-id and rewrite the file")
+		}
+	}
 	if cfg.AgentID == "" {
 		// Use a short timeout for host info
 		hCtx, hCancel := context.WithTimeout(ctx, 5*time.Second)
@@ -143,6 +165,17 @@ func run(ctx context.Context, args []string, getenv func(string) string) error {
 			logger.Warn().Err(err).Msg("Failed to fetch host info for Agent ID generation")
 		}
 	}
+	// Persist the resolved ID for next run if the file is configured but did
+	// not yet exist (or held a different value). Subsequent restarts will
+	// then read from the file and skip machine-id detection entirely.
+	if cfg.AgentID != "" && cfg.AgentIDFile != "" {
+		if err := writeAgentIDFile(cfg.AgentIDFile, cfg.AgentID); err != nil {
+			logger.Warn().
+				Err(err).
+				Str("path", cfg.AgentIDFile).
+				Msg("Failed to persist agent ID; ID will be re-derived on next start")
+		}
+	}
 	if lookupHostname == "" {
 		lookupHostname = strings.TrimSpace(cfg.HostnameOverride)
 		if lookupHostname == "" {
@@ -392,6 +425,64 @@ func run(ctx context.Context, args []string, getenv func(string) string) error {
 	return nil
 }
 
+// readAgentIDFile reads a persisted agent identifier from the given path.
+// Returns the trimmed contents on success, an empty string + nil if the file
+// is empty, or an error wrapping fs.ErrNotExist when the file is missing
+// (callers can use errors.Is to distinguish first-start from broken state).
+func readAgentIDFile(path string) (string, error) {
+	if path == "" {
+		return "", nil
+	}
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return "", err
+	}
+	return strings.TrimSpace(string(data)), nil
+}
+
+// writeAgentIDFile persists the agent identifier to disk so a future restart
+// of the same agent can re-use it. The file is written atomically (write to
+// tmp + rename) so a crash mid-write cannot leave a half-truncated file.
+// Parent directories are created if missing. Permissions are 0600 because
+// the value is a long-lived identity tied to the agent's auth token; world
+// readability is unnecessary.
+func writeAgentIDFile(path, id string) error {
+	if path == "" || id == "" {
+		return nil
+	}
+	dir := filepath.Dir(path)
+	if dir != "" && dir != "." {
+		if err := os.MkdirAll(dir, 0o700); err != nil {
+			return fmt.Errorf("create agent-id-file directory: %w", err)
+		}
+	}
+	tmp, err := os.CreateTemp(dir, ".agent-id-*")
+	if err != nil {
+		return fmt.Errorf("create agent-id-file temp: %w", err)
+	}
+	tmpPath := tmp.Name()
+	cleanup := func() { _ = os.Remove(tmpPath) }
+	if _, err := tmp.WriteString(id + "\n"); err != nil {
+		_ = tmp.Close()
+		cleanup()
+		return fmt.Errorf("write agent-id-file: %w", err)
+	}
+	if err := tmp.Chmod(0o600); err != nil {
+		_ = tmp.Close()
+		cleanup()
+		return fmt.Errorf("chmod agent-id-file: %w", err)
+	}
+	if err := tmp.Close(); err != nil {
+		cleanup()
+		return fmt.Errorf("close agent-id-file: %w", err)
+	}
+	if err := os.Rename(tmpPath, path); err != nil {
+		cleanup()
+		return fmt.Errorf("rename agent-id-file: %w", err)
+	}
+	return nil
+}
+
 func cleanupDockerAgent(agent RunnableCloser, logger *zerolog.Logger) {
 	if agent == nil || reflect.ValueOf(agent).IsNil() {
 		return
@@ -458,6 +549,7 @@ type Config struct {
 	Interval           time.Duration
 	HostnameOverride   string
 	AgentID            string
+	AgentIDFile        string
 	Tags               []string
 	InsecureSkipVerify bool
 	LogLevel           zerolog.Level
@@ -507,6 +599,7 @@ func loadConfig(args []string, getenv func(string) string) (Config, error) {
 	envInterval := strings.TrimSpace(getenv("PULSE_INTERVAL"))
 	envHostname := strings.TrimSpace(getenv("PULSE_HOSTNAME"))
 	envAgentID := strings.TrimSpace(getenv("PULSE_AGENT_ID"))
+	envAgentIDFile := strings.TrimSpace(getenv("PULSE_AGENT_ID_FILE"))
 	envInsecure := strings.TrimSpace(getenv("PULSE_INSECURE_SKIP_VERIFY"))
 	envTags := strings.TrimSpace(getenv("PULSE_TAGS"))
 	envLogLevel := strings.TrimSpace(getenv("LOG_LEVEL"))
@@ -577,6 +670,7 @@ func loadConfig(args []string, getenv func(string) string) (Config, error) {
 	intervalFlag := fs.Duration("interval", defaultInterval, "Reporting interval")
 	hostnameFlag := fs.String("hostname", envHostname, "Override hostname")
 	agentIDFlag := fs.String("agent-id", envAgentID, "Override agent identifier")
+	agentIDFileFlag := fs.String("agent-id-file", envAgentIDFile, "Path to a file storing the agent identifier (read on start, written on first start). Mount this file as a volume to keep the agent's identity stable across container recreation, which prevents the server's bound-token from rejecting the agent with 401 (#1447).")
 	insecureFlag := fs.Bool("insecure", utils.ParseBool(envInsecure), "Skip TLS verification")
 	logLevelFlag := fs.String("log-level", defaultLogLevel(envLogLevel), "Log level")
 
@@ -657,6 +751,7 @@ func loadConfig(args []string, getenv func(string) string) (Config, error) {
 		Interval:                  *intervalFlag,
 		HostnameOverride:          strings.TrimSpace(*hostnameFlag),
 		AgentID:                   strings.TrimSpace(*agentIDFlag),
+		AgentIDFile:               strings.TrimSpace(*agentIDFileFlag),
 		Tags:                      tags,
 		InsecureSkipVerify:        *insecureFlag,
 		LogLevel:                  logLevel,

cmd/pulse-agent/main_test.go

@@ -5,9 +5,11 @@ import (
 	"errors"
 	"flag"
 	"io"
+	"io/fs"
 	"net/http"
 	"net/http/httptest"
 	"os"
+	"path/filepath"
 	"reflect"
 	"strings"
 	"sync/atomic"
@@ -1521,3 +1523,80 @@ func TestRun_KubeRetry(t *testing.T) {
 		t.Errorf("expected at least 2 calls to newKubeAgent, got %d", calls)
 	}
 }
+
+func TestAgentIDFilePersistence(t *testing.T) {
+	t.Run("read returns empty when path is empty", func(t *testing.T) {
+		id, err := readAgentIDFile("")
+		if err != nil {
+			t.Fatalf("expected no error for empty path, got %v", err)
+		}
+		if id != "" {
+			t.Errorf("expected empty id, got %q", id)
+		}
+	})
+
+	t.Run("read returns fs.ErrNotExist for missing file", func(t *testing.T) {
+		dir := t.TempDir()
+		_, err := readAgentIDFile(filepath.Join(dir, "missing"))
+		if !errors.Is(err, fs.ErrNotExist) {
+			t.Fatalf("expected fs.ErrNotExist, got %v", err)
+		}
+	})
+
+	t.Run("write then read round-trips the ID", func(t *testing.T) {
+		dir := t.TempDir()
+		path := filepath.Join(dir, "subdir", "agent-id")
+		const id = "1234abcd-5678-90ef-1234-567890abcdef"
+
+		if err := writeAgentIDFile(path, id); err != nil {
+			t.Fatalf("write failed: %v", err)
+		}
+
+		got, err := readAgentIDFile(path)
+		if err != nil {
+			t.Fatalf("read failed: %v", err)
+		}
+		if got != id {
+			t.Errorf("round-trip mismatch: got %q, want %q", got, id)
+		}
+
+		// File must be 0600 — the ID is auth-adjacent, not world-readable.
+		info, err := os.Stat(path)
+		if err != nil {
+			t.Fatalf("stat: %v", err)
+		}
+		if mode := info.Mode().Perm(); mode != 0o600 {
+			t.Errorf("file permissions = %v, want 0600", mode)
+		}
+	})
+
+	t.Run("write strips trailing newline on read", func(t *testing.T) {
+		dir := t.TempDir()
+		path := filepath.Join(dir, "agent-id")
+		// Simulate a hand-crafted file with whitespace.
+		if err := os.WriteFile(path, []byte("  abc-123  \n\n"), 0o600); err != nil {
+			t.Fatalf("seed file: %v", err)
+		}
+		got, err := readAgentIDFile(path)
+		if err != nil {
+			t.Fatalf("read: %v", err)
+		}
+		if got != "abc-123" {
+			t.Errorf("expected trimmed value, got %q", got)
+		}
+	})
+
+	t.Run("write is a no-op when path is empty or id is empty", func(t *testing.T) {
+		if err := writeAgentIDFile("", "some-id"); err != nil {
+			t.Errorf("expected no-op for empty path, got %v", err)
+		}
+		dir := t.TempDir()
+		path := filepath.Join(dir, "agent-id")
+		if err := writeAgentIDFile(path, ""); err != nil {
+			t.Errorf("expected no-op for empty id, got %v", err)
+		}
+		if _, err := os.Stat(path); !errors.Is(err, fs.ErrNotExist) {
+			t.Errorf("expected file not to be created, got err=%v", err)
+		}
+	})
+}