Pulse/internal/hostagent/agent.go

package hostagent

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"net"
	"net/http"
	"net/netip"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/agenttls"
	"github.com/rcourtman/pulse-go-rewrite/internal/agentupdate"
	"github.com/rcourtman/pulse-go-rewrite/internal/sensors"
	"github.com/rcourtman/pulse-go-rewrite/internal/utils"
	agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host"
	"github.com/rs/zerolog"
	gohost "github.com/shirou/gopsutil/v4/host"
)

// Config controls the behaviour of the runtime-side Unified Agent.
type Config struct {
	PulseURL           string
	APIToken           string
	Interval           time.Duration
	HostnameOverride   string
	AgentID            string
	AgentType          string // "unified" when running as part of pulse-agent, empty for standalone
	AgentVersion       string // Version to report; if empty, uses the Unified Agent binary version
	Tags               []string
	InsecureSkipVerify bool
	CACertPath         string
	RunOnce            bool
	LogLevel           zerolog.Level
	Logger             *zerolog.Logger

	// Proxmox integration
	EnableProxmox bool   // If true, creates Proxmox API token and registers node on startup
	ProxmoxType   string // "pve", "pbs", or "" for auto-detect

	// Security options
	EnableCommands bool // If true, enables the command execution feature (AI auto-fix)

	// Enrollment
	Enroll bool // If true, exchange bootstrap token for runtime token on startup

	// Disk filtering
	DiskExclude []string // Mount points or path prefixes to exclude from disk monitoring

	// State directory for persistent files (agent-id, proxmox registration, etc.)
	StateDir string // Default: /var/lib/pulse-agent

	// Network configuration
	ReportIP    string // IP address to report instead of auto-detected (for multi-NIC systems)
	DisableCeph bool   // If true, disables local Ceph status polling

	Collector SystemCollector // Optional: override default system information collector (for testing)
}

// Agent is responsible for collecting host metrics and shipping them to Pulse.
type Agent struct {
	cfg        Config
	logger     zerolog.Logger
	httpClient *http.Client

	hostInfo               *gohost.InfoStat
	hostname               string
	displayName            string
	platform               string
	osName                 string
	osVersion              string
	kernelVersion          string
	architecture           string
	machineID              string
	agentID                string
	agentVersion           string
	updatedFrom            string // Previous version if recently auto-updated (reported once)
	reportIP               string // User-specified IP to report (for multi-NIC systems)
	interval               time.Duration
	stateDir               string
	trimmedPulseURL        string
	reportBuffer           *utils.Queue[agentshost.Report]
	commandClient          *CommandClient
	commandClientMu        sync.Mutex
	commandClientRunCancel context.CancelFunc
	commandClientParentCtx context.Context
	collector              SystemCollector
}

const defaultInterval = 30 * time.Second
const defaultStateDir = "/var/lib/pulse-agent"
const agentReportEndpoint = "/api/agents/agent/report"

type reportHTTPStatusError struct {
	Endpoint   string
	Status     string
	StatusCode int
}

func (e *reportHTTPStatusError) Error() string {
	return fmt.Sprintf("pulse responded with status %s", e.Status)
}

var newCommandClient = NewCommandClient
var runCommandClient = func(client *CommandClient, ctx context.Context) error {
	return client.Run(ctx)
}
var getUpdatedFromVersion = agentupdate.GetUpdatedFromVersion

// New constructs a fully initialised runtime-side Unified Agent.
func New(cfg Config) (*Agent, error) {
	if cfg.Interval <= 0 {
		cfg.Interval = defaultInterval
	}

	if strings.TrimSpace(cfg.StateDir) == "" {
		cfg.StateDir = defaultStateDir
	}

	if zerolog.GlobalLevel() == zerolog.DebugLevel && cfg.LogLevel != zerolog.DebugLevel {
		zerolog.SetGlobalLevel(cfg.LogLevel)
	}

	if cfg.Logger == nil {
		defaultLogger := zerolog.New(zerolog.NewConsoleWriter()).
			Level(cfg.LogLevel).
			With().
			Timestamp().
			Logger()
		cfg.Logger = &defaultLogger
	}

	logger := cfg.Logger.Level(cfg.LogLevel).With().Str("component", "agent").Logger()

	if cfg.Enroll && strings.TrimSpace(cfg.APIToken) == "" {
		return nil, fmt.Errorf("api token is required when enrollment is enabled")
	}

	pulseURL := strings.TrimSpace(cfg.PulseURL)
	if pulseURL == "" {
		pulseURL = "http://localhost:7655"
	}
	var err error
	pulseURL, err = normalizePulseURL(pulseURL)
	if err != nil {
		return nil, fmt.Errorf("invalid pulse URL: %w", err)
	}
	cfg.PulseURL = pulseURL

	cfg.ProxmoxType, err = normalizeProxmoxType(cfg.ProxmoxType)
	if err != nil {
		return nil, err
	}

	cfg.ReportIP, err = normalizeReportIP(cfg.ReportIP)
	if err != nil {
		return nil, err
	}

	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	defer cancel()

	collector := cfg.Collector
	if collector == nil {
		collector = &defaultCollector{}
	}

	info, err := collector.HostInfo(ctx)
	if err != nil {
		return nil, fmt.Errorf("fetch host info: %w", err)
	}

	hostname := strings.TrimSpace(cfg.HostnameOverride)
	if hostname == "" {
		hostname = strings.TrimSpace(info.Hostname)
	}
	if hostname == "" {
		hostname = "unknown-host"
	}

	displayName := hostname

	machineID := GetReliableMachineID(collector, info.HostID, logger)

	agentID := strings.TrimSpace(cfg.AgentID)
	if agentID == "" {
		agentID = machineID
	}
	if agentID == "" {
		agentID = hostname
	}

	platform := normalisePlatform(info.Platform)
	// Use Platform (specific distro like "ubuntu") over PlatformFamily (distro family like "debian")
	// This ensures Ubuntu shows as "ubuntu 24.04" not "debian 24.04" (refs #927)
	osName := strings.TrimSpace(info.Platform)
	if osName == "" {
		osName = strings.TrimSpace(info.PlatformFamily)
	}
	osVersion := strings.TrimSpace(info.PlatformVersion)
	kernelVersion := strings.TrimSpace(info.KernelVersion)
	arch := strings.TrimSpace(info.KernelArch)
	if arch == "" {
		arch = runtime.GOARCH
	}
	tlsConfig, err := agenttls.NewClientTLSConfig(cfg.CACertPath, cfg.InsecureSkipVerify)
	if err != nil {
		return nil, fmt.Errorf("invalid CA bundle: %w", err)
	}

	client := &http.Client{
		Timeout: 15 * time.Second,
		Transport: &http.Transport{
			Proxy:           http.ProxyFromEnvironment,
			TLSClientConfig: tlsConfig,
		},
		// Disallow redirects for agent API calls. If a reverse proxy redirects
		// HTTP to HTTPS, Go's default behavior converts POST to GET (per HTTP spec),
		// causing 405 errors. Return an error with guidance instead.
		CheckRedirect: func(req *http.Request, via []*http.Request) error {
			return fmt.Errorf("server returned redirect to %s - if using a reverse proxy, ensure you use the correct protocol (https:// instead of http://) in your --url flag", req.URL)
		},
	}

	trimmedTags := make([]string, 0, len(cfg.Tags))
	seenTags := make(map[string]struct{}, len(cfg.Tags))
	for _, tag := range cfg.Tags {
		tag = strings.TrimSpace(tag)
		if tag == "" {
			continue
		}
		if _, exists := seenTags[tag]; exists {
			continue
		}
		seenTags[tag] = struct{}{}
		trimmedTags = append(trimmedTags, tag)
	}
	cfg.Tags = trimmedTags

	// Use configured version or fall back to package version
	agentVersion := cfg.AgentVersion
	if agentVersion == "" {
		agentVersion = Version
	}

	const bufferCapacity = 60

	// Check if agent was recently auto-updated (only reported once per restart)
	updatedFrom := getUpdatedFromVersion()
	if updatedFrom != "" {
		logger.Info().
			Str("previousVersion", updatedFrom).
			Str("currentVersion", agentVersion).
			Msg("Agent was auto-updated")
	}

	agent := &Agent{
		cfg:             cfg,
		logger:          logger,
		httpClient:      client,
		hostInfo:        info,
		hostname:        hostname,
		displayName:     displayName,
		platform:        platform,
		osName:          osName,
		osVersion:       osVersion,
		kernelVersion:   kernelVersion,
		architecture:    arch,
		machineID:       machineID,
		agentID:         agentID,
		agentVersion:    agentVersion,
		updatedFrom:     updatedFrom,
		reportIP:        cfg.ReportIP,
		stateDir:        cfg.StateDir,
		interval:        cfg.Interval,
		trimmedPulseURL: pulseURL,
		reportBuffer:    utils.New[agentshost.Report](bufferCapacity),
		collector:       collector,
	}

	// Create command client for AI command execution (only if enabled)
	if cfg.EnableCommands {
		agent.commandClient = newCommandClient(cfg, agentID, hostname, platform, agentVersion)
		cfg.Logger.Info().Msg("Command execution enabled via --enable-commands flag")
	} else {
		cfg.Logger.Info().Msg("Command execution disabled (use --enable-commands to enable)")
	}

	return agent, nil
}

func normalizeProxmoxType(raw string) (string, error) {
	value := strings.TrimSpace(strings.ToLower(raw))
	switch value {
	case "", "auto":
		return "", nil
	case "pve", "pbs":
		return value, nil
	default:
		return "", fmt.Errorf("invalid proxmox type %q: must be pve, pbs, or auto", raw)
	}
}

func normalizeReportIP(raw string) (string, error) {
	value := strings.TrimSpace(raw)
	if value == "" {
		return "", nil
	}

	addr, err := netip.ParseAddr(value)
	if err != nil {
		return "", fmt.Errorf("invalid report IP %q: must be a valid IPv4 or IPv6 address", raw)
	}
	if addr.IsUnspecified() {
		return "", fmt.Errorf("invalid report IP %q: unspecified addresses are not allowed", raw)
	}

	return addr.String(), nil
}

// Run executes the agent until the context is cancelled.
func (a *Agent) Run(ctx context.Context) error {
	if a.cfg.RunOnce {
		return a.runOnce(ctx)
	}

	a.commandClientMu.Lock()
	a.commandClientParentCtx = ctx
	commandClient := a.commandClient
	a.commandClientMu.Unlock()
	defer func() {
		a.commandClientMu.Lock()
		a.commandClientParentCtx = nil
		a.commandClientMu.Unlock()
		a.stopCommandClient(true)
	}()

	// Enrollment: exchange bootstrap token for runtime token (must run before
	// Proxmox setup and reporting, which require the runtime token's scopes).
	if a.cfg.Enroll {
		if err := a.runEnrollmentLoop(ctx); err != nil {
			return fmt.Errorf("enrollment failed: %w", err)
		}
		// Re-create command client with the new runtime token, since the
		// original was built with the bootstrap token during New().
		if a.cfg.EnableCommands {
			a.commandClientMu.Lock()
			a.commandClient = newCommandClient(a.cfg, a.agentID, a.hostname, a.platform, a.agentVersion)
			commandClient = a.commandClient
			a.commandClientMu.Unlock()
		}
	}

	// Proxmox setup (if enabled)
	if a.cfg.EnableProxmox {
		a.runProxmoxSetup(ctx)
	}

	// Start command client in background for AI command execution
	if commandClient != nil {
		a.startCommandClient(commandClient)
	}

	// Load any reports buffered from a previous shutdown
	a.loadPersistedBuffer()

	ticker := time.NewTicker(a.interval)
	defer ticker.Stop()
	defer a.persistBuffer()

	if err := a.process(ctx); err != nil && !errors.Is(err, context.Canceled) {
		a.logger.Error().
			Err(err).
			Str("hostname", a.hostname).
			Msg("Initial host report failed")
	}

	for {
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-ticker.C:
			if err := a.process(ctx); err != nil {
				if errors.Is(err, context.Canceled) {
					return err
				}
				a.logger.Error().
					Err(err).
					Str("hostname", a.hostname).
					Msg("Failed to process host report")
			}
		}
	}
}

func (a *Agent) startCommandClient(client *CommandClient) {
	if client == nil {
		return
	}

	a.commandClientMu.Lock()
	if a.commandClientRunCancel != nil {
		a.commandClientMu.Unlock()
		return
	}

	parentCtx := a.commandClientParentCtx
	if parentCtx == nil {
		parentCtx = context.Background()
	}
	runCtx, cancel := context.WithCancel(parentCtx)
	a.commandClientRunCancel = cancel
	a.commandClientMu.Unlock()

	go func() {
		if err := runCommandClient(client, runCtx); err != nil && !errors.Is(err, context.Canceled) {
			a.logger.Error().Err(err).Msg("Command client stopped with error")
		}
	}()
}

func (a *Agent) stopCommandClient(clearClient bool) {
	a.commandClientMu.Lock()
	client := a.commandClient
	cancel := a.commandClientRunCancel
	a.commandClientRunCancel = nil
	if clearClient {
		a.commandClient = nil
	}
	a.commandClientMu.Unlock()

	if cancel != nil {
		cancel()
	}
	if client != nil {
		if err := client.Close(); err != nil {
			a.logger.Debug().Err(err).Msg("Error closing command client connection")
		}
	}
}

func (a *Agent) runOnce(ctx context.Context) error {
	return a.process(ctx)
}

func (a *Agent) process(ctx context.Context) error {
	report, err := a.buildReport(ctx)
	if err != nil {
		return fmt.Errorf("build report: %w", err)
	}
	if err := a.sendReport(ctx, report); err != nil {
		var statusErr *reportHTTPStatusError
		if errors.As(err, &statusErr) && statusErr.StatusCode == http.StatusForbidden {
			a.logger.Error().
				Err(err).
				Str("endpoint", statusErr.Endpoint).
				Int("status_code", statusErr.StatusCode).
				Msg("Failed to send Unified Agent report (403 Forbidden). API token may lack 'Unified Agent reporting' scope. Set PULSE_ENABLE_HOST=false if host monitoring is not needed.")
			return nil
		}

		a.reportBuffer.Push(report)
		event := a.logger.Warn().
			Err(err).
			Str("endpoint", agentReportEndpoint).
			Int("buffered_reports", a.reportBuffer.Len())
		if errors.As(err, &statusErr) {
			event.Str("endpoint", statusErr.Endpoint)
			event.Int("status_code", statusErr.StatusCode)
		}
		event.Msg("Failed to send report, buffering")
		return nil
	}

	// If successful, try to flush buffer
	a.flushBuffer(ctx)

	a.logger.Debug().
		Str("hostname", report.Host.Hostname).
		Str("platform", report.Host.Platform).
		Msg("Unified Agent report sent")
	return nil
}

func (a *Agent) flushBuffer(ctx context.Context) {
	if a.reportBuffer.IsEmpty() {
		return
	}

	a.logger.Info().Int("count", a.reportBuffer.Len()).Msg("Flushing buffered reports")

	for !a.reportBuffer.IsEmpty() {
		// Peek first
		report, ok := a.reportBuffer.Peek()
		if !ok {
			break
		}

		if err := a.sendReport(ctx, report); err != nil {
			var statusErr *reportHTTPStatusError
			event := a.logger.Warn().
				Err(err).
				Str("endpoint", agentReportEndpoint).
				Int("remaining_buffered_reports", a.reportBuffer.Len())
			if errors.As(err, &statusErr) {
				event.Str("endpoint", statusErr.Endpoint)
				event.Int("status_code", statusErr.StatusCode)
			}
			event.Msg("Failed to flush buffered report, stopping flush")
			return
		}

		// Pop only on success
		a.reportBuffer.Pop()
	}
}

const bufferFileName = "report-buffer.json"

// persistBuffer writes buffered reports to disk on shutdown so they can be
// retransmitted on the next startup. Uses atomic write (tmp + rename) to
// prevent corruption if the process is killed mid-write.
func (a *Agent) persistBuffer() {
	items := a.reportBuffer.Items()
	if len(items) == 0 {
		return
	}

	if a.stateDir == "" {
		a.logger.Debug().Msg("No state dir configured, skipping buffer persistence")
		return
	}

	data, err := json.Marshal(items)
	if err != nil {
		a.logger.Warn().Err(err).Msg("Failed to marshal report buffer for persistence")
		return
	}

	if err := os.MkdirAll(a.stateDir, 0700); err != nil {
		a.logger.Warn().Err(err).Str("dir", a.stateDir).Msg("Failed to create state dir for buffer persistence")
		return
	}

	path := filepath.Join(a.stateDir, bufferFileName)
	tmpPath := path + ".tmp"
	if err := os.WriteFile(tmpPath, data, 0600); err != nil {
		a.logger.Warn().Err(err).Str("path", tmpPath).Msg("Failed to write report buffer temp file")
		return
	}
	if err := os.Rename(tmpPath, path); err != nil {
		a.logger.Warn().Err(err).Str("path", path).Msg("Failed to rename report buffer file")
		_ = os.Remove(tmpPath)
		return
	}

	a.logger.Info().Int("count", len(items)).Str("path", path).Msg("Persisted report buffer to disk")
}

// loadPersistedBuffer loads buffered reports from a previous shutdown and
// attempts to flush them. The file is deleted after loading regardless of
// whether the flush succeeds (items are pushed back into the in-memory buffer).
func (a *Agent) loadPersistedBuffer() {
	if a.stateDir == "" {
		return
	}

	path := filepath.Join(a.stateDir, bufferFileName)
	data, err := os.ReadFile(path)
	if err != nil {
		if !os.IsNotExist(err) {
			a.logger.Warn().Err(err).Str("path", path).Msg("Failed to read persisted report buffer")
		}
		return
	}

	// Always delete the file after reading
	_ = os.Remove(path)

	var items []agentshost.Report
	if err := json.Unmarshal(data, &items); err != nil {
		a.logger.Warn().Err(err).Str("path", path).Msg("Corrupt report buffer file, discarding")
		return
	}

	if len(items) == 0 {
		return
	}

	for _, item := range items {
		a.reportBuffer.Push(item)
	}

	a.logger.Info().Int("count", len(items)).Msg("Loaded persisted report buffer from disk")
}

func (a *Agent) buildReport(ctx context.Context) (agentshost.Report, error) {
	collectCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
	defer cancel()

	uptime, err := a.collector.HostUptime(collectCtx)
	if err != nil {
		a.logger.Debug().Err(err).Msg("Failed to collect host uptime; defaulting to 0")
	}
	snapshot, err := a.collector.Metrics(collectCtx, a.cfg.DiskExclude)
	if err != nil {
		return agentshost.Report{}, fmt.Errorf("collect metrics: %w", err)
	}

	// Collect temperature data (best effort - don't fail if unavailable)
	sensorData := a.collectTemperatures(collectCtx)

	// Collect S.M.A.R.T. disk data (best effort - don't fail if unavailable)
	smartData := a.collectSMARTData(collectCtx)
	if len(smartData) > 0 {
		sensorData.SMART = smartData
	}

	// Collect RAID array data (best effort - don't fail if unavailable)
	raidData := a.collectRAIDArrays(collectCtx)

	// Collect Unraid array topology (best effort - only on Unraid hosts).
	unraidData := a.collectUnraidStorage(collectCtx)

	// Collect Ceph cluster data (best effort - only on Ceph nodes)
	cephData := a.collectCephStatus(collectCtx)

	// Collect temperature data from Proxmox cluster peers via SSH (best effort).
	// Uses parent ctx, not collectCtx — cluster SSH has its own 15s budget that
	// would be capped by collectCtx's 10s timeout.
	clusterSensors := a.collectClusterSensors(ctx)

	// Carry updated_from on the first freshly built v6 report only. If that
	// report is buffered, the buffered copy still retains the field for retry.
	updatedFrom := a.updatedFrom
	if updatedFrom != "" {
		a.updatedFrom = ""
	}

	report := agentshost.Report{
		Agent: agentshost.AgentInfo{
			ID:              a.agentID,
			Version:         a.agentVersion,
			Type:            a.cfg.AgentType,
			IntervalSeconds: int(a.interval / time.Second),
			Hostname:        a.hostname,
			UpdatedFrom:     updatedFrom,
			CommandsEnabled: a.cfg.EnableCommands,
			DiskExclude:     append([]string(nil), a.cfg.DiskExclude...),
		},
		Host: agentshost.HostInfo{
			ID:            a.machineID,
			Hostname:      a.hostname,
			DisplayName:   a.displayName,
			MachineID:     a.machineID,
			Platform:      a.platform,
			OSName:        a.osName,
			OSVersion:     a.osVersion,
			KernelVersion: a.kernelVersion,
			Architecture:  a.architecture,
			CPUModel:      "",
			CPUCount:      snapshot.CPUCount,
			UptimeSeconds: int64(uptime),
			LoadAverage:   append([]float64(nil), snapshot.LoadAverage...),
			ReportIP:      a.reportIP,
		},
		Metrics: agentshost.Metrics{
			CPUUsagePercent: snapshot.CPUUsagePercent,
			Memory:          snapshot.Memory,
		},
		Disks:          append([]agentshost.Disk(nil), snapshot.Disks...),
		DiskIO:         append([]agentshost.DiskIO(nil), snapshot.DiskIO...),
		Network:        append([]agentshost.NetworkInterface(nil), snapshot.Network...),
		Sensors:        sensorData,
		RAID:           raidData,
		Unraid:         unraidData,
		Ceph:           cephData,
		ClusterSensors: clusterSensors,
		Tags:           append([]string(nil), a.cfg.Tags...),
		Timestamp:      a.collector.Now(),
	}

	return report, nil
}

func (a *Agent) sendReport(ctx context.Context, report agentshost.Report) error {
	payload, err := json.Marshal(report)
	if err != nil {
		return fmt.Errorf("marshal report: %w", err)
	}

	compressed, err := utils.CompressJSON(payload)
	if err != nil {
		return fmt.Errorf("compress report: %w", err)
	}

	endpoint := agentReportEndpoint
	url := fmt.Sprintf("%s%s", a.trimmedPulseURL, endpoint)
	req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(compressed))
	if err != nil {
		return fmt.Errorf("create request: %w", err)
	}

	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("Content-Encoding", "gzip")
	if token := strings.TrimSpace(a.cfg.APIToken); token != "" {
		req.Header.Set("Authorization", "Bearer "+token)
		req.Header.Set("X-API-Token", token)
	}
	req.Header.Set("User-Agent", "pulse-agent/"+Version)

	resp, err := a.httpClient.Do(req)
	if err != nil {
		return fmt.Errorf("send request: %w", err)
	}
	defer func() {
		if closeErr := resp.Body.Close(); closeErr != nil {
			a.logger.Debug().Err(closeErr).Msg("Failed to close report response body")
		}
	}()

	if resp.StatusCode >= 300 {
		return &reportHTTPStatusError{
			Endpoint:   endpoint,
			Status:     resp.Status,
			StatusCode: resp.StatusCode,
		}
	}

	// Parse response to check for server-side config overrides
	var reportResp struct {
		Success bool   `json:"success"`
		AgentID string `json:"agentId"`
		Config  *struct {
			CommandsEnabled *bool `json:"commandsEnabled"`
		} `json:"config,omitempty"`
	}
	if err := json.NewDecoder(resp.Body).Decode(&reportResp); err != nil {
		// Non-fatal: just log and continue
		a.logger.Debug().Err(err).Msg("Failed to parse report response, ignoring config")
		return nil
	}

	// Persist the server-acknowledged agent ID so uninstall can deregister.
	canonicalAgentID := strings.TrimSpace(reportResp.AgentID)
	if canonicalAgentID != "" {
		a.persistAgentID(canonicalAgentID)
	}

	// Apply server config overrides
	if reportResp.Config != nil && reportResp.Config.CommandsEnabled != nil {
		a.applyRemoteConfig(*reportResp.Config.CommandsEnabled)
	}

	return nil
}

// persistAgentID writes the server-assigned agent ID to the state directory.
// This file is read by the uninstall script to deregister the agent from the server.
// Errors are debug-logged, never fatal — same resilience pattern as proxmox_setup.go.
func (a *Agent) persistAgentID(agentID string) {
	if a.stateDir == "" {
		return
	}
	if err := a.collector.MkdirAll(a.stateDir, 0700); err != nil {
		a.logger.Debug().Err(err).Msg("Failed to create state directory for agent-id")
		return
	}
	agentIDPath := filepath.Join(a.stateDir, "agent-id")
	if err := a.collector.WriteFile(agentIDPath, []byte(agentID), 0600); err != nil {
		a.logger.Debug().Err(err).Msg("Failed to persist agent-id")
		return
	}
	if err := a.collector.Chmod(agentIDPath, 0600); err != nil {
		a.logger.Debug().Err(err).Msg("Failed to enforce agent-id file permissions")
	}
}

// applyRemoteConfig applies server-side configuration overrides.
// Currently handles enabling/disabling the command execution feature dynamically.
func (a *Agent) applyRemoteConfig(commandsEnabled bool) {
	var (
		clientToStart *CommandClient
		shouldStop    bool
	)

	a.commandClientMu.Lock()
	currentlyEnabled := a.commandClient != nil
	switch {
	case commandsEnabled && !currentlyEnabled:
		clientToStart = NewCommandClient(a.cfg, a.agentID, a.hostname, a.platform, a.agentVersion)
		a.commandClient = clientToStart
	case !commandsEnabled && currentlyEnabled:
		shouldStop = true
	}
	a.commandClientMu.Unlock()

	if clientToStart != nil {
		a.logger.Info().Msg("Server enabled command execution - starting command client")
		a.startCommandClient(clientToStart)
	} else if shouldStop {
		a.logger.Info().Msg("Server disabled command execution - stopping command client")
		a.stopCommandClient(true)
	}
}

func normalisePlatform(platform string) string {
	platform = strings.ToLower(strings.TrimSpace(platform))
	switch platform {
	case "darwin":
		return "macos"
	default:
		return platform
	}
}

func normalizePulseURL(rawURL string) (string, error) {
	parsed, err := url.Parse(rawURL)
	if err != nil {
		return "", fmt.Errorf("pulse URL %q is invalid: %w", rawURL, err)
	}

	if parsed.Scheme == "" {
		return "", fmt.Errorf("pulse URL %q must include scheme (https:// or loopback http://)", rawURL)
	}
	if parsed.Host == "" {
		return "", fmt.Errorf("pulse URL %q must include host", rawURL)
	}
	if parsed.User != nil {
		return "", fmt.Errorf("pulse URL %q must not include user credentials", rawURL)
	}
	if parsed.RawQuery != "" || parsed.Fragment != "" {
		return "", fmt.Errorf("pulse URL %q must not include query or fragment", rawURL)
	}

	scheme := strings.ToLower(parsed.Scheme)
	switch scheme {
	case "https":
		// Always allowed.
	case "http":
		if !isLoopbackOrPrivateHost(parsed.Hostname()) {
			return "", fmt.Errorf("pulse URL %q must use https unless host is loopback or private network", rawURL)
		}
	default:
		return "", fmt.Errorf("pulse URL %q has unsupported scheme %q", rawURL, parsed.Scheme)
	}

	parsed.Scheme = scheme
	parsed.Path = strings.TrimRight(parsed.Path, "/")
	parsed.RawPath = strings.TrimRight(parsed.RawPath, "/")

	return parsed.String(), nil
}

// isLoopbackOrPrivateHost returns true for loopback and RFC1918 private
// network addresses.  HTTP (non-TLS) is safe over a local/private network;
// the scheme guard only needs to prevent plaintext over the public internet.
func isLoopbackOrPrivateHost(host string) bool {
	if strings.EqualFold(host, "localhost") {
		return true
	}

	ip := net.ParseIP(host)
	if ip == nil {
		return false
	}
	return ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast()
}

// collectTemperatures attempts to collect temperature data from the local system.
// Returns an empty Sensors struct if collection fails (best-effort).
func (a *Agent) collectTemperatures(ctx context.Context) agentshost.Sensors {
	switch a.collector.GOOS() {
	case "linux":
		// Continue below with lm-sensors path
	case "freebsd":
		return a.collectFreeBSDTemperatures(ctx)
	default:
		return agentshost.Sensors{}
	}

	// Collect sensor JSON output
	jsonOutput, err := a.collector.SensorsLocal(ctx)
	if err != nil {
		a.logger.Debug().Err(err).Msg("Failed to collect sensor data (lm-sensors may not be installed)")
		return agentshost.Sensors{}
	}

	// Parse the sensor output
	tempData, err := a.collector.SensorsParse(jsonOutput)
	if err != nil {
		a.logger.Debug().Err(err).Msg("Failed to parse sensor data")
		return agentshost.Sensors{}
	}

	if !tempData.Available {
		a.logger.Debug().Msg("No temperature sensors available on this system")
		return agentshost.Sensors{}
	}

	result := convertTemperatureDataToSensors(tempData)

	// Collect power consumption data (Intel RAPL, etc.)
	if powerData, err := a.collector.SensorsPower(ctx); err == nil && powerData != nil && powerData.Available {
		result.PowerWatts = make(map[string]float64)
		if powerData.PackageWatts > 0 {
			result.PowerWatts["cpu_package"] = powerData.PackageWatts
		}
		if powerData.CoreWatts > 0 {
			result.PowerWatts["cpu_core"] = powerData.CoreWatts
		}
		if powerData.DRAMWatts > 0 {
			result.PowerWatts["dram"] = powerData.DRAMWatts
		}
		a.logger.Debug().
			Float64("packageWatts", powerData.PackageWatts).
			Str("source", powerData.Source).
			Msg("Collected power data")
	}

	a.logger.Debug().
		Int("temperatureCount", len(result.TemperatureCelsius)).
		Int("fanCount", len(result.FanRPM)).
		Int("powerCount", len(result.PowerWatts)).
		Int("additionalCount", len(result.Additional)).
		Msg("Collected sensor data")

	return result
}

// convertTemperatureDataToSensors converts parsed sensor data into the agent report
// sensor format. This is shared between local collection and cluster peer collection.
func convertTemperatureDataToSensors(tempData *sensors.TemperatureData) agentshost.Sensors {
	result := agentshost.Sensors{
		TemperatureCelsius: make(map[string]float64),
	}

	// Add CPU package temperature
	if tempData.CPUPackage > 0 {
		result.TemperatureCelsius["cpu_package"] = tempData.CPUPackage
	}

	// Add individual core temperatures
	for coreName, temp := range tempData.Cores {
		normalizedName := strings.ToLower(strings.ReplaceAll(coreName, " ", "_"))
		result.TemperatureCelsius["cpu_"+normalizedName] = temp
	}

	// Add NVMe temperatures
	for nvmeName, temp := range tempData.NVMe {
		result.TemperatureCelsius[nvmeName] = temp
	}

	// Add GPU temperatures
	for gpuName, temp := range tempData.GPU {
		result.TemperatureCelsius[gpuName] = temp
	}

	// Add fan speeds (RPM)
	if len(tempData.Fans) > 0 {
		result.FanRPM = make(map[string]float64)
		for fanName, rpm := range tempData.Fans {
			result.FanRPM[fanName] = rpm
		}
	}

	// Add other temperatures (DDR5, motherboard, etc.)
	if len(tempData.Other) > 0 {
		result.Additional = make(map[string]float64)
		for sensorName, temp := range tempData.Other {
			result.Additional[sensorName] = temp
		}
	}

	return result
}

// collectFreeBSDTemperatures reads CPU and ACPI thermal zone temperatures
// from sysctl on FreeBSD. Returns an empty Sensors struct on any error.
func (a *Agent) collectFreeBSDTemperatures(ctx context.Context) agentshost.Sensors {
	cmdCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
	defer cancel()

	cmd := exec.CommandContext(cmdCtx, "sysctl", "-a")
	output, err := cmd.Output()
	if err != nil {
		a.logger.Debug().Err(err).Msg("Failed to collect FreeBSD temperature data via sysctl")
		return agentshost.Sensors{}
	}

	result := parseFreeBSDSysctlTemperatures(string(output))

	if len(result.TemperatureCelsius) > 0 {
		a.logger.Debug().
			Int("temperatureCount", len(result.TemperatureCelsius)).
			Msg("Collected FreeBSD temperature data via sysctl")
	}

	return result
}

// parseFreeBSDSysctlTemperatures parses sysctl output for temperature readings.
// FreeBSD exposes CPU temps via dev.cpu.N.temperature and ACPI thermal zones via
// hw.acpi.thermal.tzN.temperature. Values are formatted as "45.0C".
func parseFreeBSDSysctlTemperatures(sysctlOutput string) agentshost.Sensors {
	result := agentshost.Sensors{
		TemperatureCelsius: make(map[string]float64),
	}

	for _, line := range strings.Split(sysctlOutput, "\n") {
		if !strings.Contains(line, ".temperature:") {
			continue
		}
		parts := strings.SplitN(line, ":", 2)
		if len(parts) != 2 {
			continue
		}
		key := strings.TrimSpace(parts[0])
		valStr := strings.TrimSpace(parts[1])
		valStr = strings.TrimSuffix(valStr, "C")

		temp, err := strconv.ParseFloat(valStr, 64)
		if err != nil || temp <= 0 {
			continue
		}

		keyParts := strings.Split(key, ".")
		switch {
		case strings.HasPrefix(key, "dev.cpu.") && len(keyParts) >= 3:
			result.TemperatureCelsius["cpu_core_"+keyParts[2]] = temp
		case strings.HasPrefix(key, "hw.acpi.thermal.") && len(keyParts) >= 4:
			result.TemperatureCelsius["acpi_"+keyParts[3]] = temp
		default:
			normalized := strings.ReplaceAll(key, ".", "_")
			normalized = strings.TrimSuffix(normalized, "_temperature")
			result.TemperatureCelsius[normalized] = temp
		}
	}

	return result
}

// collectRAIDArrays attempts to collect mdadm RAID array information.
// Returns an empty slice if collection fails (best-effort).
func (a *Agent) collectRAIDArrays(ctx context.Context) []agentshost.RAIDArray {
	// Only collect on Linux (mdadm is Linux-specific)
	if a.collector.GOOS() != "linux" {
		return nil
	}

	arrays, err := a.collector.RAIDArrays(ctx)
	if err != nil {
		a.logger.Debug().Err(err).Msg("Failed to collect RAID array data (mdadm may not be installed)")
		return nil
	}

	if len(arrays) > 0 {
		a.logger.Debug().
			Int("arrayCount", len(arrays)).
			Msg("Collected RAID array data")
	}

	return arrays
}

// collectUnraidStorage attempts to collect Unraid array topology.
// Returns nil when not running on Unraid or if collection fails.
func (a *Agent) collectUnraidStorage(ctx context.Context) *agentshost.UnraidStorage {
	if a.collector.GOOS() != "linux" {
		return nil
	}

	storage, err := a.collector.UnraidStorage(ctx)
	if err != nil {
		a.logger.Debug().Err(err).Msg("Failed to collect Unraid storage data")
		return nil
	}
	if storage == nil {
		return nil
	}

	a.logger.Debug().
		Bool("arrayStarted", storage.ArrayStarted).
		Int("diskCount", len(storage.Disks)).
		Msg("Collected Unraid storage data")

	return storage
}

// collectCephStatus attempts to collect Ceph cluster status.
// Returns nil if Ceph is not available or not configured on this host.
func (a *Agent) collectCephStatus(ctx context.Context) *agentshost.CephCluster {
	if a.cfg.DisableCeph {
		return nil
	}
	// Only collect on Linux
	if a.collector.GOOS() != "linux" {
		return nil
	}

	status, err := a.collector.CephStatus(ctx)
	if err != nil {
		a.logger.Debug().Err(err).Msg("Failed to collect Ceph status")
		return nil
	}
	if status == nil {
		return nil
	}

	// Convert internal ceph types to agent report types
	result := &agentshost.CephCluster{
		FSID: status.FSID,
		Health: agentshost.CephHealth{
			Status: string(status.Health.Status),
			Checks: make(map[string]agentshost.CephCheck),
		},
		MonMap: agentshost.CephMonitorMap{
			Epoch:   status.MonMap.Epoch,
			NumMons: status.MonMap.NumMons,
		},
		MgrMap: agentshost.CephManagerMap{
			Available: status.MgrMap.Available,
			NumMgrs:   status.MgrMap.NumMgrs,
			ActiveMgr: status.MgrMap.ActiveMgr,
			Standbys:  status.MgrMap.Standbys,
		},
		OSDMap: agentshost.CephOSDMap{
			Epoch:   status.OSDMap.Epoch,
			NumOSDs: status.OSDMap.NumOSDs,
			NumUp:   status.OSDMap.NumUp,
			NumIn:   status.OSDMap.NumIn,
			NumDown: status.OSDMap.NumDown,
			NumOut:  status.OSDMap.NumOut,
		},
		PGMap: agentshost.CephPGMap{
			NumPGs:           status.PGMap.NumPGs,
			BytesTotal:       status.PGMap.BytesTotal,
			BytesUsed:        status.PGMap.BytesUsed,
			BytesAvailable:   status.PGMap.BytesAvailable,
			DataBytes:        status.PGMap.DataBytes,
			UsagePercent:     status.PGMap.UsagePercent,
			DegradedRatio:    status.PGMap.DegradedRatio,
			MisplacedRatio:   status.PGMap.MisplacedRatio,
			ReadBytesPerSec:  status.PGMap.ReadBytesPerSec,
			WriteBytesPerSec: status.PGMap.WriteBytesPerSec,
			ReadOpsPerSec:    status.PGMap.ReadOpsPerSec,
			WriteOpsPerSec:   status.PGMap.WriteOpsPerSec,
		},
		CollectedAt: status.CollectedAt.Format(time.RFC3339),
	}

	// Convert monitors
	for _, mon := range status.MonMap.Monitors {
		result.MonMap.Monitors = append(result.MonMap.Monitors, agentshost.CephMonitor{
			Name:   mon.Name,
			Rank:   mon.Rank,
			Addr:   mon.Addr,
			Status: mon.Status,
		})
	}

	// Convert health checks
	for name, check := range status.Health.Checks {
		result.Health.Checks[name] = agentshost.CephCheck{
			Severity: string(check.Severity),
			Message:  check.Message,
			Detail:   check.Detail,
		}
	}

	// Convert health summary
	for _, s := range status.Health.Summary {
		result.Health.Summary = append(result.Health.Summary, agentshost.CephHealthSummary{
			Severity: string(s.Severity),
			Message:  s.Message,
		})
	}

	// Convert pools
	for _, pool := range status.Pools {
		result.Pools = append(result.Pools, agentshost.CephPool{
			ID:             pool.ID,
			Name:           pool.Name,
			BytesUsed:      pool.BytesUsed,
			BytesAvailable: pool.BytesAvailable,
			Objects:        pool.Objects,
			PercentUsed:    pool.PercentUsed,
		})
	}

	// Convert services
	for _, svc := range status.Services {
		result.Services = append(result.Services, agentshost.CephService{
			Type:    string(svc.Type),
			Running: svc.Running,
			Total:   svc.Total,
			Daemons: svc.Daemons,
		})
	}

	a.logger.Debug().
		Str("fsid", result.FSID).
		Str("health", result.Health.Status).
		Int("osds", result.OSDMap.NumOSDs).
		Int("pools", len(result.Pools)).
		Msg("Collected Ceph cluster status")

	return result
}

// collectSMARTData collects S.M.A.R.T. data from local disks.
// Returns nil if smartctl is not available or no disks are found.
func (a *Agent) collectSMARTData(ctx context.Context) []agentshost.DiskSMART {
	goos := a.collector.GOOS()
	if goos != "linux" && goos != "freebsd" {
		return nil
	}

	smartData, err := a.collector.SMARTLocal(ctx, a.cfg.DiskExclude)
	if err != nil {
		a.logger.Debug().Err(err).Msg("Failed to collect S.M.A.R.T. data (smartctl may not be installed)")
		return nil
	}

	if len(smartData) == 0 {
		return nil
	}

	// Convert internal smartctl types to agent report types
	result := make([]agentshost.DiskSMART, 0, len(smartData))
	for _, disk := range smartData {
		entry := agentshost.DiskSMART{
			Device:      disk.Device,
			Model:       disk.Model,
			Serial:      disk.Serial,
			WWN:         disk.WWN,
			Type:        disk.Type,
			Temperature: disk.Temperature,
			Health:      disk.Health,
			Standby:     disk.Standby,
		}
		if disk.Attributes != nil {
			entry.Attributes = &agentshost.SMARTAttributes{
				PowerOnHours:         disk.Attributes.PowerOnHours,
				PowerCycles:          disk.Attributes.PowerCycles,
				ReallocatedSectors:   disk.Attributes.ReallocatedSectors,
				PendingSectors:       disk.Attributes.PendingSectors,
				OfflineUncorrectable: disk.Attributes.OfflineUncorrectable,
				UDMACRCErrors:        disk.Attributes.UDMACRCErrors,
				PercentageUsed:       disk.Attributes.PercentageUsed,
				AvailableSpare:       disk.Attributes.AvailableSpare,
				MediaErrors:          disk.Attributes.MediaErrors,
				UnsafeShutdowns:      disk.Attributes.UnsafeShutdowns,
			}
		}
		result = append(result, entry)
	}

	a.logger.Debug().
		Int("diskCount", len(result)).
		Msg("Collected S.M.A.R.T. disk data")

	return result
}

// runProxmoxSetup performs one-time Proxmox API token setup and node registration.
// Supports hosts with multiple Proxmox products (e.g., PVE + PBS on same host).
func (a *Agent) runProxmoxSetup(ctx context.Context) {
	a.logger.Info().Msg("Proxmox mode enabled, checking setup...")

	setup := NewProxmoxSetup(
		a.logger,
		a.httpClient,
		a.collector,
		a.trimmedPulseURL,
		a.cfg.APIToken,
		a.cfg.ProxmoxType,
		a.hostname,
		a.reportIP,
		a.stateDir,
		a.cfg.InsecureSkipVerify,
	)

	// Use RunAll to detect and register all Proxmox products on this host
	results, err := setup.RunAll(ctx)
	if err != nil {
		a.logger.Error().Err(err).Msg("Proxmox setup failed")
		return
	}

	if len(results) == 0 {
		// All types already registered
		a.logger.Info().Msg("All detected Proxmox products already registered")
		return
	}

	// Log results for each registered type
	for _, result := range results {
		if result.Registered {
			a.logger.Info().
				Str("type", result.ProxmoxType).
				Str("host", result.NodeHost).
				Str("token_id", result.TokenID).
				Msg("Proxmox node registered successfully")
		} else {
			a.logger.Warn().
				Str("type", result.ProxmoxType).
				Str("host", result.NodeHost).
				Msg("Proxmox token created but registration failed (node may need manual configuration)")
		}
	}
}

// isLXCContainer detects if we're running inside an LXC container.
// LXC containers share the host's /sys/class/dmi/id/product_uuid, which causes
// gopsutil to return identical HostIDs for all LXC containers on the same host.
func isLXCContainer(c SystemCollector) bool {
	// Check systemd-detect-virt if available
	if data, err := c.ReadFile("/run/systemd/container"); err == nil {
		container := strings.TrimSpace(string(data))
		if strings.Contains(container, "lxc") {
			return true
		}
	}

	// Check /proc/1/environ for container=lxc
	if data, err := c.ReadFile("/proc/1/environ"); err == nil {
		if strings.Contains(string(data), "container=lxc") {
			return true
		}
	}

	// Check /proc/1/cgroup for lxc markers
	if data, err := c.ReadFile("/proc/1/cgroup"); err == nil {
		text := string(data)
		if strings.Contains(text, "/lxc/") || strings.Contains(text, "lxc.payload") {
			return true
		}
	}

	return false
}

// getReliableMachineID returns a machine ID that's unique per container/host.
// On Linux, /etc/machine-id is always preferred over gopsutil's HostID because:
// - LXC containers share the host's /sys/class/dmi/id/product_uuid
// - Cloned VMs/hosts may share the same DMI product UUID
// - Proxmox cluster nodes with identical hardware may have the same UUID
// The /etc/machine-id file is guaranteed unique per installation.
// GetReliableMachineID attempts to find a stable machine ID.
func GetReliableMachineID(c SystemCollector, gopsutilHostID string, logger zerolog.Logger) string {
	gopsutilID := strings.TrimSpace(gopsutilHostID)

	// On Linux, prefer /etc/machine-id when available.
	// This avoids ID collisions from:
	// - LXC containers sharing host's DMI product UUID
	// - Cloned VMs with identical hardware UUIDs
	// - Proxmox cluster nodes with same hardware configuration
	if c.GOOS() == "linux" {
		if data, err := c.ReadFile("/etc/machine-id"); err == nil {
			machineID := strings.TrimSpace(string(data))
			if machineID != "" && len(machineID) >= 8 {
				// Format as UUID if it's a 32-char hex string (like machine-id typically is).
				if len(machineID) == 32 && utils.IsHexString(machineID) {
					machineID = fmt.Sprintf("%s-%s-%s-%s-%s",
						machineID[0:8], machineID[8:12], machineID[12:16],
						machineID[16:20], machineID[20:32])
				}
				if isLXCContainer(c) {
					logger.Debug().
						Str("machineID", machineID).
						Msg("LXC container detected, using /etc/machine-id for unique identification")
				} else {
					logger.Debug().
						Str("machineID", machineID).
						Msg("Linux host detected, using /etc/machine-id for unique identification")
				}
				return machineID
			}
		}

		if macID := getPrimaryMACIdentifier(c); macID != "" {
			logger.Debug().
				Str("machineID", macID).
				Msg("Linux host missing usable /etc/machine-id, using MAC address for unique identification")
			return macID
		}
	}

	return gopsutilID
}

func getPrimaryMACIdentifier(c SystemCollector) string {
	interfaces, err := c.NetInterfaces()
	if err != nil {
		return ""
	}

	sort.Slice(interfaces, func(i, j int) bool {
		return interfaces[i].Name < interfaces[j].Name
	})

	// Prefer a stable-looking interface name first to avoid selecting docker bridges
	// or other virtual interfaces when physical interfaces are present.
	for pass := 0; pass < 2; pass++ {
		for _, iface := range interfaces {
			if len(iface.HardwareAddr) == 0 {
				continue
			}
			if iface.Flags&net.FlagLoopback != 0 {
				continue
			}
			if pass == 0 && isLikelyVirtualInterfaceName(iface.Name) {
				continue
			}

			mac := strings.ToLower(iface.HardwareAddr.String())
			normalized := strings.NewReplacer(":", "", "-", "", ".", "").Replace(mac)
			if normalized == "" {
				continue
			}
			return "mac-" + normalized
		}
	}

	return ""
}

func isLikelyVirtualInterfaceName(name string) bool {
	name = strings.ToLower(strings.TrimSpace(name))
	switch {
	case name == "":
		return true
	case name == "lo":
		return true
	case strings.HasPrefix(name, "docker"):
		return true
	case strings.HasPrefix(name, "veth"):
		return true
	case strings.HasPrefix(name, "br-"):
		return true
	case strings.HasPrefix(name, "cni"):
		return true
	case strings.HasPrefix(name, "flannel"):
		return true
	case strings.HasPrefix(name, "virbr"):
		return true
	case strings.HasPrefix(name, "zt"):
		return true
	default:
		return false
	}
}