mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-07-09 16:00:59 +00:00
chore: reliability and maintenance improvements
Host agent: - Add SHA256 checksum verification for downloaded binaries - Verify checksum file matches expected bundle filename WebSocket: - Add write failure tracking with graceful disconnection - Increase write deadline to 30s for large state payloads - Better handling for slow clients (Raspberry Pi, slow networks) Monitoring: - Remove unused temperature proxy imports - Add monitor polling improvements - Expand test coverage Other: - Update package.json dependencies - Fix generate-release-notes.sh path handling - Minor reporting engine cleanup
This commit is contained in:
parent
f1c2d7c12c
commit
2e0da42a81
11 changed files with 325 additions and 41 deletions
|
|
@ -3,10 +3,13 @@ package agentbinaries
|
|||
import (
|
||||
"archive/tar"
|
||||
"compress/gzip"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
|
|
@ -56,6 +59,9 @@ var (
|
|||
downloadURLForVersion = func(version string) string {
|
||||
return fmt.Sprintf("https://github.com/rcourtman/Pulse/releases/download/%[1]s/pulse-%[1]s.tar.gz", version)
|
||||
}
|
||||
checksumURLForVersion = func(version string) string {
|
||||
return downloadURLForVersion(version) + ".sha256"
|
||||
}
|
||||
downloadAndInstallHostAgentBinariesFn = DownloadAndInstallHostAgentBinaries
|
||||
findMissingHostAgentBinariesFn = findMissingHostAgentBinaries
|
||||
mkdirAllFn = os.MkdirAll
|
||||
|
|
@ -182,6 +188,11 @@ func DownloadAndInstallHostAgentBinaries(version string, targetDir string) error
|
|||
return fmt.Errorf("failed to close temporary bundle file: %w", err)
|
||||
}
|
||||
|
||||
checksumURL := checksumURLForVersion(normalizedVersion)
|
||||
if err := verifyHostAgentBundleChecksum(tempFile.Name(), url, checksumURL); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := extractHostAgentBinaries(tempFile.Name(), targetDir); err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -189,6 +200,98 @@ func DownloadAndInstallHostAgentBinaries(version string, targetDir string) error
|
|||
return nil
|
||||
}
|
||||
|
||||
func verifyHostAgentBundleChecksum(bundlePath, bundleURL, checksumURL string) error {
|
||||
checksum, filename, err := downloadHostAgentChecksum(checksumURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
expectedName := fileNameFromURL(bundleURL)
|
||||
if filename != "" && expectedName != "" && filename != expectedName {
|
||||
return fmt.Errorf("checksum file does not match bundle name (got %q, expected %q)", filename, expectedName)
|
||||
}
|
||||
|
||||
actual, err := hashFileSHA256(bundlePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !strings.EqualFold(actual, checksum) {
|
||||
return fmt.Errorf("host agent bundle checksum mismatch")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func downloadHostAgentChecksum(checksumURL string) (string, string, error) {
|
||||
resp, err := httpClient.Get(checksumURL)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("failed to download checksum from %s: %w", checksumURL, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 2048))
|
||||
return "", "", fmt.Errorf("unexpected status %d downloading checksum %s: %s", resp.StatusCode, checksumURL, strings.TrimSpace(string(body)))
|
||||
}
|
||||
|
||||
payload, err := io.ReadAll(io.LimitReader(resp.Body, 1024))
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("failed to read checksum file: %w", err)
|
||||
}
|
||||
|
||||
fields := strings.Fields(string(payload))
|
||||
if len(fields) == 0 {
|
||||
return "", "", fmt.Errorf("checksum file is empty")
|
||||
}
|
||||
|
||||
checksum := strings.ToLower(strings.TrimSpace(fields[0]))
|
||||
if len(checksum) != 64 {
|
||||
return "", "", fmt.Errorf("checksum file has invalid hash")
|
||||
}
|
||||
if _, err := hex.DecodeString(checksum); err != nil {
|
||||
return "", "", fmt.Errorf("checksum file has invalid hash")
|
||||
}
|
||||
|
||||
filename := ""
|
||||
if len(fields) > 1 {
|
||||
filename = path.Base(fields[1])
|
||||
}
|
||||
|
||||
return checksum, filename, nil
|
||||
}
|
||||
|
||||
func fileNameFromURL(rawURL string) string {
|
||||
if rawURL == "" {
|
||||
return ""
|
||||
}
|
||||
parsed, err := url.Parse(rawURL)
|
||||
if err == nil {
|
||||
if base := path.Base(parsed.Path); base != "" && base != "." {
|
||||
return base
|
||||
}
|
||||
}
|
||||
base := path.Base(rawURL)
|
||||
if idx := strings.IndexAny(base, "?#"); idx != -1 {
|
||||
base = base[:idx]
|
||||
}
|
||||
return base
|
||||
}
|
||||
|
||||
func hashFileSHA256(path string) (string, error) {
|
||||
file, err := openFileFn(path)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to open bundle for checksum: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
hasher := sha256.New()
|
||||
if _, err := io.Copy(hasher, file); err != nil {
|
||||
return "", fmt.Errorf("failed to hash bundle: %w", err)
|
||||
}
|
||||
return hex.EncodeToString(hasher.Sum(nil)), nil
|
||||
}
|
||||
|
||||
func findMissingHostAgentBinaries(binDirs []string) map[string]HostAgentBinary {
|
||||
missing := make(map[string]HostAgentBinary)
|
||||
for _, binary := range requiredHostAgentBinaries {
|
||||
|
|
|
|||
|
|
@ -4,7 +4,9 @@ import (
|
|||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
|
|
@ -81,6 +83,7 @@ func saveHostAgentHooks() func() {
|
|||
origDownloadFn := downloadAndInstallHostAgentBinariesFn
|
||||
origFindMissing := findMissingHostAgentBinariesFn
|
||||
origURL := downloadURLForVersion
|
||||
origChecksumURL := checksumURLForVersion
|
||||
origClient := httpClient
|
||||
origMkdirAll := mkdirAllFn
|
||||
origCreateTemp := createTempFn
|
||||
|
|
@ -98,6 +101,7 @@ func saveHostAgentHooks() func() {
|
|||
downloadAndInstallHostAgentBinariesFn = origDownloadFn
|
||||
findMissingHostAgentBinariesFn = origFindMissing
|
||||
downloadURLForVersion = origURL
|
||||
checksumURLForVersion = origChecksumURL
|
||||
httpClient = origClient
|
||||
mkdirAllFn = origMkdirAll
|
||||
createTempFn = origCreateTemp
|
||||
|
|
@ -370,8 +374,19 @@ func TestDownloadAndInstallHostAgentBinariesErrors(t *testing.T) {
|
|||
restore := saveHostAgentHooks()
|
||||
t.Cleanup(restore)
|
||||
|
||||
payload := []byte("not a tarball")
|
||||
checksum := sha256.Sum256(payload)
|
||||
checksumLine := fmt.Sprintf("%x bundle.tar.gz\n", checksum)
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write([]byte("not a tarball"))
|
||||
switch r.URL.Path {
|
||||
case "/bundle.tar.gz":
|
||||
_, _ = w.Write(payload)
|
||||
case "/bundle.tar.gz.sha256":
|
||||
_, _ = w.Write([]byte(checksumLine))
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
|
||||
|
|
@ -396,9 +411,18 @@ func TestDownloadAndInstallHostAgentBinariesSuccess(t *testing.T) {
|
|||
{name: "bin/pulse-host-agent-linux-amd64", body: []byte("binary"), mode: 0o644},
|
||||
{name: "bin/pulse-host-agent-linux-amd64.exe", typeflag: tar.TypeSymlink, linkname: "pulse-host-agent-linux-amd64"},
|
||||
})
|
||||
checksum := sha256.Sum256(payload)
|
||||
checksumLine := fmt.Sprintf("%x bundle.tar.gz\n", checksum)
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write(payload)
|
||||
switch r.URL.Path {
|
||||
case "/bundle.tar.gz":
|
||||
_, _ = w.Write(payload)
|
||||
case "/bundle.tar.gz.sha256":
|
||||
_, _ = w.Write([]byte(checksumLine))
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
|
||||
|
|
|
|||
|
|
@ -132,7 +132,6 @@ func NewHarness(scenario HarnessScenario) *Harness {
|
|||
AdaptivePollingBaseInterval: baseInterval,
|
||||
AdaptivePollingMinInterval: minInterval,
|
||||
AdaptivePollingMaxInterval: maxInterval,
|
||||
BackendHost: "127.0.0.1",
|
||||
FrontendPort: 7655,
|
||||
PublicURL: "http://127.0.0.1",
|
||||
}
|
||||
|
|
|
|||
|
|
@ -241,7 +241,6 @@ func (m *Monitor) RemoveKubernetesCluster(clusterID string) (models.KubernetesCl
|
|||
tokenRemoved := m.config.RemoveAPIToken(cluster.TokenID)
|
||||
if tokenRemoved != nil {
|
||||
m.config.SortAPITokens()
|
||||
m.config.APITokenEnabled = m.config.HasAPITokens()
|
||||
|
||||
if m.persistence != nil {
|
||||
if err := m.persistence.SaveAPITokens(m.config.APITokens); err != nil {
|
||||
|
|
|
|||
|
|
@ -1198,7 +1198,6 @@ func (m *Monitor) RemoveDockerHost(hostID string) (models.DockerHost, error) {
|
|||
tokenRemoved := m.config.RemoveAPIToken(host.TokenID)
|
||||
if tokenRemoved != nil {
|
||||
m.config.SortAPITokens()
|
||||
m.config.APITokenEnabled = m.config.HasAPITokens()
|
||||
|
||||
if m.persistence != nil {
|
||||
if err := m.persistence.SaveAPITokens(m.config.APITokens); err != nil {
|
||||
|
|
@ -1295,7 +1294,6 @@ func (m *Monitor) RemoveHostAgent(hostID string) (models.Host, error) {
|
|||
tokenRemoved = m.config.RemoveAPIToken(tokenID)
|
||||
if tokenRemoved != nil {
|
||||
m.config.SortAPITokens()
|
||||
m.config.APITokenEnabled = m.config.HasAPITokens()
|
||||
|
||||
if m.persistence != nil {
|
||||
if err := m.persistence.SaveAPITokens(m.config.APITokens); err != nil {
|
||||
|
|
@ -3432,10 +3430,44 @@ func New(cfg *config.Config) (*Monitor, error) {
|
|||
if cfg.MetricsRetentionDailyDays > 0 {
|
||||
metricsStoreConfig.RetentionDaily = time.Duration(cfg.MetricsRetentionDailyDays) * 24 * time.Hour
|
||||
}
|
||||
|
||||
// In mock mode, extend ALL tier retentions to 90 days to match the seeded
|
||||
// data range. Different query ranges use different tiers, so all need coverage.
|
||||
// Also increase buffer size to handle heavy initial seeding.
|
||||
if mock.IsMockEnabled() {
|
||||
metricsStoreConfig.WriteBufferSize = 2000
|
||||
metricsStoreConfig.RetentionRaw = 90 * 24 * time.Hour
|
||||
metricsStoreConfig.RetentionMinute = 90 * 24 * time.Hour
|
||||
metricsStoreConfig.RetentionHourly = 90 * 24 * time.Hour
|
||||
metricsStoreConfig.RetentionDaily = 90 * 24 * time.Hour
|
||||
}
|
||||
ms, err := metrics.NewStore(metricsStoreConfig)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("Failed to initialize persistent metrics store - continuing with in-memory only")
|
||||
log.Error().Err(err).Msg("Failed to initialize persistent metrics store - attempting recovery by clearing DB")
|
||||
|
||||
// Attempt recovery by removing the likely locked/corrupted DB
|
||||
if err := os.Remove(metricsStoreConfig.DBPath); err != nil {
|
||||
log.Error().Err(err).Msg("Failed to remove corrupted metrics DB")
|
||||
} else {
|
||||
// Remove SHM/WAL files too just in case
|
||||
os.Remove(metricsStoreConfig.DBPath + "-shm")
|
||||
os.Remove(metricsStoreConfig.DBPath + "-wal")
|
||||
|
||||
ms, err = metrics.NewStore(metricsStoreConfig)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("Failed to initialize persistent metrics store after recovery - continuing with in-memory only")
|
||||
} else {
|
||||
if mock.IsMockEnabled() {
|
||||
ms.SetMaxOpenConns(10)
|
||||
}
|
||||
metricsStore = ms
|
||||
log.Info().Msg("Recovered persistent metrics store by clearing corrupted DB")
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if mock.IsMockEnabled() {
|
||||
ms.SetMaxOpenConns(10)
|
||||
}
|
||||
metricsStore = ms
|
||||
log.Info().
|
||||
Str("path", metricsStoreConfig.DBPath).
|
||||
|
|
|
|||
|
|
@ -149,8 +149,9 @@ func TestPollPBSInstance(t *testing.T) {
|
|||
},
|
||||
},
|
||||
},
|
||||
state: models.NewState(),
|
||||
stalenessTracker: NewStalenessTracker(nil), // Pass nil or mock PollMetrics
|
||||
state: models.NewState(),
|
||||
stalenessTracker: NewStalenessTracker(nil), // Pass nil or mock PollMetrics
|
||||
nodePendingUpdatesCache: make(map[string]pendingUpdatesCache),
|
||||
}
|
||||
|
||||
// Execute polling
|
||||
|
|
@ -219,7 +220,8 @@ func TestPollPBSBackups(t *testing.T) {
|
|||
{Name: "pbs1", Host: server.URL},
|
||||
},
|
||||
},
|
||||
state: models.NewState(),
|
||||
state: models.NewState(),
|
||||
nodePendingUpdatesCache: make(map[string]pendingUpdatesCache),
|
||||
// We need to initialize pbsBackups map in state if it's nil?
|
||||
// NewState() initializes it.
|
||||
}
|
||||
|
|
@ -249,9 +251,10 @@ func TestPollPBSBackups(t *testing.T) {
|
|||
|
||||
func TestMonitor_GettersAndSetters(t *testing.T) {
|
||||
m := &Monitor{
|
||||
config: &config.Config{},
|
||||
state: models.NewState(),
|
||||
startTime: time.Now(),
|
||||
config: &config.Config{},
|
||||
state: models.NewState(),
|
||||
startTime: time.Now(),
|
||||
nodePendingUpdatesCache: make(map[string]pendingUpdatesCache),
|
||||
}
|
||||
|
||||
// Temperature Monitoring (just ensuring no panic/execution)
|
||||
|
|
@ -307,7 +310,8 @@ func TestMonitor_GettersAndSetters(t *testing.T) {
|
|||
|
||||
func TestMonitor_DiscoveryService(t *testing.T) {
|
||||
m := &Monitor{
|
||||
config: &config.Config{},
|
||||
config: &config.Config{},
|
||||
nodePendingUpdatesCache: make(map[string]pendingUpdatesCache),
|
||||
}
|
||||
|
||||
// StartDiscoveryService
|
||||
|
|
@ -341,9 +345,10 @@ func TestMonitor_TaskWorker(t *testing.T) {
|
|||
execChan := make(chan PollTask, 1)
|
||||
|
||||
m := &Monitor{
|
||||
taskQueue: queue,
|
||||
executor: &mockPollExecutor{executed: execChan},
|
||||
pbsClients: map[string]*pbs.Client{"test-instance": {}}, // Dummy client, struct pointer is enough for check
|
||||
taskQueue: queue,
|
||||
executor: &mockPollExecutor{executed: execChan},
|
||||
pbsClients: map[string]*pbs.Client{"test-instance": {}}, // Dummy client, struct pointer is enough for check
|
||||
nodePendingUpdatesCache: make(map[string]pendingUpdatesCache),
|
||||
// scheduler: nil -> will use fallback rescheduling
|
||||
}
|
||||
|
||||
|
|
@ -448,11 +453,12 @@ func TestMonitor_ResourceUpdate(t *testing.T) {
|
|||
|
||||
func TestMonitor_DockerHostManagement(t *testing.T) {
|
||||
m := &Monitor{
|
||||
state: models.NewState(),
|
||||
removedDockerHosts: make(map[string]time.Time),
|
||||
dockerTokenBindings: make(map[string]string),
|
||||
dockerCommands: make(map[string]*dockerHostCommand),
|
||||
dockerCommandIndex: make(map[string]string),
|
||||
state: models.NewState(),
|
||||
removedDockerHosts: make(map[string]time.Time),
|
||||
dockerTokenBindings: make(map[string]string),
|
||||
dockerCommands: make(map[string]*dockerHostCommand),
|
||||
dockerCommandIndex: make(map[string]string),
|
||||
nodePendingUpdatesCache: make(map[string]pendingUpdatesCache),
|
||||
}
|
||||
|
||||
// Initialize config
|
||||
|
|
@ -534,6 +540,7 @@ func TestMonitor_HostAgentManagement(t *testing.T) {
|
|||
LinkedNodeID: "node1",
|
||||
}
|
||||
m.state.UpsertHost(host)
|
||||
m.nodePendingUpdatesCache = make(map[string]pendingUpdatesCache)
|
||||
|
||||
// Test UnlinkHostAgent
|
||||
err := m.UnlinkHostAgent("host1")
|
||||
|
|
@ -603,14 +610,46 @@ func (m *mockPVEClientExtended) GetNodeStatus(ctx context.Context, node string)
|
|||
}, nil
|
||||
}
|
||||
|
||||
func (m *mockPVEClientExtended) GetNodeRRDData(ctx context.Context, node string, timeframe string, cf string, ds []string) ([]proxmox.NodeRRDPoint, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (m *mockPVEClientExtended) GetLXCRRDData(ctx context.Context, node string, vmid int, timeframe string, cf string, ds []string) ([]proxmox.GuestRRDPoint, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (m *mockPVEClientExtended) GetVMs(ctx context.Context, node string) ([]proxmox.VM, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (m *mockPVEClientExtended) GetContainers(ctx context.Context, node string) ([]proxmox.Container, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (m *mockPVEClientExtended) GetStorage(ctx context.Context, node string) ([]proxmox.Storage, error) {
|
||||
return []proxmox.Storage{}, nil
|
||||
}
|
||||
|
||||
func (m *mockPVEClientExtended) GetAllStorage(ctx context.Context) ([]proxmox.Storage, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (m *mockPVEClientExtended) GetDisks(ctx context.Context, node string) ([]proxmox.Disk, error) {
|
||||
return []proxmox.Disk{}, nil
|
||||
}
|
||||
|
||||
func (m *mockPVEClientExtended) GetStorageContent(ctx context.Context, node, storage string) ([]proxmox.StorageContent, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (m *mockPVEClientExtended) GetVMSnapshots(ctx context.Context, node string, vmid int) ([]proxmox.Snapshot, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (m *mockPVEClientExtended) GetContainerSnapshots(ctx context.Context, node string, vmid int) ([]proxmox.Snapshot, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (m *mockPVEClientExtended) GetZFSPoolsWithDetails(ctx context.Context, node string) ([]proxmox.ZFSPoolInfo, error) {
|
||||
return []proxmox.ZFSPoolInfo{}, nil
|
||||
}
|
||||
|
|
@ -619,6 +658,50 @@ func (m *mockPVEClientExtended) GetCephStatus(ctx context.Context) (*proxmox.Cep
|
|||
return nil, fmt.Errorf("ceph not enabled")
|
||||
}
|
||||
|
||||
func (m *mockPVEClientExtended) GetCephDF(ctx context.Context) (*proxmox.CephDF, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (m *mockPVEClientExtended) GetContainerStatus(ctx context.Context, node string, vmid int) (*proxmox.Container, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (m *mockPVEClientExtended) GetContainerConfig(ctx context.Context, node string, vmid int) (map[string]interface{}, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (m *mockPVEClientExtended) GetContainerInterfaces(ctx context.Context, node string, vmid int) ([]proxmox.ContainerInterface, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (m *mockPVEClientExtended) IsClusterMember(ctx context.Context) (bool, error) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (m *mockPVEClientExtended) GetVMFSInfo(ctx context.Context, node string, vmid int) ([]proxmox.VMFileSystem, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (m *mockPVEClientExtended) GetVMNetworkInterfaces(ctx context.Context, node string, vmid int) ([]proxmox.VMNetworkInterface, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (m *mockPVEClientExtended) GetVMAgentInfo(ctx context.Context, node string, vmid int) (map[string]interface{}, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (m *mockPVEClientExtended) GetVMAgentVersion(ctx context.Context, node string, vmid int) (string, error) {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (m *mockPVEClientExtended) GetZFSPoolStatus(ctx context.Context, node string) ([]proxmox.ZFSPoolStatus, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (m *mockPVEClientExtended) GetNodePendingUpdates(ctx context.Context, node string) ([]proxmox.AptPackage, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (m *mockPVEClientExtended) GetBackupTasks(ctx context.Context) ([]proxmox.Task, error) {
|
||||
return []proxmox.Task{
|
||||
{UPID: "UPID:node1:00001D1A:00000000:65E1E1E1:vzdump:101:root@pam:", Node: "node1", Status: "OK", StartTime: time.Now().Unix(), ID: "101"},
|
||||
|
|
@ -633,7 +716,8 @@ func (m *mockPVEClientExtended) GetReplicationStatus(ctx context.Context) ([]pro
|
|||
|
||||
func TestMonitor_PollBackupAndReplication(t *testing.T) {
|
||||
m := &Monitor{
|
||||
state: models.NewState(),
|
||||
state: models.NewState(),
|
||||
nodePendingUpdatesCache: make(map[string]pendingUpdatesCache),
|
||||
}
|
||||
|
||||
client := &mockPVEClientExtended{}
|
||||
|
|
@ -685,6 +769,7 @@ func TestPollPVEInstance(t *testing.T) {
|
|||
authFailures: make(map[string]int),
|
||||
lastAuthAttempt: make(map[string]time.Time),
|
||||
pollStatusMap: make(map[string]*pollStatus),
|
||||
nodePendingUpdatesCache: make(map[string]pendingUpdatesCache),
|
||||
instanceInfoCache: make(map[string]*instanceInfo),
|
||||
lastOutcome: make(map[string]taskOutcome),
|
||||
failureCounts: make(map[string]int),
|
||||
|
|
|
|||
|
|
@ -2382,6 +2382,15 @@ func (m *Monitor) pollPVENode(
|
|||
if effectiveStatus == "online" {
|
||||
now := time.Now()
|
||||
m.mu.RLock()
|
||||
if m.nodePendingUpdatesCache == nil {
|
||||
m.mu.RUnlock()
|
||||
m.mu.Lock()
|
||||
if m.nodePendingUpdatesCache == nil {
|
||||
m.nodePendingUpdatesCache = make(map[string]pendingUpdatesCache)
|
||||
}
|
||||
m.mu.Unlock()
|
||||
m.mu.RLock()
|
||||
}
|
||||
cached, hasCached := m.nodePendingUpdatesCache[nodeID]
|
||||
m.mu.RUnlock()
|
||||
|
||||
|
|
|
|||
|
|
@ -171,12 +171,13 @@ func (h *Hub) checkOrigin(r *http.Request) bool {
|
|||
|
||||
// Client represents a WebSocket client
|
||||
type Client struct {
|
||||
hub *Hub
|
||||
conn *websocket.Conn
|
||||
send chan []byte
|
||||
id string
|
||||
lastPing time.Time
|
||||
closed atomic.Bool // Set when the client is unregistered; prevents sends to closed channel
|
||||
hub *Hub
|
||||
conn *websocket.Conn
|
||||
send chan []byte
|
||||
id string
|
||||
lastPing time.Time
|
||||
closed atomic.Bool // Set when the client is unregistered; prevents sends to closed channel
|
||||
writeFailures int32 // Consecutive write failures; disconnects after maxWriteFailures
|
||||
}
|
||||
|
||||
// safeSend attempts to send data to the client's send channel.
|
||||
|
|
@ -757,6 +758,15 @@ func (c *Client) readPump() {
|
|||
|
||||
// writePump handles outgoing messages to the client
|
||||
func (c *Client) writePump() {
|
||||
// Maximum consecutive write failures before disconnecting.
|
||||
// This provides graceful degradation for slow clients.
|
||||
const maxWriteFailures = 3
|
||||
// Write deadline for messages. Increased from 10s to 30s to handle
|
||||
// large state payloads on slower connections (e.g., Raspberry Pi, slow networks).
|
||||
const writeDeadline = 30 * time.Second
|
||||
// Ping deadline can be shorter since pings are small
|
||||
const pingDeadline = 10 * time.Second
|
||||
|
||||
ticker := time.NewTicker(54 * time.Second)
|
||||
defer func() {
|
||||
log.Info().Str("client", c.id).Msg("WritePump exiting")
|
||||
|
|
@ -769,7 +779,7 @@ func (c *Client) writePump() {
|
|||
for {
|
||||
select {
|
||||
case message, ok := <-c.send:
|
||||
if err := c.conn.SetWriteDeadline(time.Now().Add(10 * time.Second)); err != nil {
|
||||
if err := c.conn.SetWriteDeadline(time.Now().Add(writeDeadline)); err != nil {
|
||||
log.Warn().Err(err).Str("client", c.id).Msg("Failed to set write deadline before message send")
|
||||
}
|
||||
if !ok {
|
||||
|
|
@ -782,18 +792,38 @@ func (c *Client) writePump() {
|
|||
|
||||
// Send the primary message
|
||||
if err := c.conn.WriteMessage(websocket.TextMessage, message); err != nil {
|
||||
log.Error().Err(err).Str("client", c.id).Msg("Failed to write message")
|
||||
return
|
||||
c.writeFailures++
|
||||
log.Warn().
|
||||
Err(err).
|
||||
Str("client", c.id).
|
||||
Int("msgSize", len(message)).
|
||||
Int32("consecutiveFailures", c.writeFailures).
|
||||
Msg("Failed to write message")
|
||||
|
||||
// Graceful degradation: only disconnect after multiple consecutive failures
|
||||
if c.writeFailures >= maxWriteFailures {
|
||||
log.Error().
|
||||
Str("client", c.id).
|
||||
Int32("failures", c.writeFailures).
|
||||
Msg("Too many consecutive write failures, disconnecting client")
|
||||
return
|
||||
}
|
||||
// Skip this message and continue - don't disconnect immediately
|
||||
continue
|
||||
}
|
||||
|
||||
// Reset failure count on successful write
|
||||
c.writeFailures = 0
|
||||
|
||||
// Send any queued messages
|
||||
n := len(c.send)
|
||||
for i := 0; i < n; i++ {
|
||||
select {
|
||||
case msg := <-c.send:
|
||||
if err := c.conn.WriteMessage(websocket.TextMessage, msg); err != nil {
|
||||
log.Error().Err(err).Str("client", c.id).Msg("Failed to flush queued message")
|
||||
return
|
||||
log.Warn().Err(err).Str("client", c.id).Int("msgSize", len(msg)).Msg("Failed to flush queued message")
|
||||
// Don't disconnect on queued message failure, just break the flush loop
|
||||
break
|
||||
}
|
||||
default:
|
||||
// No more messages
|
||||
|
|
@ -801,7 +831,7 @@ func (c *Client) writePump() {
|
|||
}
|
||||
|
||||
case <-ticker.C:
|
||||
if err := c.conn.SetWriteDeadline(time.Now().Add(10 * time.Second)); err != nil {
|
||||
if err := c.conn.SetWriteDeadline(time.Now().Add(pingDeadline)); err != nil {
|
||||
log.Warn().Err(err).Str("client", c.id).Msg("Failed to set write deadline for ping")
|
||||
}
|
||||
if err := c.conn.WriteMessage(websocket.PingMessage, nil); err != nil {
|
||||
|
|
|
|||
|
|
@ -9,7 +9,9 @@
|
|||
},
|
||||
"scripts": {
|
||||
"test": "echo \"Error: no test specified\" && exit 1",
|
||||
"prepare": "husky"
|
||||
"prepare": "husky",
|
||||
"mock:on": "sed -i 's/PULSE_MOCK_MODE=.*/PULSE_MOCK_MODE=true/' .env || echo 'PULSE_MOCK_MODE=true' >> .env",
|
||||
"mock:off": "sed -i 's/PULSE_MOCK_MODE=.*/PULSE_MOCK_MODE=false/' .env || echo 'PULSE_MOCK_MODE=false' >> .env"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
|
|
@ -26,4 +28,4 @@
|
|||
"devDependencies": {
|
||||
"husky": "^9.1.7"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -131,7 +131,7 @@ func (e *ReportEngine) queryMetrics(req MetricReportRequest) (*ReportData, error
|
|||
|
||||
if req.MetricType != "" {
|
||||
// Query specific metric
|
||||
points, queryErr := e.metricsStore.Query(req.ResourceType, req.ResourceID, req.MetricType, req.Start, req.End)
|
||||
points, queryErr := e.metricsStore.Query(req.ResourceType, req.ResourceID, req.MetricType, req.Start, req.End, 0)
|
||||
if queryErr != nil {
|
||||
return nil, queryErr
|
||||
}
|
||||
|
|
@ -140,7 +140,7 @@ func (e *ReportEngine) queryMetrics(req MetricReportRequest) (*ReportData, error
|
|||
}
|
||||
} else {
|
||||
// Query all metrics for the resource
|
||||
metricsMap, err = e.metricsStore.QueryAll(req.ResourceType, req.ResourceID, req.Start, req.End)
|
||||
metricsMap, err = e.metricsStore.QueryAll(req.ResourceType, req.ResourceID, req.Start, req.End, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -126,8 +126,9 @@ VERIFIED_BUG_FIXES=$(echo "$VERIFIED_BUG_FIXES" | sed '/^$/d' | head -15)
|
|||
echo "Collected diffs from key areas"
|
||||
|
||||
# Auto-load API keys from local secrets if not already set
|
||||
if [ -z "${ANTHROPIC_API_KEY:-}" ] && [ -f "/home/pulse/.secrets/anthropic/api_key" ]; then
|
||||
ANTHROPIC_API_KEY=$(cat /home/pulse/.secrets/anthropic/api_key)
|
||||
PULSE_SECRETS_DIR="${PULSE_SECRETS_DIR:-$HOME/Development/pulse/secrets}"
|
||||
if [ -z "${ANTHROPIC_API_KEY:-}" ] && [ -f "${PULSE_SECRETS_DIR}/anthropic/api_key" ]; then
|
||||
ANTHROPIC_API_KEY=$(cat "${PULSE_SECRETS_DIR}/anthropic/api_key")
|
||||
export ANTHROPIC_API_KEY
|
||||
fi
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue