diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index af1540198..782d80fb8 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -314,6 +314,8 @@ These env vars override system.json values. When set, the UI will show a warning - `CONNECTION_TIMEOUT` - API timeout in seconds (default: 10) - `ALLOWED_ORIGINS` - CORS origins (default: same-origin only) - `LOG_LEVEL` - Log verbosity: debug/info/warn/error (default: info) +- `ENABLE_BACKUP_POLLING` - Set to `false` to disable polling of Proxmox backup/snapshot APIs (default: true) +- `BACKUP_POLLING_INTERVAL` - Override the backup polling cadence. Accepts Go duration syntax (e.g. `30m`, `6h`) or seconds. Use `0` for Pulse's default (~90s) cadence. - `PULSE_PUBLIC_URL` - Full URL to access Pulse (e.g., `http://192.168.1.100:7655`) - **Auto-detected** if not set (except inside Docker where detection is disabled) - Used in webhook notifications for "View in Pulse" links diff --git a/frontend-modern/src/api/system.ts b/frontend-modern/src/api/system.ts index b015128fc..a5205b9e6 100644 --- a/frontend-modern/src/api/system.ts +++ b/frontend-modern/src/api/system.ts @@ -7,6 +7,8 @@ export interface SystemSettings { autoUpdateEnabled: boolean; autoUpdateCheckInterval?: number; autoUpdateTime?: string; + backupPollingInterval?: number; + backupPollingEnabled?: boolean; // apiToken removed - now handled via security API } diff --git a/frontend-modern/src/components/Settings/Settings.tsx b/frontend-modern/src/components/Settings/Settings.tsx index 4ee952ffc..c55b91dd0 100644 --- a/frontend-modern/src/components/Settings/Settings.tsx +++ b/frontend-modern/src/components/Settings/Settings.tsx @@ -257,6 +257,18 @@ const SETTINGS_HEADER_META: Record = (props) => { const [autoUpdateEnabled, setAutoUpdateEnabled] = createSignal(false); const [autoUpdateCheckInterval, setAutoUpdateCheckInterval] = createSignal(24); const [autoUpdateTime, setAutoUpdateTime] = createSignal('03:00'); + const [backupPollingEnabled, setBackupPollingEnabled] = createSignal(true); + const [backupPollingInterval, setBackupPollingInterval] = createSignal(0); + const [backupPollingCustomMinutes, setBackupPollingCustomMinutes] = createSignal(60); + const backupPollingEnvLocked = () => + Boolean(envOverrides()['ENABLE_BACKUP_POLLING'] || envOverrides()['BACKUP_POLLING_INTERVAL']); + const backupIntervalSelectValue = () => { + const seconds = backupPollingInterval(); + return BACKUP_INTERVAL_OPTIONS.some((option) => option.value === seconds) + ? String(seconds) + : 'custom'; + }; + const backupIntervalSummary = () => { + if (!backupPollingEnabled()) { + return 'Backup polling is disabled.'; + } + const seconds = backupPollingInterval(); + if (seconds <= 0) { + return 'Pulse checks backups and snapshots at the default cadence (~every 90 seconds).'; + } + if (seconds % 86400 === 0) { + const days = seconds / 86400; + return `Pulse checks backups every ${days === 1 ? 'day' : `${days} days`}.`; + } + if (seconds % 3600 === 0) { + const hours = seconds / 3600; + return `Pulse checks backups every ${hours === 1 ? 'hour' : `${hours} hours`}.`; + } + const minutes = Math.max(1, Math.round(seconds / 60)); + return `Pulse checks backups every ${minutes === 1 ? 'minute' : `${minutes} minutes`}.`; + }; // Diagnostics const [diagnosticsData, setDiagnosticsData] = createSignal(null); @@ -988,6 +1030,20 @@ const Settings: Component = (props) => { // Load embedding settings setAllowEmbedding(systemSettings.allowEmbedding ?? false); setAllowedEmbedOrigins(systemSettings.allowedEmbedOrigins || ''); + // Backup polling controls + if (typeof systemSettings.backupPollingEnabled === 'boolean') { + setBackupPollingEnabled(systemSettings.backupPollingEnabled); + } else { + setBackupPollingEnabled(true); + } + const intervalSeconds = + typeof systemSettings.backupPollingInterval === 'number' + ? Math.max(0, Math.floor(systemSettings.backupPollingInterval)) + : 0; + setBackupPollingInterval(intervalSeconds); + if (intervalSeconds > 0) { + setBackupPollingCustomMinutes(Math.max(1, Math.round(intervalSeconds / 60))); + } // Load auto-update settings setAutoUpdateEnabled(systemSettings.autoUpdateEnabled || false); setAutoUpdateCheckInterval(systemSettings.autoUpdateCheckInterval || 24); @@ -1077,6 +1133,8 @@ const Settings: Component = (props) => { autoUpdateEnabled: autoUpdateEnabled(), autoUpdateCheckInterval: autoUpdateCheckInterval(), autoUpdateTime: autoUpdateTime(), + backupPollingEnabled: backupPollingEnabled(), + backupPollingInterval: backupPollingInterval(), allowEmbedding: allowEmbedding(), allowedEmbedOrigins: allowedEmbedOrigins(), }); @@ -3418,12 +3476,160 @@ const Settings: Component = (props) => { border={false} class="border border-gray-200 dark:border-gray-700" > - +
+

+ + + + + Backup polling +

+

+ Control how often Pulse queries Proxmox backup tasks, datastore contents, and guest snapshots. + Longer intervals reduce disk activity and API load. +

+
+
+
+

+ Enable backup polling +

+

+ Required for dashboard backup status, storage snapshots, and alerting. +

+
+ +
+ + +
+
+
+ +

+ {backupIntervalSummary()} +

+
+ +
+ + +
+ +
+ { + const value = Number(e.currentTarget.value); + if (Number.isNaN(value)) { + return; + } + const clamped = Math.max( + 1, + Math.min(BACKUP_INTERVAL_MAX_MINUTES, Math.floor(value)), + ); + setBackupPollingCustomMinutes(clamped); + setBackupPollingInterval(clamped * 60); + if (!backupPollingEnvLocked()) { + setHasUnsavedChanges(true); + } + }} + class="w-24 px-3 py-1.5 text-sm border border-gray-300 dark:border-gray-600 rounded-lg bg-white dark:bg-gray-800 disabled:opacity-50" + /> + + 1 – {BACKUP_INTERVAL_MAX_MINUTES} minutes (≈7 days max) + +
+
+ +
+ + + +
+ + + +
+

Environment override detected

+

+ The ENABLE_BACKUP_POLLING or{' '} + BACKUP_POLLING_INTERVAL environment + variables are set. Remove them and restart Pulse to manage backup polling here. +

+
+
+ +
+
+ +
{/* Export Section */} diff --git a/frontend-modern/src/components/shared/NodeSummaryTable.tsx b/frontend-modern/src/components/shared/NodeSummaryTable.tsx index 63af253bb..f45d8a7f7 100644 --- a/frontend-modern/src/components/shared/NodeSummaryTable.tsx +++ b/frontend-modern/src/components/shared/NodeSummaryTable.tsx @@ -621,11 +621,17 @@ export const NodeSummaryTable: Component = (props) => { : 'text-green-600 dark:text-green-400'; const temp = node!.temperature; - const hasMinMax = temp && temp.cpuMin && temp.cpuMin > 0 && temp.cpuMaxRecord && temp.cpuMaxRecord > 0; + const cpuMin = temp?.cpuMin; + const cpuMax = temp?.cpuMaxRecord; + const hasMinMax = + typeof cpuMin === 'number' && + cpuMin > 0 && + typeof cpuMax === 'number' && + cpuMax > 0; if (hasMinMax) { - const min = Math.round(temp.cpuMin); - const max = Math.round(temp.cpuMaxRecord); + const min = Math.round(cpuMin); + const max = Math.round(cpuMax); const getTooltipColor = (temp: number) => { if (temp >= 80) return 'text-red-400'; diff --git a/frontend-modern/src/types/config.ts b/frontend-modern/src/types/config.ts index dd5080a49..364c8a4ed 100644 --- a/frontend-modern/src/types/config.ts +++ b/frontend-modern/src/types/config.ts @@ -34,6 +34,8 @@ export interface SystemConfig { updateChannel?: string; // Update channel: 'stable' | 'rc' | 'beta' autoUpdateCheckInterval?: number; // Hours between update checks autoUpdateTime?: string; // Time for updates (HH:MM format) + backupPollingInterval?: number; // Backup polling interval in seconds (0 = default cadence) + backupPollingEnabled?: boolean; // Enable backup polling of PVE/PBS data allowedOrigins?: string; // CORS allowed origins backendPort?: number; // Backend API port (default: 7655) frontendPort?: number; // Frontend UI port (default: 7655) @@ -147,6 +149,8 @@ export const DEFAULT_CONFIG: { updateChannel: 'stable', autoUpdateCheckInterval: 24, autoUpdateTime: '03:00', + backupPollingEnabled: true, + backupPollingInterval: 0, allowedOrigins: '', backendPort: 7655, frontendPort: 7655, diff --git a/frontend-modern/src/types/settings.ts b/frontend-modern/src/types/settings.ts index feb3fc9ec..e6fca5b71 100644 --- a/frontend-modern/src/types/settings.ts +++ b/frontend-modern/src/types/settings.ts @@ -15,6 +15,8 @@ export interface MonitoringSettings { // Note: PVE polling is hardcoded to 10s server-side concurrentPolling: boolean; backupPollingCycles: number; + backupPollingIntervalMs: number; + backupPollingEnabled: boolean; metricsRetentionDays: number; } diff --git a/internal/api/config_handlers.go b/internal/api/config_handlers.go index 8e8e01745..9fe1ad3ad 100644 --- a/internal/api/config_handlers.go +++ b/internal/api/config_handlers.go @@ -2523,6 +2523,7 @@ func (h *ConfigHandlers) HandleGetSystemSettings(w http.ResponseWriter, r *http. settings := config.SystemSettings{ // Note: PVE polling is hardcoded to 10s PBSPollingInterval: int(h.config.PBSPollingInterval.Seconds()), + BackupPollingInterval: int(h.config.BackupPollingInterval.Seconds()), BackendPort: h.config.BackendPort, FrontendPort: h.config.FrontendPort, AllowedOrigins: h.config.AllowedOrigins, @@ -2538,6 +2539,8 @@ func (h *ConfigHandlers) HandleGetSystemSettings(w http.ResponseWriter, r *http. DiscoveryEnabled: persistedSettings.DiscoveryEnabled, // Include discoveryEnabled from persisted settings DiscoverySubnet: persistedSettings.DiscoverySubnet, // Include discoverySubnet from persisted settings } + backupEnabled := h.config.EnableBackupPolling + settings.BackupPollingEnabled = &backupEnabled w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(settings) @@ -2670,6 +2673,10 @@ func (h *ConfigHandlers) HandleUpdateSystemSettingsOLD(w http.ResponseWriter, r http.Error(w, "PBS polling interval must be positive", http.StatusBadRequest) return } + if settings.BackupPollingInterval < 0 { + http.Error(w, "Backup polling interval cannot be negative", http.StatusBadRequest) + return + } // Update polling intervals needsReload := false @@ -2679,6 +2686,14 @@ func (h *ConfigHandlers) HandleUpdateSystemSettingsOLD(w http.ResponseWriter, r h.config.PBSPollingInterval = time.Duration(settings.PBSPollingInterval) * time.Second needsReload = true } + if settings.BackupPollingInterval > 0 || (settings.BackupPollingInterval == 0 && h.config.BackupPollingInterval != 0) { + h.config.BackupPollingInterval = time.Duration(settings.BackupPollingInterval) * time.Second + needsReload = true + } + if settings.BackupPollingEnabled != nil { + h.config.EnableBackupPolling = *settings.BackupPollingEnabled + needsReload = true + } // Trigger a monitor reload if intervals changed if needsReload && h.reloadFunc != nil { diff --git a/internal/api/system_settings.go b/internal/api/system_settings.go index e913bc2d4..75076a5c4 100644 --- a/internal/api/system_settings.go +++ b/internal/api/system_settings.go @@ -88,6 +88,22 @@ func validateSystemSettings(settings *config.SystemSettings, rawRequest map[stri } } + if val, ok := rawRequest["backupPollingInterval"]; ok { + if interval, ok := val.(float64); ok { + if interval < 0 { + return fmt.Errorf("Backup polling interval cannot be negative") + } + if interval > 0 && interval < 10 { + return fmt.Errorf("Backup polling interval must be at least 10 seconds") + } + if interval > 604800 { + return fmt.Errorf("Backup polling interval cannot exceed 604800 seconds (7 days)") + } + } else { + return fmt.Errorf("Backup polling interval must be a number") + } + } + // Validate boolean fields have correct type if val, ok := rawRequest["autoUpdateEnabled"]; ok { if _, ok := val.(bool); !ok { @@ -107,6 +123,12 @@ func validateSystemSettings(settings *config.SystemSettings, rawRequest map[stri } } + if val, ok := rawRequest["backupPollingEnabled"]; ok { + if _, ok := val.(bool); !ok { + return fmt.Errorf("backupPollingEnabled must be a boolean") + } + } + // Validate auto-update check interval (min 1 hour, max 7 days) if val, ok := rawRequest["autoUpdateCheckInterval"]; ok { if interval, ok := val.(float64); ok { @@ -184,6 +206,11 @@ func (h *SystemSettingsHandler) HandleGetSystemSettings(w http.ResponseWriter, r log.Debug(). Str("theme", settings.Theme). Msg("Loaded system settings for API response") + + // Always expose effective backup polling configuration + settings.BackupPollingInterval = int(h.config.BackupPollingInterval.Seconds()) + enabled := h.config.EnableBackupPolling + settings.BackupPollingEnabled = &enabled } // Include env override information @@ -279,6 +306,9 @@ func (h *SystemSettingsHandler) HandleUpdateSystemSettings(w http.ResponseWriter if _, ok := rawRequest["pmgPollingInterval"]; ok { settings.PMGPollingInterval = updates.PMGPollingInterval } + if _, ok := rawRequest["backupPollingInterval"]; ok { + settings.BackupPollingInterval = updates.BackupPollingInterval + } if updates.AllowedOrigins != "" { settings.AllowedOrigins = updates.AllowedOrigins } @@ -315,6 +345,9 @@ func (h *SystemSettingsHandler) HandleUpdateSystemSettings(w http.ResponseWriter if _, ok := rawRequest["allowEmbedding"]; ok { settings.AllowEmbedding = updates.AllowEmbedding } + if _, ok := rawRequest["backupPollingEnabled"]; ok { + settings.BackupPollingEnabled = updates.BackupPollingEnabled + } // Update the config // Note: PVE polling is hardcoded to 10s @@ -327,6 +360,16 @@ func (h *SystemSettingsHandler) HandleUpdateSystemSettings(w http.ResponseWriter if settings.PMGPollingInterval > 0 { h.config.PMGPollingInterval = time.Duration(settings.PMGPollingInterval) * time.Second } + if _, ok := rawRequest["backupPollingInterval"]; ok { + if settings.BackupPollingInterval <= 0 { + h.config.BackupPollingInterval = 0 + } else { + h.config.BackupPollingInterval = time.Duration(settings.BackupPollingInterval) * time.Second + } + } + if settings.BackupPollingEnabled != nil { + h.config.EnableBackupPolling = *settings.BackupPollingEnabled + } if settings.UpdateChannel != "" { h.config.UpdateChannel = settings.UpdateChannel } diff --git a/internal/config/config.go b/internal/config/config.go index b2cf0b1ea..e71fd7fa6 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -74,13 +74,15 @@ type Config struct { // Monitoring settings // Note: PVE polling is hardcoded to 10s since Proxmox cluster/resources endpoint only updates every 10s - PBSPollingInterval time.Duration `envconfig:"PBS_POLLING_INTERVAL"` // PBS polling interval (60s default) - PMGPollingInterval time.Duration `envconfig:"PMG_POLLING_INTERVAL"` // PMG polling interval (60s default) - ConcurrentPolling bool `envconfig:"CONCURRENT_POLLING" default:"true"` - ConnectionTimeout time.Duration `envconfig:"CONNECTION_TIMEOUT" default:"45s"` // Increased for slow storage operations - MetricsRetentionDays int `envconfig:"METRICS_RETENTION_DAYS" default:"7"` - BackupPollingCycles int `envconfig:"BACKUP_POLLING_CYCLES" default:"10"` - WebhookBatchDelay time.Duration `envconfig:"WEBHOOK_BATCH_DELAY" default:"10s"` + PBSPollingInterval time.Duration `envconfig:"PBS_POLLING_INTERVAL"` // PBS polling interval (60s default) + PMGPollingInterval time.Duration `envconfig:"PMG_POLLING_INTERVAL"` // PMG polling interval (60s default) + ConcurrentPolling bool `envconfig:"CONCURRENT_POLLING" default:"true"` + ConnectionTimeout time.Duration `envconfig:"CONNECTION_TIMEOUT" default:"45s"` // Increased for slow storage operations + MetricsRetentionDays int `envconfig:"METRICS_RETENTION_DAYS" default:"7"` + BackupPollingCycles int `envconfig:"BACKUP_POLLING_CYCLES" default:"10"` + BackupPollingInterval time.Duration `envconfig:"BACKUP_POLLING_INTERVAL"` + EnableBackupPolling bool `envconfig:"ENABLE_BACKUP_POLLING" default:"true"` + WebhookBatchDelay time.Duration `envconfig:"WEBHOOK_BATCH_DELAY" default:"10s"` // Logging settings LogLevel string `envconfig:"LOG_LEVEL" default:"info"` @@ -247,29 +249,31 @@ func Load() (*Config, error) { // Initialize config with defaults cfg := &Config{ - BackendHost: "0.0.0.0", - BackendPort: 3000, - FrontendHost: "0.0.0.0", - FrontendPort: 7655, - ConfigPath: dataDir, - DataPath: dataDir, - ConcurrentPolling: true, - ConnectionTimeout: 60 * time.Second, - MetricsRetentionDays: 7, - BackupPollingCycles: 10, - WebhookBatchDelay: 10 * time.Second, - LogLevel: "info", - LogMaxSize: 100, - LogMaxAge: 30, - LogCompress: true, - AllowedOrigins: "", // Empty means no CORS headers (same-origin only) - IframeEmbeddingAllow: "SAMEORIGIN", - PBSPollingInterval: 60 * time.Second, // Default PBS polling (slower) - PMGPollingInterval: 60 * time.Second, // Default PMG polling (aggregated stats) - DiscoveryEnabled: true, - DiscoverySubnet: "auto", - EnvOverrides: make(map[string]bool), - OIDC: NewOIDCConfig(), + BackendHost: "0.0.0.0", + BackendPort: 3000, + FrontendHost: "0.0.0.0", + FrontendPort: 7655, + ConfigPath: dataDir, + DataPath: dataDir, + ConcurrentPolling: true, + ConnectionTimeout: 60 * time.Second, + MetricsRetentionDays: 7, + BackupPollingCycles: 10, + BackupPollingInterval: 0, + EnableBackupPolling: true, + WebhookBatchDelay: 10 * time.Second, + LogLevel: "info", + LogMaxSize: 100, + LogMaxAge: 30, + LogCompress: true, + AllowedOrigins: "", // Empty means no CORS headers (same-origin only) + IframeEmbeddingAllow: "SAMEORIGIN", + PBSPollingInterval: 60 * time.Second, // Default PBS polling (slower) + PMGPollingInterval: 60 * time.Second, // Default PMG polling (aggregated stats) + DiscoveryEnabled: true, + DiscoverySubnet: "auto", + EnvOverrides: make(map[string]bool), + OIDC: NewOIDCConfig(), } // Initialize persistence @@ -301,6 +305,15 @@ func Load() (*Config, error) { cfg.PMGPollingInterval = time.Duration(systemSettings.PMGPollingInterval) * time.Second } + if systemSettings.BackupPollingInterval > 0 { + cfg.BackupPollingInterval = time.Duration(systemSettings.BackupPollingInterval) * time.Second + } else if systemSettings.BackupPollingInterval == 0 { + cfg.BackupPollingInterval = 0 + } + if systemSettings.BackupPollingEnabled != nil { + cfg.EnableBackupPolling = *systemSettings.BackupPollingEnabled + } + if systemSettings.UpdateChannel != "" { cfg.UpdateChannel = systemSettings.UpdateChannel } @@ -370,6 +383,53 @@ func Load() (*Config, error) { // Limited environment variable support // NOTE: Node configuration is NOT done via env vars - use the web UI instead + if cyclesStr := strings.TrimSpace(os.Getenv("BACKUP_POLLING_CYCLES")); cyclesStr != "" { + if cycles, err := strconv.Atoi(cyclesStr); err == nil { + if cycles < 0 { + log.Warn().Str("value", cyclesStr).Msg("Ignoring negative BACKUP_POLLING_CYCLES from environment") + } else { + cfg.BackupPollingCycles = cycles + cfg.EnvOverrides["BACKUP_POLLING_CYCLES"] = true + log.Info().Int("cycles", cycles).Msg("Overriding backup polling cycles from environment") + } + } else { + log.Warn().Str("value", cyclesStr).Msg("Invalid BACKUP_POLLING_CYCLES value, ignoring") + } + } + + if intervalStr := strings.TrimSpace(os.Getenv("BACKUP_POLLING_INTERVAL")); intervalStr != "" { + if dur, err := time.ParseDuration(intervalStr); err == nil { + if dur < 0 { + log.Warn().Str("value", intervalStr).Msg("Ignoring negative BACKUP_POLLING_INTERVAL from environment") + } else { + cfg.BackupPollingInterval = dur + cfg.EnvOverrides["BACKUP_POLLING_INTERVAL"] = true + log.Info().Dur("interval", dur).Msg("Overriding backup polling interval from environment") + } + } else if seconds, err := strconv.Atoi(intervalStr); err == nil { + if seconds < 0 { + log.Warn().Str("value", intervalStr).Msg("Ignoring negative BACKUP_POLLING_INTERVAL (seconds) from environment") + } else { + cfg.BackupPollingInterval = time.Duration(seconds) * time.Second + cfg.EnvOverrides["BACKUP_POLLING_INTERVAL"] = true + log.Info().Int("seconds", seconds).Msg("Overriding backup polling interval (seconds) from environment") + } + } else { + log.Warn().Str("value", intervalStr).Msg("Invalid BACKUP_POLLING_INTERVAL value, expected duration or seconds") + } + } + + if enabledStr := strings.TrimSpace(os.Getenv("ENABLE_BACKUP_POLLING")); enabledStr != "" { + switch strings.ToLower(enabledStr) { + case "0", "false", "no", "off": + cfg.EnableBackupPolling = false + default: + cfg.EnableBackupPolling = true + } + cfg.EnvOverrides["ENABLE_BACKUP_POLLING"] = true + log.Info().Bool("enabled", cfg.EnableBackupPolling).Msg("Overriding backup polling enabled flag from environment") + } + // Support both FRONTEND_PORT (preferred) and PORT (legacy) env vars if frontendPort := os.Getenv("FRONTEND_PORT"); frontendPort != "" { if p, err := strconv.Atoi(frontendPort); err == nil { diff --git a/internal/config/persistence.go b/internal/config/persistence.go index 3a6896d00..8be2837d1 100644 --- a/internal/config/persistence.go +++ b/internal/config/persistence.go @@ -541,6 +541,8 @@ type SystemSettings struct { // Note: PVE polling is hardcoded to 10s since Proxmox cluster/resources endpoint only updates every 10s PBSPollingInterval int `json:"pbsPollingInterval"` // PBS polling interval in seconds PMGPollingInterval int `json:"pmgPollingInterval"` // PMG polling interval in seconds + BackupPollingInterval int `json:"backupPollingInterval,omitempty"` + BackupPollingEnabled *bool `json:"backupPollingEnabled,omitempty"` BackendPort int `json:"backendPort,omitempty"` FrontendPort int `json:"frontendPort,omitempty"` AllowedOrigins string `json:"allowedOrigins,omitempty"` diff --git a/internal/config/settings.go b/internal/config/settings.go index 0e47d082e..5b0915483 100644 --- a/internal/config/settings.go +++ b/internal/config/settings.go @@ -28,10 +28,12 @@ type PortSettings struct { // MonitoringSettings contains monitoring-related configuration type MonitoringSettings struct { - PollingInterval int `json:"pollingInterval" yaml:"pollingInterval" mapstructure:"pollingInterval"` // milliseconds - ConcurrentPolling bool `json:"concurrentPolling" yaml:"concurrentPolling" mapstructure:"concurrentPolling"` - BackupPollingCycles int `json:"backupPollingCycles" yaml:"backupPollingCycles" mapstructure:"backupPollingCycles"` // How often to poll backups - MetricsRetentionDays int `json:"metricsRetentionDays" yaml:"metricsRetentionDays" mapstructure:"metricsRetentionDays"` + PollingInterval int `json:"pollingInterval" yaml:"pollingInterval" mapstructure:"pollingInterval"` // milliseconds + ConcurrentPolling bool `json:"concurrentPolling" yaml:"concurrentPolling" mapstructure:"concurrentPolling"` + BackupPollingCycles int `json:"backupPollingCycles" yaml:"backupPollingCycles" mapstructure:"backupPollingCycles"` // How often to poll backups + BackupPollingIntervalMs int `json:"backupPollingIntervalMs" yaml:"backupPollingIntervalMs" mapstructure:"backupPollingIntervalMs"` // 0 = use cycle-based scheduling + BackupPollingEnabled bool `json:"backupPollingEnabled" yaml:"backupPollingEnabled" mapstructure:"backupPollingEnabled"` + MetricsRetentionDays int `json:"metricsRetentionDays" yaml:"metricsRetentionDays" mapstructure:"metricsRetentionDays"` } // LoggingSettings contains logging configuration @@ -66,10 +68,12 @@ func DefaultSettings() *Settings { }, }, Monitoring: MonitoringSettings{ - PollingInterval: 5000, // 5 seconds - ConcurrentPolling: true, - BackupPollingCycles: 10, // Poll backups every 10 cycles - MetricsRetentionDays: 7, + PollingInterval: 5000, // 5 seconds + ConcurrentPolling: true, + BackupPollingCycles: 10, // Poll backups every 10 cycles + BackupPollingIntervalMs: 0, + BackupPollingEnabled: true, + MetricsRetentionDays: 7, }, Logging: LoggingSettings{ Level: "info", @@ -118,8 +122,12 @@ func (s *Settings) Validate() error { return fmt.Errorf("polling interval must be at least 1000ms (1 second)") } - if s.Monitoring.BackupPollingCycles < 1 { - return fmt.Errorf("backup polling cycles must be at least 1") + if s.Monitoring.BackupPollingCycles < 0 { + return fmt.Errorf("backup polling cycles cannot be negative") + } + + if s.Monitoring.BackupPollingIntervalMs < 0 { + return fmt.Errorf("backup polling interval cannot be negative") } // Validate logging level diff --git a/internal/monitoring/monitor.go b/internal/monitoring/monitor.go index fb9d5dc65..42be789a3 100644 --- a/internal/monitoring/monitor.go +++ b/internal/monitoring/monitor.go @@ -269,6 +269,8 @@ type Monitor struct { lastAuthAttempt map[string]time.Time // Track last auth attempt time lastClusterCheck map[string]time.Time // Track last cluster check for standalone nodes lastPhysicalDiskPoll map[string]time.Time // Track last physical disk poll time per instance + lastPVEBackupPoll map[string]time.Time // Track last PVE backup poll per instance + lastPBSBackupPoll map[string]time.Time // Track last PBS backup poll per instance persistence *config.ConfigPersistence // Add persistence for saving updated configs pbsBackupPollers map[string]bool // Track PBS backup polling goroutines per instance runtimeCtx context.Context // Context used while monitor is running @@ -318,6 +320,42 @@ func safeFloat(val float64) float64 { return val } +// shouldRunBackupPoll determines whether a backup polling cycle should execute. +// Returns whether polling should run, a human-readable skip reason, and the timestamp to record. +func (m *Monitor) shouldRunBackupPoll(last time.Time, now time.Time) (bool, string, time.Time) { + if m == nil || m.config == nil { + return false, "configuration unavailable", last + } + + if !m.config.EnableBackupPolling { + return false, "backup polling globally disabled", last + } + + interval := m.config.BackupPollingInterval + if interval > 0 { + if !last.IsZero() && now.Sub(last) < interval { + next := last.Add(interval) + return false, fmt.Sprintf("next run scheduled for %s", next.Format(time.RFC3339)), last + } + return true, "", now + } + + backupCycles := m.config.BackupPollingCycles + if backupCycles <= 0 { + backupCycles = 10 + } + + if m.pollCounter%int64(backupCycles) == 0 || m.pollCounter == 1 { + return true, "", now + } + + remaining := int64(backupCycles) - (m.pollCounter % int64(backupCycles)) + if remaining <= 0 { + remaining = int64(backupCycles) + } + return false, fmt.Sprintf("next run in %d polling cycles", remaining), last +} + const ( dockerConnectionPrefix = "docker-" dockerOfflineGraceMultiplier = 4 @@ -1286,6 +1324,8 @@ func New(cfg *config.Config) (*Monitor, error) { lastAuthAttempt: make(map[string]time.Time), lastClusterCheck: make(map[string]time.Time), lastPhysicalDiskPoll: make(map[string]time.Time), + lastPVEBackupPoll: make(map[string]time.Time), + lastPBSBackupPoll: make(map[string]time.Time), persistence: config.NewConfigPersistence(cfg.DataPath), pbsBackupPollers: make(map[string]bool), nodeSnapshots: make(map[string]NodeMemorySnapshot), @@ -3062,44 +3102,70 @@ func (m *Monitor) pollPVEInstance(ctx context.Context, instanceName string, clie } } - // Poll backups if enabled - using configurable cycle count - // This prevents slow backup/snapshot queries from blocking real-time stats - // Also poll on first cycle (pollCounter == 1) to ensure data loads quickly - backupCycles := 10 // default - if m.config.BackupPollingCycles > 0 { - backupCycles = m.config.BackupPollingCycles - } - if instanceCfg.MonitorBackups && (m.pollCounter%int64(backupCycles) == 0 || m.pollCounter == 1) { - select { - case <-ctx.Done(): - return - default: - // Run backup polling in a separate goroutine to not block main polling - go func() { - timeout := m.calculateBackupOperationTimeout(instanceName) - startTime := time.Now() - log.Info(). - Str("instance", instanceName). - Dur("timeout", timeout). - Msg("Starting background backup/snapshot polling") - // Create a separate context with longer timeout for backup operations - backupCtx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() + // Poll backups if enabled - respect configured interval or cycle gating + if instanceCfg.MonitorBackups { + if !m.config.EnableBackupPolling { + log.Debug(). + Str("instance", instanceName). + Msg("Skipping backup polling - globally disabled") + } else { + now := time.Now() - // Poll backup tasks - m.pollBackupTasks(backupCtx, instanceName, client) + m.mu.RLock() + lastPoll := m.lastPVEBackupPoll[instanceName] + m.mu.RUnlock() - // Poll storage backups - pass nodes to avoid duplicate API calls - m.pollStorageBackupsWithNodes(backupCtx, instanceName, client, nodes) + shouldPoll, reason, newLast := m.shouldRunBackupPoll(lastPoll, now) + if !shouldPoll { + if reason != "" { + log.Debug(). + Str("instance", instanceName). + Str("reason", reason). + Msg("Skipping PVE backup polling this cycle") + } + } else { + select { + case <-ctx.Done(): + return + default: + m.mu.Lock() + m.lastPVEBackupPoll[instanceName] = newLast + m.mu.Unlock() - // Poll guest snapshots - m.pollGuestSnapshots(backupCtx, instanceName, client) + // Run backup polling in a separate goroutine to avoid blocking real-time stats + go func(startTime time.Time, inst string, pveClient PVEClientInterface) { + timeout := m.calculateBackupOperationTimeout(inst) + log.Info(). + Str("instance", inst). + Dur("timeout", timeout). + Msg("Starting background backup/snapshot polling") - log.Info(). - Str("instance", instanceName). - Dur("duration", time.Since(startTime)). - Msg("Completed background backup/snapshot polling") - }() + // Create a separate context with longer timeout for backup operations + backupCtx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + // Poll backup tasks + m.pollBackupTasks(backupCtx, inst, pveClient) + + // Poll storage backups - pass nodes to avoid duplicate API calls + m.pollStorageBackupsWithNodes(backupCtx, inst, pveClient, nodes) + + // Poll guest snapshots + m.pollGuestSnapshots(backupCtx, inst, pveClient) + + duration := time.Since(startTime) + log.Info(). + Str("instance", inst). + Dur("duration", duration). + Msg("Completed background backup/snapshot polling") + + // Record actual completion time for interval scheduling + m.mu.Lock() + m.lastPVEBackupPoll[inst] = time.Now() + m.mu.Unlock() + }(now, instanceName, client) + } + } } } } @@ -4963,49 +5029,64 @@ func (m *Monitor) pollPBSInstance(ctx context.Context, instanceName string, clie Str("instance", instanceName). Msg("No PBS datastores available for backup polling") } else { - backupCycles := 10 - if m.config.BackupPollingCycles > 0 { - backupCycles = m.config.BackupPollingCycles - } - - shouldPoll := m.pollCounter%int64(backupCycles) == 0 || m.pollCounter == 1 - if !shouldPoll { + if !m.config.EnableBackupPolling { log.Debug(). Str("instance", instanceName). - Int64("cycle", m.pollCounter). - Int("backupCycles", backupCycles). - Msg("Skipping PBS backup polling this cycle") + Msg("Skipping PBS backup polling - globally disabled") } else { - m.mu.Lock() - if m.pbsBackupPollers[instanceName] { - m.mu.Unlock() - log.Debug(). - Str("instance", instanceName). - Msg("PBS backup polling already in progress") - } else { - m.pbsBackupPollers[instanceName] = true - m.mu.Unlock() + now := time.Now() + m.mu.RLock() + lastPoll := m.lastPBSBackupPoll[instanceName] + m.mu.RUnlock() + + shouldPoll, reason, newLast := m.shouldRunBackupPoll(lastPoll, now) + if !shouldPoll { + if reason != "" { + log.Debug(). + Str("instance", instanceName). + Str("reason", reason). + Msg("Skipping PBS backup polling this cycle") + } + } else { datastoreSnapshot := make([]models.PBSDatastore, len(pbsInst.Datastores)) copy(datastoreSnapshot, pbsInst.Datastores) - go func(ds []models.PBSDatastore) { - defer func() { - m.mu.Lock() - delete(m.pbsBackupPollers, instanceName) - m.mu.Unlock() - }() - - log.Info(). + m.mu.Lock() + if m.pbsBackupPollers[instanceName] { + m.mu.Unlock() + log.Debug(). Str("instance", instanceName). - Int("datastores", len(ds)). - Msg("Starting background PBS backup polling") + Msg("PBS backup polling already in progress") + } else { + m.pbsBackupPollers[instanceName] = true + m.lastPBSBackupPoll[instanceName] = newLast + m.mu.Unlock() - backupCtx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) - defer cancel() + go func(ds []models.PBSDatastore, inst string, start time.Time, pbsClient *pbs.Client) { + defer func() { + m.mu.Lock() + delete(m.pbsBackupPollers, inst) + m.lastPBSBackupPoll[inst] = time.Now() + m.mu.Unlock() + }() - m.pollPBSBackups(backupCtx, instanceName, client, ds) - }(datastoreSnapshot) + log.Info(). + Str("instance", inst). + Int("datastores", len(ds)). + Msg("Starting background PBS backup polling") + + backupCtx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + m.pollPBSBackups(backupCtx, inst, pbsClient, ds) + + log.Info(). + Str("instance", inst). + Dur("duration", time.Since(start)). + Msg("Completed background PBS backup polling") + }(datastoreSnapshot, instanceName, now, client) + } } } }