mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-19 10:33:29 +00:00
feat(agent): add secure custom numeric sensors
This commit is contained in:
parent
9c772a7200
commit
c0233ad56b
43 changed files with 1766 additions and 45 deletions
|
|
@ -399,6 +399,7 @@ func run(ctx context.Context, args []string, getenv func(string) string) error {
|
|||
InsecureSkipVerify: cfg.InsecureSkipVerify,
|
||||
CACertPath: cfg.CACertPath,
|
||||
ServerFingerprint: cfg.ServerFingerprint,
|
||||
CustomSensorsFile: cfg.CustomSensorsFile,
|
||||
DeploySSHUser: cfg.DeploySSHUser,
|
||||
LogLevel: cfg.LogLevel,
|
||||
Logger: &logger,
|
||||
|
|
@ -814,6 +815,7 @@ type Config struct {
|
|||
ServerFingerprint string
|
||||
ObserversFile string
|
||||
Observers []agenttarget.Observer
|
||||
CustomSensorsFile string
|
||||
DeploySSHUser string
|
||||
LogLevel zerolog.Level
|
||||
LogFile string
|
||||
|
|
@ -980,6 +982,7 @@ func loadConfig(args []string, getenv func(string) string) (Config, error) {
|
|||
envReportIP := strings.TrimSpace(getenv("PULSE_REPORT_IP"))
|
||||
envDisableCeph := strings.TrimSpace(getenv("PULSE_DISABLE_CEPH"))
|
||||
envObserversFile := strings.TrimSpace(getenv("PULSE_OBSERVERS_FILE"))
|
||||
envCustomSensorsFile := strings.TrimSpace(getenv("PULSE_CUSTOM_SENSORS_FILE"))
|
||||
|
||||
// Defaults
|
||||
defaultInterval := 30 * time.Second
|
||||
|
|
@ -1034,6 +1037,7 @@ func loadConfig(args []string, getenv func(string) string) (Config, error) {
|
|||
caCertFlag := fs.String("cacert", envCACertPath, "Path to custom CA bundle for agent HTTPS transport")
|
||||
serverFingerprintFlag := fs.String("server-fingerprint", envServerFingerprint, "Expected Pulse server TLS certificate fingerprint (SHA256)")
|
||||
observersFileFlag := fs.String("observers-file", envObserversFile, "Absolute path to a private JSON file defining report-only Pulse observer destinations")
|
||||
customSensorsFileFlag := fs.String("custom-sensors-file", envCustomSensorsFile, "Absolute path to a private YAML file defining local numeric sensor commands")
|
||||
deploySSHUserFlag := fs.String("deploy-ssh-user", envDeploySSHUser, "SSH user for peer deploy fan-out (default: root; non-root requires passwordless sudo)")
|
||||
logLevelFlag := fs.String("log-level", defaultLogLevel(envLogLevel), "Log level")
|
||||
logFileFlag := fs.String("log-file", envLogFile, "Write rotating JSON logs to this file")
|
||||
|
|
@ -1183,6 +1187,7 @@ func loadConfig(args []string, getenv func(string) string) (Config, error) {
|
|||
ServerFingerprint: strings.TrimSpace(*serverFingerprintFlag),
|
||||
ObserversFile: strings.TrimSpace(*observersFileFlag),
|
||||
Observers: observers,
|
||||
CustomSensorsFile: strings.TrimSpace(*customSensorsFileFlag),
|
||||
DeploySSHUser: deploySSHUser,
|
||||
LogLevel: logLevel,
|
||||
LogFile: strings.TrimSpace(*logFileFlag),
|
||||
|
|
|
|||
|
|
@ -950,6 +950,7 @@ func TestLoadConfig(t *testing.T) {
|
|||
"PULSE_CACERT": "/etc/pulse/ca.pem",
|
||||
"PULSE_SERVER_FINGERPRINT": "aabbccdd",
|
||||
"PULSE_DEPLOY_SSH_USER": "pulse-deploy",
|
||||
"PULSE_CUSTOM_SENSORS_FILE": "/etc/pulse/custom-sensors.yaml",
|
||||
}
|
||||
cfg, err := loadConfig([]string{}, func(s string) string { return env[s] })
|
||||
if err != nil {
|
||||
|
|
@ -976,10 +977,13 @@ func TestLoadConfig(t *testing.T) {
|
|||
if cfg.DeploySSHUser != "pulse-deploy" {
|
||||
t.Errorf("expected deploy SSH user from env, got %s", cfg.DeploySSHUser)
|
||||
}
|
||||
if cfg.CustomSensorsFile != "/etc/pulse/custom-sensors.yaml" {
|
||||
t.Errorf("expected custom sensors file from env, got %s", cfg.CustomSensorsFile)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("flag overrides", func(t *testing.T) {
|
||||
cfg, err := loadConfig([]string{"-url", "http://flag.example.com", "-token", "flag-token", "-enable-host=false", "-cacert", "/tmp/custom-ca.pem", "-server-fingerprint", "1122", "-deploy-ssh-user", "pulse-deploy"}, func(s string) string { return "" })
|
||||
cfg, err := loadConfig([]string{"-url", "http://flag.example.com", "-token", "flag-token", "-enable-host=false", "-cacert", "/tmp/custom-ca.pem", "-server-fingerprint", "1122", "-deploy-ssh-user", "pulse-deploy", "-custom-sensors-file", "/tmp/custom-sensors.yaml"}, func(s string) string { return "" })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -1001,6 +1005,9 @@ func TestLoadConfig(t *testing.T) {
|
|||
if cfg.DeploySSHUser != "pulse-deploy" {
|
||||
t.Errorf("expected deploy SSH user from flag, got %s", cfg.DeploySSHUser)
|
||||
}
|
||||
if cfg.CustomSensorsFile != "/tmp/custom-sensors.yaml" {
|
||||
t.Errorf("expected custom sensors file from flag, got %s", cfg.CustomSensorsFile)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("state directory flag overrides platform default", func(t *testing.T) {
|
||||
|
|
|
|||
|
|
@ -96,6 +96,7 @@ func (ws *windowsService) Execute(args []string, r <-chan svc.ChangeRequest, cha
|
|||
InsecureSkipVerify: ws.cfg.InsecureSkipVerify,
|
||||
CACertPath: ws.cfg.CACertPath,
|
||||
ServerFingerprint: ws.cfg.ServerFingerprint,
|
||||
CustomSensorsFile: ws.cfg.CustomSensorsFile,
|
||||
DeploySSHUser: ws.cfg.DeploySSHUser,
|
||||
LogLevel: ws.cfg.LogLevel,
|
||||
Logger: &ws.logger,
|
||||
|
|
|
|||
|
|
@ -42,6 +42,17 @@ When enabled, commands still flow through Pulse's command policy and approval
|
|||
surfaces instead of silently turning every agent into an unrestricted remote
|
||||
shell.
|
||||
|
||||
Custom numeric sensors are a separate, local configuration boundary. Enabling
|
||||
them with `--custom-sensors-file` does not enable remote commands and
|
||||
`--enable-commands` is not required. The server cannot add or alter a custom
|
||||
sensor command. The agent accepts only absolute executable paths with no
|
||||
arguments or shell interpretation, bounds concurrency, time, and output, and
|
||||
revalidates the command before every run. On POSIX systems the configuration,
|
||||
commands, and immediate command directories must pass ownership, symlink, and
|
||||
write-permission checks. Treat the configured executables as trusted agent
|
||||
code: the service commonly runs as root, so only administrators should be able
|
||||
to replace them.
|
||||
|
||||
Agent command tokens must be bound to a host or agent identity before command
|
||||
registration is accepted. Proxmox install-command tokens are the only first-use
|
||||
exception: because the server mints them before the installer knows the final
|
||||
|
|
|
|||
|
|
@ -136,6 +136,61 @@ containers or hardened service units without `CAP_NET_RAW`, ICMP checks fail;
|
|||
prefer TCP or HTTP checks there, or grant the capability. See "ICMP probe
|
||||
privileges" in docs/CONFIGURATION.md.
|
||||
|
||||
## Custom numeric sensors
|
||||
|
||||
The host module can report numeric metrics produced by local executables. This
|
||||
is intended for site-specific signals such as queue depth, UPS load, pending
|
||||
updates, or an application counter that Pulse cannot collect natively.
|
||||
|
||||
Create a private YAML file:
|
||||
|
||||
```yaml
|
||||
version: 1
|
||||
sensors:
|
||||
- id: queue_depth
|
||||
name: Queue depth
|
||||
command: /usr/local/libexec/pulse-queue-depth
|
||||
unit: items
|
||||
interval: 1m
|
||||
timeout: 2s
|
||||
warningAbove: 20
|
||||
criticalAbove: 50
|
||||
alertOnError: true
|
||||
```
|
||||
|
||||
Then start or restart the agent with
|
||||
`--custom-sensors-file /etc/pulse/custom-sensors.yaml`, or set
|
||||
`PULSE_CUSTOM_SENSORS_FILE` to that absolute path. Each command must be an
|
||||
absolute executable path, receives no arguments, and must write exactly one
|
||||
finite number to standard output. Put any query logic in the executable or
|
||||
script itself.
|
||||
|
||||
The agent evaluates optional `warningAbove`, `criticalAbove`, `warningBelow`,
|
||||
and `criticalBelow` thresholds locally. Pulse displays the typed value and unit
|
||||
under **Custom Metrics** and creates normal warning/critical alerts. A collection
|
||||
failure alerts by default; set `alertOnError: false` to make failures
|
||||
report-only. If a probe fails after a successful reading, the last good value is
|
||||
shown as stale with its original observation time.
|
||||
|
||||
This feature is deliberately local-only. The Pulse server and remote agent
|
||||
configuration cannot supply commands or arguments. The file is limited to 32
|
||||
sensors; intervals must be between 10 seconds and 24 hours; timeouts must be
|
||||
between 100 milliseconds and 10 seconds and shorter than the interval. At most
|
||||
four probes run concurrently, output is bounded, and each executable is
|
||||
revalidated before use.
|
||||
|
||||
On POSIX systems, the YAML file must be a regular, non-symlink file owned by
|
||||
the agent service user with no group/other permissions (normally mode `0600`).
|
||||
Commands and their immediate parent directories must also be regular,
|
||||
non-symlink, owned by the service user, and not group/other writable. Commands
|
||||
must have an executable bit. For example:
|
||||
|
||||
```bash
|
||||
sudo chown root:root /etc/pulse/custom-sensors.yaml /usr/local/libexec/pulse-queue-depth
|
||||
sudo chmod 0600 /etc/pulse/custom-sensors.yaml
|
||||
sudo chmod 0700 /usr/local/libexec/pulse-queue-depth
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
| Flag | Env Var | Description | Default |
|
||||
|
|
@ -143,6 +198,7 @@ privileges" in docs/CONFIGURATION.md.
|
|||
| `--url` | `PULSE_URL` | Pulse server URL | `http://localhost:7655` |
|
||||
| `--token` | `PULSE_TOKEN` | API token | *(required)* |
|
||||
| `--observers-file` | `PULSE_OBSERVERS_FILE` | Private JSON file defining report-only destinations | *(none)* |
|
||||
| `--custom-sensors-file` | `PULSE_CUSTOM_SENSORS_FILE` | Private YAML file defining local numeric sensor executables and thresholds | *(none)* |
|
||||
| `--token-file` | - | Read API token from file | *(unset)* |
|
||||
| `--interval` | `PULSE_INTERVAL` | Reporting interval | `30s` |
|
||||
| `--enable-host` | `PULSE_ENABLE_HOST` | Enable host metrics | `true` |
|
||||
|
|
|
|||
|
|
@ -1502,6 +1502,22 @@ the intentionally sparse public response.
|
|||
the report or command protocol.
|
||||
Neither side may infer lifecycle health, command authority, enrollment
|
||||
state, or GPU workload/process inventory from `nvidia-smi` output.
|
||||
Site-defined numeric sensors belong in the typed `sensors.custom` report
|
||||
contract rather than the temperature-oriented `sensors.additional` map.
|
||||
Their execution authority is local-only: the agent may load a private,
|
||||
versioned YAML file named by `--custom-sensors-file` or
|
||||
`PULSE_CUSTOM_SENSORS_FILE`, but server profiles, remote configuration, and
|
||||
command transport must not author executable paths or arguments. The
|
||||
runtime must reject shell strings and relative, symlinked, or insecure
|
||||
executable/config paths; enforce bounded sensor count, intervals, timeouts,
|
||||
output, and concurrency; revalidate executables before use; and accept only
|
||||
one finite numeric value per execution. Typed identity, unit, status,
|
||||
observation time, stale last-good value, and bounded error state must survive
|
||||
report ingest, model/front-end conversion, unified-resource cloning, and
|
||||
alert cleanup. Threshold evaluation remains agent-authored, while the server
|
||||
maps warning, critical, and opted-in error states into canonical alerts.
|
||||
Enabling custom sensors must not imply `agent:exec`, AI command authority, or
|
||||
any general remote-shell capability.
|
||||
Runtime RAID collection uses `/proc/mdstat` as the canonical discovery
|
||||
baseline for Linux md arrays. `mdadm --detail` may enrich level, state,
|
||||
member, UUID, and rebuild fields when available, but missing or failing
|
||||
|
|
@ -2283,6 +2299,20 @@ Agent` secondary handoff against the live setup wizard instead of relying
|
|||
|
||||
## Current State
|
||||
|
||||
### Local custom numeric sensors are a bounded reporting module
|
||||
|
||||
The host agent now accepts an optional private version-1 YAML file through
|
||||
`--custom-sensors-file` / `PULSE_CUSTOM_SENSORS_FILE`. Each of at most 32
|
||||
definitions names one absolute local executable, collection interval, timeout,
|
||||
unit, and optional upper/lower warning and critical thresholds. The executable
|
||||
receives no arguments and must emit exactly one finite number. Collection is
|
||||
bounded to four concurrent executions inside the existing ten-second host
|
||||
collection context; last-good values survive failures as stale evidence with
|
||||
their original observation time. The resulting typed `sensors.custom` array is
|
||||
reporting data only and grants no command, enrollment, lifecycle, remote-shell,
|
||||
or AI authority. Agent initialization fails closed on invalid configuration,
|
||||
and Windows service startup carries the same optional file path.
|
||||
|
||||
### Rootful Docker outranks rootless socket discovery and runtime labels are truthful
|
||||
|
||||
Installer container-runtime discovery
|
||||
|
|
|
|||
|
|
@ -334,6 +334,19 @@ inspectability, or convert missing/stale evidence into health.
|
|||
|
||||
## Current State
|
||||
|
||||
### Agent custom sensors use canonical health-assessment alerts
|
||||
|
||||
Typed `HostSensorSummary.Custom` readings with `warning` or `critical` status
|
||||
now map to one canonical `custom-sensor` alert per host and sensor ID.
|
||||
Collection errors map to warning only when the local definition reports
|
||||
`alertOnError`; healthy reports, removed definitions, disabled host thresholds,
|
||||
host removal, and telemetry expiry clear the same identity. Alert evaluation
|
||||
consumes the agent-authored typed status and never executes or receives the
|
||||
local sensor command. `TestHostCustomSensorAlertLifecycle` and
|
||||
`TestHostCustomSensorErrorCanBeReportOnly` in
|
||||
`internal/alerts/host_unraid_lifecycle_test.go` pin creation, recovery,
|
||||
opt-out, and cleanup.
|
||||
|
||||
The alert resource-incident panel
|
||||
(`frontend-modern/src/features/alerts/AlertResourceIncidentsPanel.tsx`)
|
||||
dropped its "Open in Infrastructure / Workloads / Storage / Recovery"
|
||||
|
|
|
|||
|
|
@ -3771,6 +3771,18 @@ auto-register mutation boundary.
|
|||
|
||||
## Current State
|
||||
|
||||
### Host sensor payloads carry typed custom numeric readings
|
||||
|
||||
Agent, host-model, frontend, and unified-resource sensor payloads now include a
|
||||
`custom` array whose entries preserve `id`, `name`, optional `unit` and numeric
|
||||
`value`, agent-authored `status`, `observedAt`, bounded `error`,
|
||||
`alertOnError`, and `stale`. This is an additive wire field: older agents and
|
||||
clients may omit it, and normalization exposes an empty collection rather than
|
||||
reinterpreting the legacy temperature-oriented `additional` map. Accepted
|
||||
report ingest and the JSON projection are pinned by
|
||||
`TestApplyHostReportPreservesTypedSensorSummary` and
|
||||
`TestHostCustomSensorMetaJSONContract`.
|
||||
|
||||
### System settings shed the dead auto-update schedule fields
|
||||
|
||||
The system settings payload (`internal/config.SystemSettings`, projected by
|
||||
|
|
|
|||
|
|
@ -902,6 +902,18 @@ changes.
|
|||
|
||||
## Current State
|
||||
|
||||
### Custom sensor evidence stays typed through host monitoring
|
||||
|
||||
Authenticated host reports now copy bounded `sensors.custom` entries into the
|
||||
host model and the unified read state without folding their units or values
|
||||
into temperature maps. Value pointers and collections are cloned at report,
|
||||
model/frontend, and read-state boundaries; a stale last-good value keeps its
|
||||
original observation time. Monitoring passes the typed status to the alert
|
||||
subsystem but does not execute probes or recompute locally configured
|
||||
thresholds. `TestApplyHostReportPreservesTypedSensorSummary` and
|
||||
`TestHostSensorsFromReadStateViewPreservesTypedSensorData` pin accepted ingest
|
||||
and read-state preservation.
|
||||
|
||||
### PBS health is one completed-poll outcome
|
||||
|
||||
PBS client construction is transport setup, not connectivity evidence. Initial
|
||||
|
|
|
|||
|
|
@ -881,6 +881,17 @@ without contention or cross-delivery.
|
|||
|
||||
## Current State
|
||||
|
||||
### Custom metric drawer projection remains resource-local and bounded
|
||||
|
||||
The Infrastructure detail drawer formats `sensors.custom` from the already
|
||||
selected resource payload. It performs no fetch, subscription, global resource
|
||||
scan, or history query, and the native agent limits the source collection to 32
|
||||
definitions with four concurrent executions. Formatting is a bounded
|
||||
sort-and-map over that collection and remains outside table-row and virtualized
|
||||
viewport hot paths.
|
||||
`UnifiedResourceTable.performance.contract.test.tsx` pins the resource-local
|
||||
projection and no-fetch boundary.
|
||||
|
||||
### Workload refreshes retain one coherent paged generation
|
||||
|
||||
The workload polling cache accepts a REST refresh only when every advertised
|
||||
|
|
|
|||
|
|
@ -597,6 +597,20 @@ tokens, and path-normalization variants.
|
|||
|
||||
## Current State
|
||||
|
||||
### Custom sensor execution authority is local-only and fail-closed
|
||||
|
||||
`pulse-agent` may load site-defined numeric probes only from the absolute path
|
||||
named by `--custom-sensors-file` or `PULSE_CUSTOM_SENSORS_FILE`. Server
|
||||
profiles, remote configuration, AI tooling, and command transport cannot author
|
||||
the executable or arguments. The strict versioned YAML schema accepts absolute
|
||||
executables with no shell or arguments; POSIX ownership, permissions, and
|
||||
symlink checks protect the config, executable, and immediate parent directory,
|
||||
and the executable is revalidated before each run. Count, concurrency, timeout,
|
||||
stdout, and error sizes are bounded. Probe stderr is discarded so credentials
|
||||
or query detail cannot enter reports or alerts; tests in
|
||||
`internal/hostagent/custom_sensors_test.go` pin unsafe-config rejection and the
|
||||
stderr boundary.
|
||||
|
||||
### System settings mutation surface no longer accepts dead schedule inputs
|
||||
|
||||
The governed system-settings write path in `internal/api/system_settings.go`
|
||||
|
|
|
|||
|
|
@ -2032,6 +2032,17 @@ capability over storage or recovery data.
|
|||
|
||||
## Current State
|
||||
|
||||
### Unified host read state preserves custom sensor evidence
|
||||
|
||||
The shared host/unified-resource read-state boundary now carries typed custom
|
||||
numeric readings as cloned evidence. Storage and recovery consumers may display
|
||||
or preserve that host snapshot, but must not reinterpret custom units as
|
||||
temperatures, execute probe paths, or treat a stale last-good value as a fresh
|
||||
observation. No custom executable path or arguments enter the server model,
|
||||
durable recovery data, or frontend payload. Adapter and clone isolation are
|
||||
pinned by `TestResourceFromHostPreservesCustomSensorMeta` and
|
||||
`TestCloneResourceIsolatesCustomSensorValues`.
|
||||
|
||||
### Shared system-settings boundary dropped dead auto-update schedule fields
|
||||
|
||||
The shared `internal/api` system-settings surface this subsystem consumes
|
||||
|
|
|
|||
|
|
@ -1680,6 +1680,18 @@ AI-only summary payloads, or page-local heuristics.
|
|||
|
||||
## Current State
|
||||
|
||||
### Agent custom sensors remain typed, cloned host metadata
|
||||
|
||||
`resourceFromHost` projects `models.HostSensorSummary.Custom` into
|
||||
`Agent.Sensors.Custom` with a private slice and numeric value pointers.
|
||||
Read-state cloning preserves identity, name, unit, status, observation time,
|
||||
bounded error, error-alert preference, and stale state without aliasing mutable
|
||||
host-model memory. The detail drawer reads this typed collection under
|
||||
**Custom Metrics**; it does not fold custom values into thermal metadata or
|
||||
change canonical resource identity. `TestResourceFromHostPreservesCustomSensorMeta`,
|
||||
`TestCloneResourceIsolatesCustomSensorValues`, and
|
||||
`TestHostCustomSensorMetaJSONContract` pin adapter, clone, and JSON behavior.
|
||||
|
||||
### Host views expose the integration-source discriminator
|
||||
|
||||
`HostView.IntegrationSource()` names the platform integration ("vmware",
|
||||
|
|
|
|||
|
|
@ -168,6 +168,7 @@ const HostDetailsDisclosure: Component<{
|
|||
<DisksCard disks={agent().disks} />
|
||||
<RaidCard arrays={props.drawer.agentMeta()?.raid} />
|
||||
<TemperaturesCard rows={props.drawer.temperatureRows()} title="Thermals" />
|
||||
<TemperaturesCard rows={props.drawer.customSensorRows()} title="Custom Metrics" />
|
||||
</>
|
||||
)}
|
||||
</Show>
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import type { Resource } from '@/types/resource';
|
|||
import { UnifiedResourceTable } from '@/components/Infrastructure/UnifiedResourceTable';
|
||||
import { ResourceFacetSummary } from '@/components/Infrastructure/ResourceFacetSummary';
|
||||
import {
|
||||
buildCustomSensorRows,
|
||||
buildTemperatureRows,
|
||||
formatSensorName,
|
||||
} from '@/components/Infrastructure/resourceDetailMappers';
|
||||
|
|
@ -628,6 +629,30 @@ describe('UnifiedResourceTable performance contract', () => {
|
|||
expect(resourceDetailMappersSource).not.toContain('fetch(');
|
||||
});
|
||||
|
||||
it('keeps custom metric formatting bounded to the selected resource payload', () => {
|
||||
expect(
|
||||
buildCustomSensorRows({
|
||||
custom: [
|
||||
{
|
||||
id: 'queue_depth',
|
||||
name: 'Queue depth',
|
||||
unit: 'items',
|
||||
value: 12.5,
|
||||
status: 'warning',
|
||||
observedAt: '2026-07-30T19:00:00Z',
|
||||
},
|
||||
],
|
||||
}),
|
||||
).toEqual([
|
||||
{
|
||||
label: 'Queue depth',
|
||||
value: '12.5 items',
|
||||
valueTitle: '12.5 items · Warning',
|
||||
},
|
||||
]);
|
||||
expect(resourceDetailMappersSource).not.toContain('fetch(');
|
||||
});
|
||||
|
||||
it('keeps power and fan detail rows bounded to the selected resource payload', () => {
|
||||
expect(
|
||||
buildTemperatureRows({
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
buildCustomSensorRows,
|
||||
buildTemperatureRows,
|
||||
formatInteger,
|
||||
formatSensorName,
|
||||
|
|
@ -202,6 +203,68 @@ describe('resourceDetailMappers', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('buildCustomSensorRows', () => {
|
||||
it('formats units, status, and stale last-good values without treating them as temperatures', () => {
|
||||
expect(
|
||||
buildCustomSensorRows({
|
||||
custom: [
|
||||
{
|
||||
id: 'queue_depth',
|
||||
name: 'Queue depth',
|
||||
unit: 'items',
|
||||
value: 12.25,
|
||||
status: 'warning',
|
||||
observedAt: '2026-07-30T19:00:00Z',
|
||||
},
|
||||
{
|
||||
id: 'cache_hit_rate',
|
||||
name: 'Cache hit rate',
|
||||
unit: '%',
|
||||
value: 98.5,
|
||||
status: 'error',
|
||||
observedAt: '2026-07-30T18:55:00Z',
|
||||
error: 'probe timed out',
|
||||
stale: true,
|
||||
},
|
||||
],
|
||||
}),
|
||||
).toEqual([
|
||||
{
|
||||
label: 'Cache hit rate',
|
||||
value: '98.5% (stale)',
|
||||
valueTitle: '98.5% (stale) · Error · probe timed out',
|
||||
},
|
||||
{
|
||||
label: 'Queue depth',
|
||||
value: '12.25 items',
|
||||
valueTitle: '12.25 items · Warning',
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('shows unavailable for a failed sensor with no last-good value', () => {
|
||||
expect(
|
||||
buildCustomSensorRows({
|
||||
custom: [
|
||||
{
|
||||
id: 'failed',
|
||||
name: 'Failed probe',
|
||||
status: 'error',
|
||||
observedAt: '2026-07-30T19:00:00Z',
|
||||
error: 'invalid output',
|
||||
},
|
||||
],
|
||||
}),
|
||||
).toEqual([
|
||||
{
|
||||
label: 'Failed probe',
|
||||
value: 'Unavailable',
|
||||
valueTitle: 'Unavailable · Error · invalid output',
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('toNodeFromProxmox', () => {
|
||||
it('preserves canonical linkedAgentId for hybrid hosts', () => {
|
||||
const node = toNodeFromProxmox(createHybridHostResource());
|
||||
|
|
|
|||
|
|
@ -515,6 +515,38 @@ export const buildTemperatureRows = (sensors?: HostSensorSummary) => {
|
|||
return rows;
|
||||
};
|
||||
|
||||
const formatCustomSensorValue = (value: number, unit?: string): string => {
|
||||
const formatted = new Intl.NumberFormat(undefined, {
|
||||
maximumFractionDigits: 3,
|
||||
}).format(value);
|
||||
const normalizedUnit = (unit || '').trim();
|
||||
if (!normalizedUnit) return formatted;
|
||||
if (normalizedUnit === '%' || normalizedUnit.startsWith('°')) return `${formatted}${normalizedUnit}`;
|
||||
return `${formatted} ${normalizedUnit}`;
|
||||
};
|
||||
|
||||
export const buildCustomSensorRows = (sensors?: HostSensorSummary) =>
|
||||
(sensors?.custom ?? [])
|
||||
.filter((metric) => metric.id?.trim() && metric.name?.trim())
|
||||
.sort((a, b) => a.name.localeCompare(b.name))
|
||||
.map((metric) => {
|
||||
const status = (metric.status || 'unknown').trim().toLowerCase();
|
||||
const hasValue = typeof metric.value === 'number' && Number.isFinite(metric.value);
|
||||
const measured = hasValue
|
||||
? formatCustomSensorValue(metric.value as number, metric.unit)
|
||||
: 'Unavailable';
|
||||
const value = metric.stale ? `${measured} (stale)` : measured;
|
||||
const details = [
|
||||
status === 'ok' ? 'OK' : titleCaseDelimitedLabel(status),
|
||||
metric.error?.trim(),
|
||||
].filter(Boolean);
|
||||
return {
|
||||
label: metric.name.trim(),
|
||||
value,
|
||||
valueTitle: details.length > 0 ? `${value} · ${details.join(' · ')}` : value,
|
||||
};
|
||||
});
|
||||
|
||||
export const formatInteger = (value?: number): string => {
|
||||
if (value === undefined || value === null || Number.isNaN(value)) return '—';
|
||||
return new Intl.NumberFormat().format(Math.round(value));
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ import {
|
|||
import { formatResourceAnalysisSummary } from '@/utils/resourceAnalysisPresentation';
|
||||
import type { ResourceIntelligence } from '@/types/aiIntelligence';
|
||||
import {
|
||||
buildCustomSensorRows,
|
||||
buildTemperatureRows,
|
||||
toAgentFromResource,
|
||||
toNodeFromProxmox,
|
||||
|
|
@ -174,6 +175,7 @@ export const useResourceDetailDrawerDerivedState = (
|
|||
const proxmoxNode = createMemo(() => toNodeFromProxmox(resource));
|
||||
const agentInfo = createMemo(() => toAgentFromResource(resource, agentMeta()));
|
||||
const temperatureRows = createMemo(() => buildTemperatureRows(agentInfo()?.sensors));
|
||||
const customSensorRows = createMemo(() => buildCustomSensorRows(agentInfo()?.sensors));
|
||||
|
||||
const dockerHostData = createMemo(() => platformData()?.docker as DockerPlatformData | undefined);
|
||||
const dockerHostSourceId = createMemo(
|
||||
|
|
@ -312,7 +314,7 @@ export const useResourceDetailDrawerDerivedState = (
|
|||
networkInterfaceCount: agentInfo()?.networkInterfaces?.length ?? 0,
|
||||
diskCount: agentInfo()?.disks?.length ?? 0,
|
||||
raidCount: agentMeta()?.raid?.length ?? 0,
|
||||
temperatureRowCount: temperatureRows().length,
|
||||
temperatureRowCount: temperatureRows().length + customSensorRows().length,
|
||||
}),
|
||||
);
|
||||
const hasHostDetails = createMemo(() => hostDetailCards().length > 0);
|
||||
|
|
@ -425,6 +427,7 @@ export const useResourceDetailDrawerDerivedState = (
|
|||
proxmoxNode,
|
||||
agentInfo,
|
||||
temperatureRows,
|
||||
customSensorRows,
|
||||
dockerHostData,
|
||||
dockerHostSourceId,
|
||||
dockerUpdatesAvailable,
|
||||
|
|
|
|||
|
|
@ -98,6 +98,17 @@ type APIAgentSensorSummary = {
|
|||
temperatureCelsius?: Record<string, number>;
|
||||
fanRpm?: Record<string, number>;
|
||||
additional?: Record<string, number>;
|
||||
custom?: Array<{
|
||||
id: string;
|
||||
name: string;
|
||||
unit?: string;
|
||||
value?: number;
|
||||
status: string;
|
||||
observedAt: string;
|
||||
error?: string;
|
||||
alertOnError?: boolean;
|
||||
stale?: boolean;
|
||||
}>;
|
||||
thermalState?: {
|
||||
source?: string;
|
||||
pressure?: string;
|
||||
|
|
|
|||
|
|
@ -543,11 +543,24 @@ export interface HostSensorSummary {
|
|||
fanRpm?: Record<string, number>;
|
||||
powerWatts?: Record<string, number>;
|
||||
additional?: Record<string, number>;
|
||||
custom?: HostCustomSensorMetric[];
|
||||
gpu?: HostGPUSensor[];
|
||||
thermalState?: HostThermalState;
|
||||
smart?: HostDiskSMART[]; // S.M.A.R.T. disk data
|
||||
}
|
||||
|
||||
export interface HostCustomSensorMetric {
|
||||
id: string;
|
||||
name: string;
|
||||
unit?: string;
|
||||
value?: number;
|
||||
status: 'ok' | 'warning' | 'critical' | 'error' | string;
|
||||
observedAt: string;
|
||||
error?: string;
|
||||
alertOnError?: boolean;
|
||||
stale?: boolean;
|
||||
}
|
||||
|
||||
export interface HostGPUSensor {
|
||||
id?: string;
|
||||
name?: string;
|
||||
|
|
|
|||
|
|
@ -248,6 +248,7 @@ func (m *Manager) CheckHost(host models.Host) {
|
|||
m.clearHostDiskAlerts(host.ID)
|
||||
m.clearHostRAIDAlerts(host.ID)
|
||||
m.clearHostUnraidAlerts(host.ID)
|
||||
m.clearHostCustomSensorAlerts(host.ID)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -256,6 +257,7 @@ func (m *Manager) CheckHost(host models.Host) {
|
|||
m.clearHostDiskAlerts(host.ID)
|
||||
m.clearHostRAIDAlerts(host.ID)
|
||||
m.clearHostUnraidAlerts(host.ID)
|
||||
m.clearHostCustomSensorAlerts(host.ID)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -287,6 +289,7 @@ func (m *Manager) CheckHost(host models.Host) {
|
|||
if len(host.Tags) > 0 {
|
||||
baseMetadata["tags"] = append([]string(nil), host.Tags...)
|
||||
}
|
||||
m.syncHostCustomSensorAlerts(host, nodeName, instanceName, baseMetadata)
|
||||
|
||||
if thresholds.CPU != nil {
|
||||
cpuMetadata := cloneMetadata(baseMetadata)
|
||||
|
|
@ -615,6 +618,7 @@ func (m *Manager) HandleHostRemoved(host models.Host) {
|
|||
m.clearHostDiskAlerts(host.ID)
|
||||
m.clearHostRAIDAlerts(host.ID)
|
||||
m.clearHostUnraidAlerts(host.ID)
|
||||
m.clearHostCustomSensorAlerts(host.ID)
|
||||
}
|
||||
|
||||
// HandleHostTelemetryExpired re-evaluates transient storage-operation evidence
|
||||
|
|
@ -647,8 +651,10 @@ func (m *Manager) HandleHostTelemetryExpired(host models.Host) {
|
|||
if !alertsEnabled || disableAllAgents || thresholds.Disabled {
|
||||
m.clearHostRAIDAlerts(host.ID)
|
||||
m.clearHostUnraidAlerts(host.ID)
|
||||
m.clearHostCustomSensorAlerts(host.ID)
|
||||
return
|
||||
}
|
||||
m.clearHostCustomSensorAlerts(host.ID)
|
||||
|
||||
if host.Unraid == nil {
|
||||
m.clearHostUnraidAlerts(host.ID)
|
||||
|
|
@ -893,6 +899,123 @@ func (m *Manager) clearHostDiskAlerts(hostID string) {
|
|||
}
|
||||
}
|
||||
|
||||
var customSensorAssessmentCodes = []string{
|
||||
"custom_sensor_warning",
|
||||
"custom_sensor_critical",
|
||||
"custom_sensor_error",
|
||||
}
|
||||
|
||||
func (m *Manager) syncHostCustomSensorAlerts(host models.Host, nodeName, instanceName string, baseMetadata map[string]interface{}) {
|
||||
seen := make(map[string]struct{}, len(host.Sensors.Custom))
|
||||
for _, metric := range host.Sensors.Custom {
|
||||
metricID := sanitizeHostComponent(strings.TrimSpace(metric.ID))
|
||||
if metricID == "" {
|
||||
continue
|
||||
}
|
||||
resourceID := fmt.Sprintf("%s/custom:%s", hostResourceID(host.ID), metricID)
|
||||
seen[resourceID] = struct{}{}
|
||||
|
||||
reasons := make([]storagehealth.Reason, 0, 1)
|
||||
valueText := "unavailable"
|
||||
if metric.Value != nil {
|
||||
valueText = fmt.Sprintf("%g", *metric.Value)
|
||||
if unit := strings.TrimSpace(metric.Unit); unit != "" {
|
||||
valueText += " " + unit
|
||||
}
|
||||
}
|
||||
switch strings.ToLower(strings.TrimSpace(metric.Status)) {
|
||||
case "critical":
|
||||
reasons = append(reasons, storagehealth.Reason{
|
||||
Code: "custom_sensor_critical",
|
||||
Severity: storagehealth.RiskCritical,
|
||||
Summary: fmt.Sprintf("%s is critical at %s", metric.Name, valueText),
|
||||
})
|
||||
case "warning":
|
||||
reasons = append(reasons, storagehealth.Reason{
|
||||
Code: "custom_sensor_warning",
|
||||
Severity: storagehealth.RiskWarning,
|
||||
Summary: fmt.Sprintf("%s is warning at %s", metric.Name, valueText),
|
||||
})
|
||||
case "error":
|
||||
if metric.AlertOnError {
|
||||
message := strings.TrimSpace(metric.Error)
|
||||
if message == "" {
|
||||
message = "collector execution failed"
|
||||
}
|
||||
reasons = append(reasons, storagehealth.Reason{
|
||||
Code: "custom_sensor_error",
|
||||
Severity: storagehealth.RiskWarning,
|
||||
Summary: fmt.Sprintf("%s custom sensor error: %s", metric.Name, message),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
metadata := cloneMetadata(baseMetadata)
|
||||
metadata["metric"] = "customSensor"
|
||||
metadata["customSensorId"] = metric.ID
|
||||
metadata["customSensorName"] = metric.Name
|
||||
metadata["customSensorUnit"] = metric.Unit
|
||||
metadata["customSensorStatus"] = metric.Status
|
||||
metadata["customSensorStale"] = metric.Stale
|
||||
if metric.Value != nil {
|
||||
metadata["customSensorValue"] = *metric.Value
|
||||
}
|
||||
if metric.Error != "" {
|
||||
metadata["customSensorError"] = metric.Error
|
||||
}
|
||||
|
||||
resourceName := fmt.Sprintf("%s - %s", hostDisplayName(host), metric.Name)
|
||||
_, _ = m.syncCanonicalHealthAssessmentAlert(canonicalHealthAssessmentAlertParams{
|
||||
SpecID: resourceID + "-health",
|
||||
Signal: "custom-sensor",
|
||||
Codes: customSensorAssessmentCodes,
|
||||
Reasons: reasons,
|
||||
AlertID: fmt.Sprintf("host-%s-custom-%s", host.ID, metricID),
|
||||
AlertType: "custom-sensor",
|
||||
SpecResourceID: resourceID,
|
||||
ResourceID: resourceID,
|
||||
ResourceName: resourceName,
|
||||
ResourceType: unifiedresources.ResourceTypeAgent,
|
||||
Node: nodeName,
|
||||
Instance: instanceName,
|
||||
Metadata: metadata,
|
||||
MessageBuilder: func(result alertspecs.EvaluationResult) (string, float64, float64) {
|
||||
message := strings.Join(storageHealthReasonSummaries(reasons), "; ")
|
||||
value := 0.0
|
||||
if metric.Value != nil {
|
||||
value = *metric.Value
|
||||
}
|
||||
return message, value, 0
|
||||
},
|
||||
})
|
||||
}
|
||||
m.cleanupHostCustomSensorAlerts(host.ID, seen)
|
||||
}
|
||||
|
||||
func (m *Manager) clearHostCustomSensorAlerts(hostID string) {
|
||||
m.cleanupHostCustomSensorAlerts(hostID, nil)
|
||||
}
|
||||
|
||||
func (m *Manager) cleanupHostCustomSensorAlerts(hostID string, seen map[string]struct{}) {
|
||||
if strings.TrimSpace(hostID) == "" {
|
||||
return
|
||||
}
|
||||
prefix := hostResourceID(hostID) + "/custom:"
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
for storageKey, alert := range m.activeAlerts {
|
||||
if alert == nil || !strings.HasPrefix(alert.ResourceID, prefix) {
|
||||
continue
|
||||
}
|
||||
if seen != nil {
|
||||
if _, exists := seen[alert.ResourceID]; exists {
|
||||
continue
|
||||
}
|
||||
}
|
||||
m.clearAlertNoLock(storageKey)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) clearGuestMetricAlerts(guestID string, metrics ...string) int {
|
||||
if guestID == "" {
|
||||
return 0
|
||||
|
|
|
|||
|
|
@ -3,10 +3,80 @@ package alerts
|
|||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
||||
)
|
||||
|
||||
func TestHostCustomSensorAlertLifecycle(t *testing.T) {
|
||||
manager := newTestManager(t)
|
||||
manager.ClearActiveAlerts()
|
||||
value := 23.0
|
||||
host := models.Host{
|
||||
ID: "custom-sensor-host",
|
||||
Hostname: "edge-1",
|
||||
DisplayName: "Edge 1",
|
||||
Sensors: models.HostSensorSummary{
|
||||
Custom: []models.HostCustomSensorMetric{{
|
||||
ID: "queue_depth",
|
||||
Name: "Queue depth",
|
||||
Unit: "items",
|
||||
Value: &value,
|
||||
Status: "critical",
|
||||
ObservedAt: time.Now().UTC(),
|
||||
AlertOnError: true,
|
||||
}},
|
||||
},
|
||||
}
|
||||
|
||||
manager.CheckHost(host)
|
||||
if !hasAlertType(manager.GetActiveAlerts(), "custom-sensor") {
|
||||
t.Fatal("expected custom sensor alert")
|
||||
}
|
||||
|
||||
host.Sensors.Custom[0].Status = "ok"
|
||||
manager.CheckHost(host)
|
||||
if hasAlertType(manager.GetActiveAlerts(), "custom-sensor") {
|
||||
t.Fatal("healthy custom sensor did not resolve its alert")
|
||||
}
|
||||
|
||||
host.Sensors.Custom[0].Status = "error"
|
||||
host.Sensors.Custom[0].Error = "probe timed out"
|
||||
manager.CheckHost(host)
|
||||
if !hasAlertType(manager.GetActiveAlerts(), "custom-sensor") {
|
||||
t.Fatal("alertOnError custom sensor did not alert")
|
||||
}
|
||||
|
||||
host.Sensors.Custom = nil
|
||||
manager.CheckHost(host)
|
||||
if hasAlertType(manager.GetActiveAlerts(), "custom-sensor") {
|
||||
t.Fatal("removed custom sensor did not clear its alert")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostCustomSensorErrorCanBeReportOnly(t *testing.T) {
|
||||
manager := newTestManager(t)
|
||||
manager.ClearActiveAlerts()
|
||||
host := models.Host{
|
||||
ID: "custom-sensor-report-only",
|
||||
Hostname: "edge-2",
|
||||
Sensors: models.HostSensorSummary{
|
||||
Custom: []models.HostCustomSensorMetric{{
|
||||
ID: "optional_probe",
|
||||
Name: "Optional probe",
|
||||
Status: "error",
|
||||
ObservedAt: time.Now().UTC(),
|
||||
Error: "not installed",
|
||||
}},
|
||||
},
|
||||
}
|
||||
|
||||
manager.CheckHost(host)
|
||||
if hasAlertType(manager.GetActiveAlerts(), "custom-sensor") {
|
||||
t.Fatal("alertOnError=false custom sensor unexpectedly alerted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleHostOfflineExpiresUnraidOperationAlertBeforeConnectivityConfirmation(t *testing.T) {
|
||||
manager := newTestManager(t)
|
||||
manager.ClearActiveAlerts()
|
||||
|
|
|
|||
|
|
@ -51,6 +51,7 @@ type Config struct {
|
|||
InsecureSkipVerify bool
|
||||
CACertPath string
|
||||
ServerFingerprint string
|
||||
CustomSensorsFile string
|
||||
Observers []ObserverTarget
|
||||
RunOnce bool
|
||||
LogLevel zerolog.Level
|
||||
|
|
@ -103,6 +104,7 @@ type Config struct {
|
|||
updatedFromVersionFn func() string
|
||||
packageUpdates *packageUpdateManager
|
||||
storageCleanup *storageCleanupManager
|
||||
customSensorExecute customSensorExecution
|
||||
}
|
||||
|
||||
// ObserverTarget is a report-only Pulse destination. It can receive the same
|
||||
|
|
@ -162,6 +164,7 @@ type Agent struct {
|
|||
runCommandClient func(*CommandClient, context.Context) error
|
||||
packageUpdates *packageUpdateManager
|
||||
storageCleanup *storageCleanupManager
|
||||
customSensors *customSensorRuntime
|
||||
availability *availabilityProbeModule
|
||||
reportStreamID string
|
||||
reportSequence atomic.Uint64
|
||||
|
|
@ -318,6 +321,10 @@ func New(cfg Config) (*Agent, error) {
|
|||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
customSensorDefinitions, err := loadCustomSensorDefinitions(cfg.CustomSensorsFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load custom sensors: %w", err)
|
||||
}
|
||||
agenttarget.MarkConfigured("host", "primary", "primary")
|
||||
for _, observer := range observerReporters {
|
||||
agenttarget.MarkConfigured("host", observer.target.Name, "observer")
|
||||
|
|
@ -394,9 +401,16 @@ func New(cfg Config) (*Agent, error) {
|
|||
runCommandClient: runCommandClientFn,
|
||||
packageUpdates: packageUpdates,
|
||||
storageCleanup: storageCleanup,
|
||||
customSensors: newCustomSensorRuntime(customSensorDefinitions, cfg.customSensorExecute),
|
||||
availability: newAvailabilityProbeModule(logger, cfg.AvailabilityTargets),
|
||||
reportStreamID: reportStreamID,
|
||||
}
|
||||
if len(customSensorDefinitions) > 0 {
|
||||
logger.Info().
|
||||
Int("sensorCount", len(customSensorDefinitions)).
|
||||
Str("configPath", cfg.CustomSensorsFile).
|
||||
Msg("Custom sensor execution enabled from local configuration")
|
||||
}
|
||||
|
||||
// Create command client for AI command execution (only if enabled)
|
||||
if cfg.EnableCommands {
|
||||
|
|
@ -1088,6 +1102,7 @@ func (a *Agent) buildReport(ctx context.Context) (agentshost.Report, error) {
|
|||
|
||||
// Collect temperature data (best effort - don't fail if unavailable)
|
||||
sensorData := a.collectTemperatures(collectCtx)
|
||||
sensorData.Custom = a.customSensors.collect(collectCtx)
|
||||
|
||||
// Collect RAID array data (best effort - don't fail if unavailable)
|
||||
raidData := a.collectRAIDArrays(collectCtx)
|
||||
|
|
|
|||
21
internal/hostagent/custom_sensor_security_unix.go
Normal file
21
internal/hostagent/custom_sensor_security_unix.go
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
//go:build !windows
|
||||
|
||||
package hostagent
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
func validateCustomSensorFileOwner(info os.FileInfo, label string) error {
|
||||
stat, ok := info.Sys().(*syscall.Stat_t)
|
||||
if !ok {
|
||||
return fmt.Errorf("%s ownership could not be verified", label)
|
||||
}
|
||||
effectiveUID := uint32(os.Geteuid())
|
||||
if stat.Uid != effectiveUID {
|
||||
return fmt.Errorf("%s must be owned by the agent service user (uid %d)", label, effectiveUID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
9
internal/hostagent/custom_sensor_security_windows.go
Normal file
9
internal/hostagent/custom_sensor_security_windows.go
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
//go:build windows
|
||||
|
||||
package hostagent
|
||||
|
||||
import "os"
|
||||
|
||||
func validateCustomSensorFileOwner(_ os.FileInfo, _ string) error {
|
||||
return nil
|
||||
}
|
||||
537
internal/hostagent/custom_sensors.go
Normal file
537
internal/hostagent/custom_sensors.go
Normal file
|
|
@ -0,0 +1,537 @@
|
|||
package hostagent
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
"unicode"
|
||||
|
||||
agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
const (
|
||||
customSensorConfigVersion = 1
|
||||
maxCustomSensorConfigBytes = 64 * 1024
|
||||
maxCustomSensorOutputBytes = 4 * 1024
|
||||
maxCustomSensorErrorBytes = 256
|
||||
maxCustomSensors = 32
|
||||
maxConcurrentCustomSensors = 4
|
||||
defaultCustomSensorInterval = 5 * time.Minute
|
||||
defaultCustomSensorTimeout = 3 * time.Second
|
||||
minCustomSensorInterval = 10 * time.Second
|
||||
maxCustomSensorInterval = 24 * time.Hour
|
||||
minCustomSensorTimeout = 100 * time.Millisecond
|
||||
maxCustomSensorTimeout = 10 * time.Second
|
||||
)
|
||||
|
||||
var (
|
||||
customSensorIDPattern = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,63}$`)
|
||||
customSensorUnitPattern = regexp.MustCompile(`^[A-Za-z0-9%°/_(). -]{0,24}$`)
|
||||
)
|
||||
|
||||
type customSensorFile struct {
|
||||
Version int `yaml:"version"`
|
||||
Sensors []customSensorFileEntry `yaml:"sensors"`
|
||||
}
|
||||
|
||||
type customSensorFileEntry struct {
|
||||
ID string `yaml:"id"`
|
||||
Name string `yaml:"name"`
|
||||
Command string `yaml:"command"`
|
||||
Unit string `yaml:"unit,omitempty"`
|
||||
Interval string `yaml:"interval,omitempty"`
|
||||
Timeout string `yaml:"timeout,omitempty"`
|
||||
WarningAbove *float64 `yaml:"warningAbove,omitempty"`
|
||||
CriticalAbove *float64 `yaml:"criticalAbove,omitempty"`
|
||||
WarningBelow *float64 `yaml:"warningBelow,omitempty"`
|
||||
CriticalBelow *float64 `yaml:"criticalBelow,omitempty"`
|
||||
AlertOnError *bool `yaml:"alertOnError,omitempty"`
|
||||
}
|
||||
|
||||
type customSensorDefinition struct {
|
||||
ID string
|
||||
Name string
|
||||
Command string
|
||||
Unit string
|
||||
Interval time.Duration
|
||||
Timeout time.Duration
|
||||
WarningAbove *float64
|
||||
CriticalAbove *float64
|
||||
WarningBelow *float64
|
||||
CriticalBelow *float64
|
||||
AlertOnError bool
|
||||
}
|
||||
|
||||
type customSensorExecution func(context.Context, string) (string, error)
|
||||
|
||||
type customSensorRuntimeState struct {
|
||||
reading agentshost.CustomSensorMetric
|
||||
nextRun time.Time
|
||||
running bool
|
||||
}
|
||||
|
||||
type customSensorRuntime struct {
|
||||
mu sync.Mutex
|
||||
definitions []customSensorDefinition
|
||||
states map[string]customSensorRuntimeState
|
||||
execute customSensorExecution
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func loadCustomSensorDefinitions(path string) ([]customSensorDefinition, error) {
|
||||
path = strings.TrimSpace(path)
|
||||
if path == "" {
|
||||
return nil, nil
|
||||
}
|
||||
if !filepath.IsAbs(path) {
|
||||
return nil, errors.New("custom sensor config path must be absolute")
|
||||
}
|
||||
if err := validateCustomSensorConfigFile(path); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open custom sensor config: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
decoder := yaml.NewDecoder(io.LimitReader(file, maxCustomSensorConfigBytes+1))
|
||||
decoder.KnownFields(true)
|
||||
var parsed customSensorFile
|
||||
if err := decoder.Decode(&parsed); err != nil {
|
||||
return nil, fmt.Errorf("decode custom sensor config: %w", err)
|
||||
}
|
||||
var trailing any
|
||||
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
|
||||
if err == nil {
|
||||
return nil, errors.New("custom sensor config must contain exactly one YAML document")
|
||||
}
|
||||
return nil, fmt.Errorf("decode custom sensor config trailer: %w", err)
|
||||
}
|
||||
if parsed.Version != customSensorConfigVersion {
|
||||
return nil, fmt.Errorf("custom sensor config version %d is unsupported; expected %d", parsed.Version, customSensorConfigVersion)
|
||||
}
|
||||
if len(parsed.Sensors) == 0 {
|
||||
return nil, errors.New("custom sensor config must define at least one sensor")
|
||||
}
|
||||
if len(parsed.Sensors) > maxCustomSensors {
|
||||
return nil, fmt.Errorf("custom sensor config has %d sensors; maximum is %d", len(parsed.Sensors), maxCustomSensors)
|
||||
}
|
||||
|
||||
definitions := make([]customSensorDefinition, 0, len(parsed.Sensors))
|
||||
seen := make(map[string]struct{}, len(parsed.Sensors))
|
||||
for index, entry := range parsed.Sensors {
|
||||
definition, err := resolveCustomSensorDefinition(entry)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("custom sensor %d: %w", index+1, err)
|
||||
}
|
||||
if _, exists := seen[definition.ID]; exists {
|
||||
return nil, fmt.Errorf("custom sensor %d: duplicate id %q", index+1, definition.ID)
|
||||
}
|
||||
seen[definition.ID] = struct{}{}
|
||||
definitions = append(definitions, definition)
|
||||
}
|
||||
return definitions, nil
|
||||
}
|
||||
|
||||
func resolveCustomSensorDefinition(entry customSensorFileEntry) (customSensorDefinition, error) {
|
||||
id := strings.TrimSpace(entry.ID)
|
||||
if !customSensorIDPattern.MatchString(id) {
|
||||
return customSensorDefinition{}, errors.New("id must start with a lowercase letter and contain only lowercase letters, digits, underscores, or hyphens (maximum 64 characters)")
|
||||
}
|
||||
name := strings.TrimSpace(entry.Name)
|
||||
if name == "" || len(name) > 100 || strings.IndexFunc(name, func(character rune) bool {
|
||||
return !unicode.IsPrint(character)
|
||||
}) >= 0 {
|
||||
return customSensorDefinition{}, errors.New("name must be 1-100 printable characters on one line")
|
||||
}
|
||||
unit := strings.TrimSpace(entry.Unit)
|
||||
if !customSensorUnitPattern.MatchString(unit) {
|
||||
return customSensorDefinition{}, errors.New("unit contains unsupported characters or exceeds 24 characters")
|
||||
}
|
||||
|
||||
command := filepath.Clean(strings.TrimSpace(entry.Command))
|
||||
if !filepath.IsAbs(command) {
|
||||
return customSensorDefinition{}, errors.New("command must be an absolute path")
|
||||
}
|
||||
if err := validateCustomSensorCommandFile(command); err != nil {
|
||||
return customSensorDefinition{}, err
|
||||
}
|
||||
|
||||
interval, err := parseCustomSensorDuration(entry.Interval, defaultCustomSensorInterval, minCustomSensorInterval, maxCustomSensorInterval, "interval")
|
||||
if err != nil {
|
||||
return customSensorDefinition{}, err
|
||||
}
|
||||
timeout, err := parseCustomSensorDuration(entry.Timeout, defaultCustomSensorTimeout, minCustomSensorTimeout, maxCustomSensorTimeout, "timeout")
|
||||
if err != nil {
|
||||
return customSensorDefinition{}, err
|
||||
}
|
||||
if timeout >= interval {
|
||||
return customSensorDefinition{}, errors.New("timeout must be shorter than interval")
|
||||
}
|
||||
for label, value := range map[string]*float64{
|
||||
"warningAbove": entry.WarningAbove,
|
||||
"criticalAbove": entry.CriticalAbove,
|
||||
"warningBelow": entry.WarningBelow,
|
||||
"criticalBelow": entry.CriticalBelow,
|
||||
} {
|
||||
if value != nil && (math.IsNaN(*value) || math.IsInf(*value, 0)) {
|
||||
return customSensorDefinition{}, fmt.Errorf("%s must be finite", label)
|
||||
}
|
||||
}
|
||||
if entry.WarningAbove != nil && entry.CriticalAbove != nil && *entry.WarningAbove > *entry.CriticalAbove {
|
||||
return customSensorDefinition{}, errors.New("warningAbove must be less than or equal to criticalAbove")
|
||||
}
|
||||
if entry.WarningBelow != nil && entry.CriticalBelow != nil && *entry.WarningBelow < *entry.CriticalBelow {
|
||||
return customSensorDefinition{}, errors.New("warningBelow must be greater than or equal to criticalBelow")
|
||||
}
|
||||
lowerBoundary := entry.WarningBelow
|
||||
if lowerBoundary == nil {
|
||||
lowerBoundary = entry.CriticalBelow
|
||||
}
|
||||
upperBoundary := entry.WarningAbove
|
||||
if upperBoundary == nil {
|
||||
upperBoundary = entry.CriticalAbove
|
||||
}
|
||||
if lowerBoundary != nil && upperBoundary != nil && *lowerBoundary >= *upperBoundary {
|
||||
return customSensorDefinition{}, errors.New("lower thresholds must be less than upper thresholds")
|
||||
}
|
||||
|
||||
alertOnError := true
|
||||
if entry.AlertOnError != nil {
|
||||
alertOnError = *entry.AlertOnError
|
||||
}
|
||||
return customSensorDefinition{
|
||||
ID: id,
|
||||
Name: name,
|
||||
Command: command,
|
||||
Unit: unit,
|
||||
Interval: interval,
|
||||
Timeout: timeout,
|
||||
WarningAbove: cloneFloat64(entry.WarningAbove),
|
||||
CriticalAbove: cloneFloat64(entry.CriticalAbove),
|
||||
WarningBelow: cloneFloat64(entry.WarningBelow),
|
||||
CriticalBelow: cloneFloat64(entry.CriticalBelow),
|
||||
AlertOnError: alertOnError,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func parseCustomSensorDuration(raw string, fallback, minimum, maximum time.Duration, label string) (time.Duration, error) {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return fallback, nil
|
||||
}
|
||||
parsed, err := time.ParseDuration(raw)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("%s is invalid: %w", label, err)
|
||||
}
|
||||
if parsed < minimum || parsed > maximum {
|
||||
return 0, fmt.Errorf("%s must be between %s and %s", label, minimum, maximum)
|
||||
}
|
||||
return parsed, nil
|
||||
}
|
||||
|
||||
func validateCustomSensorConfigFile(path string) error {
|
||||
info, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("stat custom sensor config: %w", err)
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
||||
return errors.New("custom sensor config must be a regular file, not a symlink")
|
||||
}
|
||||
if info.Size() > maxCustomSensorConfigBytes {
|
||||
return fmt.Errorf("custom sensor config exceeds %d bytes", maxCustomSensorConfigBytes)
|
||||
}
|
||||
if runtime.GOOS != "windows" && info.Mode().Perm()&0o077 != 0 {
|
||||
return errors.New("custom sensor config permissions must not grant group or other access")
|
||||
}
|
||||
if err := validateCustomSensorFileOwner(info, "custom sensor config"); err != nil {
|
||||
return err
|
||||
}
|
||||
return validateCustomSensorParentDirectory(filepath.Dir(path), "custom sensor config")
|
||||
}
|
||||
|
||||
func validateCustomSensorCommandFile(path string) error {
|
||||
info, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("stat custom sensor command: %w", err)
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
||||
return errors.New("custom sensor command must be a regular file, not a symlink")
|
||||
}
|
||||
if runtime.GOOS != "windows" {
|
||||
if info.Mode().Perm()&0o111 == 0 {
|
||||
return errors.New("custom sensor command must be executable")
|
||||
}
|
||||
if info.Mode().Perm()&0o022 != 0 {
|
||||
return errors.New("custom sensor command must not be writable by group or others")
|
||||
}
|
||||
}
|
||||
if err := validateCustomSensorFileOwner(info, "custom sensor command"); err != nil {
|
||||
return err
|
||||
}
|
||||
return validateCustomSensorParentDirectory(filepath.Dir(path), "custom sensor command")
|
||||
}
|
||||
|
||||
func validateCustomSensorParentDirectory(path, label string) error {
|
||||
info, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("stat %s parent directory: %w", label, err)
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
return fmt.Errorf("%s parent directory must not be a symlink", label)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return fmt.Errorf("%s parent path is not a directory", label)
|
||||
}
|
||||
if runtime.GOOS != "windows" && info.Mode().Perm()&0o022 != 0 {
|
||||
return fmt.Errorf("%s parent directory must not be writable by group or others", label)
|
||||
}
|
||||
return validateCustomSensorFileOwner(info, label+" parent directory")
|
||||
}
|
||||
|
||||
func newCustomSensorRuntime(definitions []customSensorDefinition, execute customSensorExecution) *customSensorRuntime {
|
||||
if len(definitions) == 0 {
|
||||
return nil
|
||||
}
|
||||
if execute == nil {
|
||||
execute = executeCustomSensorCommand
|
||||
}
|
||||
return &customSensorRuntime{
|
||||
definitions: append([]customSensorDefinition(nil), definitions...),
|
||||
states: make(map[string]customSensorRuntimeState, len(definitions)),
|
||||
execute: execute,
|
||||
now: time.Now,
|
||||
}
|
||||
}
|
||||
|
||||
func (runtimeState *customSensorRuntime) collect(ctx context.Context) []agentshost.CustomSensorMetric {
|
||||
if runtimeState == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
now := runtimeState.now()
|
||||
type dueSensor struct {
|
||||
definition customSensorDefinition
|
||||
previous *agentshost.CustomSensorMetric
|
||||
}
|
||||
due := make([]dueSensor, 0, len(runtimeState.definitions))
|
||||
|
||||
runtimeState.mu.Lock()
|
||||
for _, definition := range runtimeState.definitions {
|
||||
state := runtimeState.states[definition.ID]
|
||||
if !state.running && (state.nextRun.IsZero() || !now.Before(state.nextRun)) {
|
||||
state.running = true
|
||||
state.nextRun = now.Add(definition.Interval)
|
||||
runtimeState.states[definition.ID] = state
|
||||
var previous *agentshost.CustomSensorMetric
|
||||
if state.reading.ID != "" {
|
||||
cloned := cloneCustomSensorMetric(state.reading)
|
||||
previous = &cloned
|
||||
}
|
||||
due = append(due, dueSensor{definition: definition, previous: previous})
|
||||
}
|
||||
}
|
||||
runtimeState.mu.Unlock()
|
||||
|
||||
type executionResult struct {
|
||||
definition customSensorDefinition
|
||||
reading agentshost.CustomSensorMetric
|
||||
}
|
||||
results := make(chan executionResult, len(due))
|
||||
semaphore := make(chan struct{}, maxConcurrentCustomSensors)
|
||||
var wait sync.WaitGroup
|
||||
for _, item := range due {
|
||||
wait.Add(1)
|
||||
go func(definition customSensorDefinition, previous *agentshost.CustomSensorMetric) {
|
||||
defer wait.Done()
|
||||
select {
|
||||
case semaphore <- struct{}{}:
|
||||
defer func() { <-semaphore }()
|
||||
case <-ctx.Done():
|
||||
results <- executionResult{definition: definition, reading: customSensorErrorReading(definition, previous, now, ctx.Err())}
|
||||
return
|
||||
}
|
||||
results <- executionResult{
|
||||
definition: definition,
|
||||
reading: runtimeState.executeOne(ctx, definition, previous, now),
|
||||
}
|
||||
}(item.definition, item.previous)
|
||||
}
|
||||
wait.Wait()
|
||||
close(results)
|
||||
|
||||
runtimeState.mu.Lock()
|
||||
for result := range results {
|
||||
state := runtimeState.states[result.definition.ID]
|
||||
state.reading = result.reading
|
||||
state.running = false
|
||||
runtimeState.states[result.definition.ID] = state
|
||||
}
|
||||
readings := make([]agentshost.CustomSensorMetric, 0, len(runtimeState.definitions))
|
||||
for _, definition := range runtimeState.definitions {
|
||||
state := runtimeState.states[definition.ID]
|
||||
if state.reading.ID != "" {
|
||||
readings = append(readings, cloneCustomSensorMetric(state.reading))
|
||||
}
|
||||
}
|
||||
runtimeState.mu.Unlock()
|
||||
return readings
|
||||
}
|
||||
|
||||
func (runtimeState *customSensorRuntime) executeOne(parent context.Context, definition customSensorDefinition, previous *agentshost.CustomSensorMetric, observedAt time.Time) agentshost.CustomSensorMetric {
|
||||
if err := validateCustomSensorCommandFile(definition.Command); err != nil {
|
||||
return customSensorErrorReading(definition, previous, observedAt, errors.New("custom sensor command failed security validation"))
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(parent, definition.Timeout)
|
||||
defer cancel()
|
||||
|
||||
output, err := runtimeState.execute(ctx, definition.Command)
|
||||
if err != nil {
|
||||
return customSensorErrorReading(definition, previous, observedAt, err)
|
||||
}
|
||||
value, err := parseCustomSensorValue(output)
|
||||
if err != nil {
|
||||
return customSensorErrorReading(definition, previous, observedAt, err)
|
||||
}
|
||||
status := customSensorStatus(definition, value)
|
||||
return agentshost.CustomSensorMetric{
|
||||
ID: definition.ID,
|
||||
Name: definition.Name,
|
||||
Unit: definition.Unit,
|
||||
Value: &value,
|
||||
Status: status,
|
||||
ObservedAt: observedAt,
|
||||
AlertOnError: definition.AlertOnError,
|
||||
}
|
||||
}
|
||||
|
||||
func customSensorErrorReading(definition customSensorDefinition, previous *agentshost.CustomSensorMetric, observedAt time.Time, err error) agentshost.CustomSensorMetric {
|
||||
reading := agentshost.CustomSensorMetric{
|
||||
ID: definition.ID,
|
||||
Name: definition.Name,
|
||||
Unit: definition.Unit,
|
||||
Status: agentshost.CustomSensorStatusError,
|
||||
ObservedAt: observedAt,
|
||||
AlertOnError: definition.AlertOnError,
|
||||
Error: boundedCustomSensorError(err),
|
||||
}
|
||||
if previous != nil && previous.Value != nil {
|
||||
value := *previous.Value
|
||||
reading.Value = &value
|
||||
reading.ObservedAt = previous.ObservedAt
|
||||
reading.Stale = true
|
||||
}
|
||||
return reading
|
||||
}
|
||||
|
||||
func parseCustomSensorValue(output string) (float64, error) {
|
||||
output = strings.TrimSpace(output)
|
||||
if output == "" {
|
||||
return 0, errors.New("custom sensor returned empty output")
|
||||
}
|
||||
value, err := strconv.ParseFloat(output, 64)
|
||||
if err != nil || math.IsNaN(value) || math.IsInf(value, 0) {
|
||||
return 0, errors.New("custom sensor output must be exactly one finite number")
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func customSensorStatus(definition customSensorDefinition, value float64) string {
|
||||
if definition.CriticalAbove != nil && value >= *definition.CriticalAbove {
|
||||
return agentshost.CustomSensorStatusCritical
|
||||
}
|
||||
if definition.CriticalBelow != nil && value <= *definition.CriticalBelow {
|
||||
return agentshost.CustomSensorStatusCritical
|
||||
}
|
||||
if definition.WarningAbove != nil && value >= *definition.WarningAbove {
|
||||
return agentshost.CustomSensorStatusWarning
|
||||
}
|
||||
if definition.WarningBelow != nil && value <= *definition.WarningBelow {
|
||||
return agentshost.CustomSensorStatusWarning
|
||||
}
|
||||
return agentshost.CustomSensorStatusOK
|
||||
}
|
||||
|
||||
func executeCustomSensorCommand(ctx context.Context, path string) (string, error) {
|
||||
var stdout boundedSensorBuffer
|
||||
stdout.limit = maxCustomSensorOutputBytes
|
||||
|
||||
command := exec.CommandContext(ctx, path)
|
||||
command.Stdout = &stdout
|
||||
command.WaitDelay = 500 * time.Millisecond
|
||||
err := command.Run()
|
||||
if stdout.truncated {
|
||||
return "", fmt.Errorf("custom sensor output exceeded %d bytes", maxCustomSensorOutputBytes)
|
||||
}
|
||||
if err != nil {
|
||||
if contextError := ctx.Err(); contextError != nil {
|
||||
return "", fmt.Errorf("custom sensor command: %w", contextError)
|
||||
}
|
||||
return "", fmt.Errorf("custom sensor command failed: %w", err)
|
||||
}
|
||||
return stdout.String(), nil
|
||||
}
|
||||
|
||||
type boundedSensorBuffer struct {
|
||||
buffer bytes.Buffer
|
||||
limit int
|
||||
truncated bool
|
||||
}
|
||||
|
||||
func (buffer *boundedSensorBuffer) Write(data []byte) (int, error) {
|
||||
originalLength := len(data)
|
||||
remaining := buffer.limit - buffer.buffer.Len()
|
||||
if remaining <= 0 {
|
||||
buffer.truncated = true
|
||||
return originalLength, nil
|
||||
}
|
||||
if len(data) > remaining {
|
||||
data = data[:remaining]
|
||||
buffer.truncated = true
|
||||
}
|
||||
_, _ = buffer.buffer.Write(data)
|
||||
return originalLength, nil
|
||||
}
|
||||
|
||||
func (buffer *boundedSensorBuffer) String() string {
|
||||
return buffer.buffer.String()
|
||||
}
|
||||
|
||||
func boundedCustomSensorError(err error) string {
|
||||
if err == nil {
|
||||
return ""
|
||||
}
|
||||
message := strings.TrimSpace(strings.ReplaceAll(err.Error(), "\x00", ""))
|
||||
if len(message) > maxCustomSensorErrorBytes {
|
||||
message = message[:maxCustomSensorErrorBytes]
|
||||
}
|
||||
return message
|
||||
}
|
||||
|
||||
func cloneFloat64(value *float64) *float64 {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
cloned := *value
|
||||
return &cloned
|
||||
}
|
||||
|
||||
func cloneCustomSensorMetric(metric agentshost.CustomSensorMetric) agentshost.CustomSensorMetric {
|
||||
metric.Value = cloneFloat64(metric.Value)
|
||||
return metric
|
||||
}
|
||||
293
internal/hostagent/custom_sensors_test.go
Normal file
293
internal/hostagent/custom_sensors_test.go
Normal file
|
|
@ -0,0 +1,293 @@
|
|||
package hostagent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host"
|
||||
)
|
||||
|
||||
func writeCustomSensorFixture(t *testing.T, configBody string) (string, string) {
|
||||
t.Helper()
|
||||
directory := t.TempDir()
|
||||
commandPath := filepath.Join(directory, "sensor-command")
|
||||
if runtime.GOOS == "windows" {
|
||||
commandPath += ".exe"
|
||||
}
|
||||
if err := os.WriteFile(commandPath, []byte("#!/bin/sh\nprintf '1\\n'\n"), 0o700); err != nil {
|
||||
t.Fatalf("write command fixture: %v", err)
|
||||
}
|
||||
configPath := filepath.Join(directory, "custom-sensors.yaml")
|
||||
configBody = strings.ReplaceAll(configBody, "__COMMAND__", commandPath)
|
||||
if err := os.WriteFile(configPath, []byte(configBody), 0o600); err != nil {
|
||||
t.Fatalf("write config fixture: %v", err)
|
||||
}
|
||||
return configPath, commandPath
|
||||
}
|
||||
|
||||
func TestLoadCustomSensorDefinitionsValidatesAndNormalizes(t *testing.T) {
|
||||
configPath, commandPath := writeCustomSensorFixture(t, `version: 1
|
||||
sensors:
|
||||
- id: pending_updates
|
||||
name: Pending updates
|
||||
command: __COMMAND__
|
||||
unit: count
|
||||
interval: 10m
|
||||
timeout: 2s
|
||||
warningAbove: 5
|
||||
criticalAbove: 20
|
||||
`)
|
||||
|
||||
definitions, err := loadCustomSensorDefinitions(configPath)
|
||||
if err != nil {
|
||||
t.Fatalf("loadCustomSensorDefinitions: %v", err)
|
||||
}
|
||||
if len(definitions) != 1 {
|
||||
t.Fatalf("definition count = %d, want 1", len(definitions))
|
||||
}
|
||||
definition := definitions[0]
|
||||
if definition.ID != "pending_updates" || definition.Name != "Pending updates" || definition.Command != commandPath {
|
||||
t.Fatalf("unexpected definition: %+v", definition)
|
||||
}
|
||||
if definition.Interval != 10*time.Minute || definition.Timeout != 2*time.Second {
|
||||
t.Fatalf("unexpected timing: interval=%s timeout=%s", definition.Interval, definition.Timeout)
|
||||
}
|
||||
if definition.WarningAbove == nil || *definition.WarningAbove != 5 || definition.CriticalAbove == nil || *definition.CriticalAbove != 20 {
|
||||
t.Fatalf("unexpected thresholds: %+v", definition)
|
||||
}
|
||||
if !definition.AlertOnError {
|
||||
t.Fatal("AlertOnError should default to true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadCustomSensorDefinitionsRejectsUnsafeAndAmbiguousConfiguration(t *testing.T) {
|
||||
t.Run("unknown field", func(t *testing.T) {
|
||||
configPath, _ := writeCustomSensorFixture(t, `version: 1
|
||||
sensors:
|
||||
- id: metric
|
||||
name: Metric
|
||||
command: __COMMAND__
|
||||
shell: true
|
||||
`)
|
||||
if _, err := loadCustomSensorDefinitions(configPath); err == nil || !strings.Contains(err.Error(), "field shell not found") {
|
||||
t.Fatalf("expected unknown field rejection, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("relative command", func(t *testing.T) {
|
||||
configPath, _ := writeCustomSensorFixture(t, `version: 1
|
||||
sensors:
|
||||
- id: metric
|
||||
name: Metric
|
||||
command: ./sensor
|
||||
`)
|
||||
if _, err := loadCustomSensorDefinitions(configPath); err == nil || !strings.Contains(err.Error(), "absolute path") {
|
||||
t.Fatalf("expected relative command rejection, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("control character in name", func(t *testing.T) {
|
||||
_, err := resolveCustomSensorDefinition(customSensorFileEntry{
|
||||
ID: "metric",
|
||||
Name: "unsafe\x01name",
|
||||
Command: commandPathForTest(t),
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "printable") {
|
||||
t.Fatalf("expected control character rejection, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("threshold ordering", func(t *testing.T) {
|
||||
configPath, _ := writeCustomSensorFixture(t, `version: 1
|
||||
sensors:
|
||||
- id: metric
|
||||
name: Metric
|
||||
command: __COMMAND__
|
||||
warningAbove: 20
|
||||
criticalAbove: 10
|
||||
`)
|
||||
if _, err := loadCustomSensorDefinitions(configPath); err == nil || !strings.Contains(err.Error(), "warningAbove") {
|
||||
t.Fatalf("expected threshold ordering rejection, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("overlapping lower and upper thresholds", func(t *testing.T) {
|
||||
configPath, _ := writeCustomSensorFixture(t, `version: 1
|
||||
sensors:
|
||||
- id: metric
|
||||
name: Metric
|
||||
command: __COMMAND__
|
||||
warningBelow: 20
|
||||
warningAbove: 10
|
||||
`)
|
||||
if _, err := loadCustomSensorDefinitions(configPath); err == nil || !strings.Contains(err.Error(), "lower thresholds") {
|
||||
t.Fatalf("expected overlapping threshold rejection, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
if runtime.GOOS != "windows" {
|
||||
t.Run("loose config permissions", func(t *testing.T) {
|
||||
configPath, _ := writeCustomSensorFixture(t, `version: 1
|
||||
sensors:
|
||||
- id: metric
|
||||
name: Metric
|
||||
command: __COMMAND__
|
||||
`)
|
||||
if err := os.Chmod(configPath, 0o644); err != nil {
|
||||
t.Fatalf("chmod config: %v", err)
|
||||
}
|
||||
if _, err := loadCustomSensorDefinitions(configPath); err == nil || !strings.Contains(err.Error(), "permissions") {
|
||||
t.Fatalf("expected loose permission rejection, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("symlinked config parent", func(t *testing.T) {
|
||||
_, commandPath := writeCustomSensorFixture(t, `version: 1
|
||||
sensors:
|
||||
- id: metric
|
||||
name: Metric
|
||||
command: __COMMAND__
|
||||
`)
|
||||
root := t.TempDir()
|
||||
realDirectory := filepath.Join(root, "real")
|
||||
if err := os.Mkdir(realDirectory, 0o700); err != nil {
|
||||
t.Fatalf("mkdir real config parent: %v", err)
|
||||
}
|
||||
linkDirectory := filepath.Join(root, "linked")
|
||||
if err := os.Symlink(realDirectory, linkDirectory); err != nil {
|
||||
t.Fatalf("symlink config parent: %v", err)
|
||||
}
|
||||
configPath := filepath.Join(linkDirectory, "custom-sensors.yaml")
|
||||
configBody := "version: 1\nsensors:\n - id: metric\n name: Metric\n command: " + commandPath + "\n"
|
||||
if err := os.WriteFile(configPath, []byte(configBody), 0o600); err != nil {
|
||||
t.Fatalf("write config: %v", err)
|
||||
}
|
||||
if _, err := loadCustomSensorDefinitions(configPath); err == nil || !strings.Contains(err.Error(), "must not be a symlink") {
|
||||
t.Fatalf("expected symlinked parent rejection, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func commandPathForTest(t *testing.T) string {
|
||||
t.Helper()
|
||||
_, commandPath := writeCustomSensorFixture(t, `version: 1
|
||||
sensors:
|
||||
- id: metric
|
||||
name: Metric
|
||||
command: __COMMAND__
|
||||
`)
|
||||
return commandPath
|
||||
}
|
||||
|
||||
func TestCustomSensorRuntimeCachesAndReportsThresholdAndErrorState(t *testing.T) {
|
||||
_, commandPath := writeCustomSensorFixture(t, `version: 1
|
||||
sensors:
|
||||
- id: queue_depth
|
||||
name: Queue depth
|
||||
command: __COMMAND__
|
||||
`)
|
||||
warning := 5.0
|
||||
critical := 20.0
|
||||
definition := customSensorDefinition{
|
||||
ID: "queue_depth",
|
||||
Name: "Queue depth",
|
||||
Command: commandPath,
|
||||
Unit: "count",
|
||||
Interval: time.Minute,
|
||||
Timeout: time.Second,
|
||||
WarningAbove: &warning,
|
||||
CriticalAbove: &critical,
|
||||
AlertOnError: true,
|
||||
}
|
||||
|
||||
var calls atomic.Int32
|
||||
execute := func(context.Context, string) (string, error) {
|
||||
switch calls.Add(1) {
|
||||
case 1:
|
||||
return "7\n", nil
|
||||
default:
|
||||
return "", errors.New("probe failed with secret detail that remains bounded")
|
||||
}
|
||||
}
|
||||
current := time.Date(2026, 7, 30, 19, 0, 0, 0, time.UTC)
|
||||
customRuntime := newCustomSensorRuntime([]customSensorDefinition{definition}, execute)
|
||||
customRuntime.now = func() time.Time { return current }
|
||||
|
||||
first := customRuntime.collect(context.Background())
|
||||
if len(first) != 1 || first[0].Value == nil || *first[0].Value != 7 {
|
||||
t.Fatalf("unexpected first reading: %+v", first)
|
||||
}
|
||||
if first[0].Status != agentshost.CustomSensorStatusWarning || first[0].Stale {
|
||||
t.Fatalf("unexpected first status: %+v", first[0])
|
||||
}
|
||||
|
||||
cached := customRuntime.collect(context.Background())
|
||||
if calls.Load() != 1 || len(cached) != 1 || cached[0].Value == nil || *cached[0].Value != 7 {
|
||||
t.Fatalf("cached read should not execute again: calls=%d reading=%+v", calls.Load(), cached)
|
||||
}
|
||||
|
||||
current = current.Add(time.Minute)
|
||||
failed := customRuntime.collect(context.Background())
|
||||
if calls.Load() != 2 || len(failed) != 1 {
|
||||
t.Fatalf("expected second execution: calls=%d reading=%+v", calls.Load(), failed)
|
||||
}
|
||||
if failed[0].Status != agentshost.CustomSensorStatusError || !failed[0].Stale || failed[0].Value == nil || *failed[0].Value != 7 {
|
||||
t.Fatalf("expected stale last-good value with error state: %+v", failed[0])
|
||||
}
|
||||
if !failed[0].ObservedAt.Equal(first[0].ObservedAt) {
|
||||
t.Fatalf("stale reading observedAt = %s, want last-good time %s", failed[0].ObservedAt, first[0].ObservedAt)
|
||||
}
|
||||
if !strings.Contains(failed[0].Error, "probe failed") {
|
||||
t.Fatalf("expected bounded execution error, got %q", failed[0].Error)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCustomSensorValueAndStatus(t *testing.T) {
|
||||
for _, invalid := range []string{"", "NaN", "+Inf", "1 2", "healthy"} {
|
||||
if _, err := parseCustomSensorValue(invalid); err == nil {
|
||||
t.Fatalf("parseCustomSensorValue(%q) unexpectedly succeeded", invalid)
|
||||
}
|
||||
}
|
||||
if value, err := parseCustomSensorValue(" 0.125\n"); err != nil || value != 0.125 {
|
||||
t.Fatalf("parseCustomSensorValue valid result = %v, %v", value, err)
|
||||
}
|
||||
|
||||
warningBelow := 10.0
|
||||
criticalBelow := 5.0
|
||||
definition := customSensorDefinition{WarningBelow: &warningBelow, CriticalBelow: &criticalBelow}
|
||||
if got := customSensorStatus(definition, 4); got != agentshost.CustomSensorStatusCritical {
|
||||
t.Fatalf("critical-below status = %q", got)
|
||||
}
|
||||
if got := customSensorStatus(definition, 8); got != agentshost.CustomSensorStatusWarning {
|
||||
t.Fatalf("warning-below status = %q", got)
|
||||
}
|
||||
if got := customSensorStatus(definition, 11); got != agentshost.CustomSensorStatusOK {
|
||||
t.Fatalf("healthy status = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteCustomSensorCommandDoesNotExposeStderr(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("shell fixture is POSIX-only")
|
||||
}
|
||||
commandPath := filepath.Join(t.TempDir(), "failing-sensor")
|
||||
if err := os.WriteFile(commandPath, []byte("#!/bin/sh\nprintf 'database-password' >&2\nexit 1\n"), 0o700); err != nil {
|
||||
t.Fatalf("write command: %v", err)
|
||||
}
|
||||
|
||||
_, err := executeCustomSensorCommand(context.Background(), commandPath)
|
||||
if err == nil {
|
||||
t.Fatal("expected command failure")
|
||||
}
|
||||
if strings.Contains(err.Error(), "database-password") {
|
||||
t.Fatalf("stderr escaped the local execution boundary: %q", err)
|
||||
}
|
||||
}
|
||||
|
|
@ -885,7 +885,7 @@ func (s DockerSwarmInfo) ToFrontend() DockerSwarmFrontend {
|
|||
}
|
||||
|
||||
func hostSensorSummaryToFrontend(src HostSensorSummary) *HostSensorSummaryFrontend {
|
||||
if len(src.TemperatureCelsius) == 0 && len(src.FanRPM) == 0 && len(src.PowerWatts) == 0 && len(src.Additional) == 0 && len(src.GPU) == 0 && src.ThermalState == nil && len(src.SMART) == 0 {
|
||||
if len(src.TemperatureCelsius) == 0 && len(src.FanRPM) == 0 && len(src.PowerWatts) == 0 && len(src.Additional) == 0 && len(src.Custom) == 0 && len(src.GPU) == 0 && src.ThermalState == nil && len(src.SMART) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -902,6 +902,22 @@ func hostSensorSummaryToFrontend(src HostSensorSummary) *HostSensorSummaryFronte
|
|||
if len(src.Additional) > 0 {
|
||||
dest.Additional = copyStringFloatMap(src.Additional)
|
||||
}
|
||||
if len(src.Custom) > 0 {
|
||||
dest.Custom = make([]HostCustomSensorMetricFrontend, len(src.Custom))
|
||||
for i, metric := range src.Custom {
|
||||
dest.Custom[i] = HostCustomSensorMetricFrontend{
|
||||
ID: metric.ID,
|
||||
Name: metric.Name,
|
||||
Unit: metric.Unit,
|
||||
Value: cloneFloat64Ptr(metric.Value),
|
||||
Status: metric.Status,
|
||||
ObservedAt: metric.ObservedAt,
|
||||
Error: metric.Error,
|
||||
AlertOnError: metric.AlertOnError,
|
||||
Stale: metric.Stale,
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(src.GPU) > 0 {
|
||||
dest.GPU = make([]HostGPUSensorFrontend, len(src.GPU))
|
||||
for i, gpu := range src.GPU {
|
||||
|
|
|
|||
|
|
@ -256,12 +256,25 @@ func cloneHostSensorSummary(src HostSensorSummary) HostSensorSummary {
|
|||
FanRPM: cloneStringFloat64Map(src.FanRPM),
|
||||
PowerWatts: cloneStringFloat64Map(src.PowerWatts),
|
||||
Additional: cloneStringFloat64Map(src.Additional),
|
||||
Custom: cloneHostCustomSensors(src.Custom),
|
||||
GPU: cloneHostGPUSensors(src.GPU),
|
||||
ThermalState: cloneHostThermalState(src.ThermalState),
|
||||
SMART: cloneHostDiskSMART(src.SMART),
|
||||
}.NormalizeCollections()
|
||||
}
|
||||
|
||||
func cloneHostCustomSensors(src []HostCustomSensorMetric) []HostCustomSensorMetric {
|
||||
if len(src) == 0 {
|
||||
return nil
|
||||
}
|
||||
dest := make([]HostCustomSensorMetric, len(src))
|
||||
for i, metric := range src {
|
||||
dest[i] = metric
|
||||
dest[i].Value = cloneFloat64Ptr(metric.Value)
|
||||
}
|
||||
return dest
|
||||
}
|
||||
|
||||
func cloneHostGPUSensors(src []HostGPUSensor) []HostGPUSensor {
|
||||
if len(src) == 0 {
|
||||
return nil
|
||||
|
|
|
|||
|
|
@ -492,11 +492,24 @@ type HostSensorSummary struct {
|
|||
FanRPM map[string]float64 `json:"fanRpm,omitempty"`
|
||||
PowerWatts map[string]float64 `json:"powerWatts,omitempty"`
|
||||
Additional map[string]float64 `json:"additional,omitempty"`
|
||||
Custom []HostCustomSensorMetric `json:"custom,omitempty"`
|
||||
GPU []HostGPUSensor `json:"gpu,omitempty"`
|
||||
ThermalState *HostThermalState `json:"thermalState,omitempty"`
|
||||
SMART []HostDiskSMART `json:"smart,omitempty"` // S.M.A.R.T. disk data
|
||||
}
|
||||
|
||||
type HostCustomSensorMetric struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Unit string `json:"unit,omitempty"`
|
||||
Value *float64 `json:"value,omitempty"`
|
||||
Status string `json:"status"`
|
||||
ObservedAt time.Time `json:"observedAt"`
|
||||
Error string `json:"error,omitempty"`
|
||||
AlertOnError bool `json:"alertOnError,omitempty"`
|
||||
Stale bool `json:"stale,omitempty"`
|
||||
}
|
||||
|
||||
type HostGPUSensor struct {
|
||||
ID string `json:"id,omitempty"`
|
||||
Name string `json:"name,omitempty"`
|
||||
|
|
@ -528,6 +541,9 @@ func (s HostSensorSummary) NormalizeCollections() HostSensorSummary {
|
|||
if s.Additional == nil {
|
||||
s.Additional = map[string]float64{}
|
||||
}
|
||||
if s.Custom == nil {
|
||||
s.Custom = []HostCustomSensorMetric{}
|
||||
}
|
||||
if s.GPU == nil {
|
||||
s.GPU = []HostGPUSensor{}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ package models
|
|||
|
||||
import (
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory"
|
||||
)
|
||||
|
|
@ -764,11 +765,24 @@ type HostSensorSummaryFrontend struct {
|
|||
FanRPM map[string]float64 `json:"fanRpm"`
|
||||
PowerWatts map[string]float64 `json:"powerWatts"`
|
||||
Additional map[string]float64 `json:"additional"`
|
||||
Custom []HostCustomSensorMetricFrontend `json:"custom"`
|
||||
GPU []HostGPUSensorFrontend `json:"gpu"`
|
||||
ThermalState *HostThermalState `json:"thermalState,omitempty"`
|
||||
SMART []HostDiskSMARTFrontend `json:"smart"` // S.M.A.R.T. disk data
|
||||
}
|
||||
|
||||
type HostCustomSensorMetricFrontend struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Unit string `json:"unit,omitempty"`
|
||||
Value *float64 `json:"value,omitempty"`
|
||||
Status string `json:"status"`
|
||||
ObservedAt time.Time `json:"observedAt"`
|
||||
Error string `json:"error,omitempty"`
|
||||
AlertOnError bool `json:"alertOnError,omitempty"`
|
||||
Stale bool `json:"stale,omitempty"`
|
||||
}
|
||||
|
||||
type HostGPUSensorFrontend struct {
|
||||
ID string `json:"id,omitempty"`
|
||||
Name string `json:"name,omitempty"`
|
||||
|
|
@ -791,6 +805,9 @@ func (s HostSensorSummaryFrontend) NormalizeCollections() HostSensorSummaryFront
|
|||
if s.Additional == nil {
|
||||
s.Additional = map[string]float64{}
|
||||
}
|
||||
if s.Custom == nil {
|
||||
s.Custom = []HostCustomSensorMetricFrontend{}
|
||||
}
|
||||
if s.GPU == nil {
|
||||
s.GPU = []HostGPUSensorFrontend{}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3533,6 +3533,22 @@ func hostSensorsFromReadStateView(sensors *unifiedresources.HostSensorMeta) mode
|
|||
out.Additional[k] = v
|
||||
}
|
||||
}
|
||||
if len(sensors.Custom) > 0 {
|
||||
out.Custom = make([]models.HostCustomSensorMetric, len(sensors.Custom))
|
||||
for i, metric := range sensors.Custom {
|
||||
out.Custom[i] = models.HostCustomSensorMetric{
|
||||
ID: metric.ID,
|
||||
Name: metric.Name,
|
||||
Unit: metric.Unit,
|
||||
Value: cloneFloat64Ptr(metric.Value),
|
||||
Status: metric.Status,
|
||||
ObservedAt: metric.ObservedAt,
|
||||
Error: metric.Error,
|
||||
AlertOnError: metric.AlertOnError,
|
||||
Stale: metric.Stale,
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(sensors.GPU) > 0 {
|
||||
out.GPU = make([]models.HostGPUSensor, len(sensors.GPU))
|
||||
for i, gpu := range sensors.GPU {
|
||||
|
|
|
|||
|
|
@ -2763,6 +2763,7 @@ func (m *Monitor) ApplyHostReport(report agentshost.Report, tokenRecord *config.
|
|||
FanRPM: cloneStringFloatMap(report.Sensors.FanRPM),
|
||||
PowerWatts: cloneStringFloatMap(report.Sensors.PowerWatts),
|
||||
Additional: cloneStringFloatMap(report.Sensors.Additional),
|
||||
Custom: convertAgentCustomSensorsToModels(report.Sensors.Custom),
|
||||
GPU: convertAgentGPUToModels(report.Sensors.GPU),
|
||||
ThermalState: convertAgentThermalStateToModels(report.Sensors.ThermalState),
|
||||
SMART: convertAgentSMARTToModels(report.Sensors.SMART),
|
||||
|
|
|
|||
|
|
@ -723,6 +723,27 @@ func convertAgentGPUToModels(gpus []agentshost.GPUSensor) []models.HostGPUSensor
|
|||
return result
|
||||
}
|
||||
|
||||
func convertAgentCustomSensorsToModels(metrics []agentshost.CustomSensorMetric) []models.HostCustomSensorMetric {
|
||||
if len(metrics) == 0 {
|
||||
return nil
|
||||
}
|
||||
result := make([]models.HostCustomSensorMetric, len(metrics))
|
||||
for i, metric := range metrics {
|
||||
result[i] = models.HostCustomSensorMetric{
|
||||
ID: strings.TrimSpace(metric.ID),
|
||||
Name: strings.TrimSpace(metric.Name),
|
||||
Unit: strings.TrimSpace(metric.Unit),
|
||||
Value: cloneFloat64Ptr(metric.Value),
|
||||
Status: strings.TrimSpace(metric.Status),
|
||||
ObservedAt: metric.ObservedAt,
|
||||
Error: strings.TrimSpace(metric.Error),
|
||||
AlertOnError: metric.AlertOnError,
|
||||
Stale: metric.Stale,
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func convertAgentThermalStateToModels(src *agentshost.ThermalState) *models.HostThermalState {
|
||||
if src == nil {
|
||||
return nil
|
||||
|
|
|
|||
|
|
@ -533,8 +533,19 @@ func TestFindLinkedProxmoxEntityWithHints_UsesExactEndpointHostnameBeforeNameFal
|
|||
}
|
||||
}
|
||||
|
||||
func TestHostSensorsFromReadStateViewPreservesSMARTSizeBytes(t *testing.T) {
|
||||
func TestHostSensorsFromReadStateViewPreservesTypedSensorData(t *testing.T) {
|
||||
customValue := 12.5
|
||||
observedAt := time.Date(2026, 7, 30, 19, 0, 0, 0, time.UTC)
|
||||
sensors := &unifiedresources.HostSensorMeta{
|
||||
Custom: []unifiedresources.HostCustomSensorMetric{{
|
||||
ID: "queue_depth",
|
||||
Name: "Queue depth",
|
||||
Unit: "items",
|
||||
Value: &customValue,
|
||||
Status: "warning",
|
||||
ObservedAt: observedAt,
|
||||
AlertOnError: true,
|
||||
}},
|
||||
SMART: []unifiedresources.HostSMARTMeta{
|
||||
{
|
||||
Device: "/dev/sda",
|
||||
|
|
@ -558,6 +569,16 @@ func TestHostSensorsFromReadStateViewPreservesSMARTSizeBytes(t *testing.T) {
|
|||
if got.SMART[0].Temperature != 32 {
|
||||
t.Fatalf("SMART temperature = %d, want 32", got.SMART[0].Temperature)
|
||||
}
|
||||
if len(got.Custom) != 1 || got.Custom[0].Value == nil || *got.Custom[0].Value != customValue {
|
||||
t.Fatalf("custom sensors = %+v, want one typed reading", got.Custom)
|
||||
}
|
||||
if got.Custom[0].ObservedAt != observedAt || got.Custom[0].Status != "warning" || !got.Custom[0].AlertOnError {
|
||||
t.Fatalf("custom sensor metadata was not preserved: %+v", got.Custom[0])
|
||||
}
|
||||
customValue = 99
|
||||
if *got.Custom[0].Value != 12.5 {
|
||||
t.Fatalf("custom sensor value was not cloned: %v", *got.Custom[0].Value)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyDockerReportNormalizesContainerCPUCapacityAcceptedIngestProof(t *testing.T) {
|
||||
|
|
@ -2609,7 +2630,7 @@ func TestApplyHostReportPersistsAgentTemperatureMetric(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestApplyHostReportPreservesGPUSensorSummary(t *testing.T) {
|
||||
func TestApplyHostReportPreservesTypedSensorSummary(t *testing.T) {
|
||||
storeCfg := metrics.DefaultConfig(t.TempDir())
|
||||
storeCfg.WriteBufferSize = 1
|
||||
store, err := metrics.NewStore(storeCfg)
|
||||
|
|
@ -2633,6 +2654,8 @@ func TestApplyHostReportPreservesGPUSensorSummary(t *testing.T) {
|
|||
utilization := 7.0
|
||||
usedBytes := int64(2 * 1024 * 1024 * 1024)
|
||||
totalBytes := int64(48 * 1024 * 1024 * 1024)
|
||||
queueDepth := 12.5
|
||||
customObservedAt := time.Date(2026, 7, 30, 19, 0, 0, 0, time.UTC)
|
||||
report := agentshost.Report{
|
||||
Agent: agentshost.AgentInfo{
|
||||
ID: "agent-gpu",
|
||||
|
|
@ -2649,6 +2672,15 @@ func TestApplyHostReportPreservesGPUSensorSummary(t *testing.T) {
|
|||
},
|
||||
Sensors: agentshost.Sensors{
|
||||
TemperatureCelsius: map[string]float64{"gpu_nvidia_0": temperature},
|
||||
Custom: []agentshost.CustomSensorMetric{{
|
||||
ID: "queue_depth",
|
||||
Name: "Queue depth",
|
||||
Unit: "items",
|
||||
Value: &queueDepth,
|
||||
Status: agentshost.CustomSensorStatusWarning,
|
||||
ObservedAt: customObservedAt,
|
||||
AlertOnError: true,
|
||||
}},
|
||||
GPU: []agentshost.GPUSensor{
|
||||
{
|
||||
ID: "0",
|
||||
|
|
@ -2687,6 +2719,13 @@ func TestApplyHostReportPreservesGPUSensorSummary(t *testing.T) {
|
|||
if host.Sensors.TemperatureCelsius["gpu_nvidia_0"] != temperature {
|
||||
t.Fatalf("legacy GPU temperature compatibility = %+v, want %.1f", host.Sensors.TemperatureCelsius, temperature)
|
||||
}
|
||||
if len(host.Sensors.Custom) != 1 || host.Sensors.Custom[0].Value == nil || *host.Sensors.Custom[0].Value != queueDepth {
|
||||
t.Fatalf("host custom sensors = %+v, want one typed reading", host.Sensors.Custom)
|
||||
}
|
||||
custom := host.Sensors.Custom[0]
|
||||
if custom.ID != "queue_depth" || custom.Unit != "items" || custom.Status != agentshost.CustomSensorStatusWarning || custom.ObservedAt != customObservedAt || !custom.AlertOnError {
|
||||
t.Fatalf("custom sensor metadata was not preserved: %+v", custom)
|
||||
}
|
||||
|
||||
expectedMetrics := map[string]float64{
|
||||
"gpu": utilization,
|
||||
|
|
|
|||
|
|
@ -248,7 +248,7 @@ func resourceFromHost(host models.Host) (Resource, ResourceIdentity) {
|
|||
storageAssessments := make([]storagehealth.Assessment, 0, len(host.RAID)+1)
|
||||
|
||||
// Populate sensors
|
||||
if len(host.Sensors.TemperatureCelsius) > 0 || len(host.Sensors.FanRPM) > 0 || len(host.Sensors.PowerWatts) > 0 || len(host.Sensors.Additional) > 0 || len(host.Sensors.GPU) > 0 || host.Sensors.ThermalState != nil || len(host.Sensors.SMART) > 0 {
|
||||
if len(host.Sensors.TemperatureCelsius) > 0 || len(host.Sensors.FanRPM) > 0 || len(host.Sensors.PowerWatts) > 0 || len(host.Sensors.Additional) > 0 || len(host.Sensors.Custom) > 0 || len(host.Sensors.GPU) > 0 || host.Sensors.ThermalState != nil || len(host.Sensors.SMART) > 0 {
|
||||
sensorMeta := &HostSensorMeta{}
|
||||
if len(host.Sensors.TemperatureCelsius) > 0 {
|
||||
sensorMeta.TemperatureCelsius = make(map[string]float64, len(host.Sensors.TemperatureCelsius))
|
||||
|
|
@ -274,6 +274,22 @@ func resourceFromHost(host models.Host) (Resource, ResourceIdentity) {
|
|||
sensorMeta.Additional[k] = v
|
||||
}
|
||||
}
|
||||
if len(host.Sensors.Custom) > 0 {
|
||||
sensorMeta.Custom = make([]HostCustomSensorMetric, len(host.Sensors.Custom))
|
||||
for i, metric := range host.Sensors.Custom {
|
||||
sensorMeta.Custom[i] = HostCustomSensorMetric{
|
||||
ID: metric.ID,
|
||||
Name: metric.Name,
|
||||
Unit: metric.Unit,
|
||||
Value: cloneFloat64Ptr(metric.Value),
|
||||
Status: metric.Status,
|
||||
ObservedAt: metric.ObservedAt,
|
||||
Error: metric.Error,
|
||||
AlertOnError: metric.AlertOnError,
|
||||
Stale: metric.Stale,
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(host.Sensors.GPU) > 0 {
|
||||
sensorMeta.GPU = make([]HostGPUSensor, len(host.Sensors.GPU))
|
||||
for i, gpu := range host.Sensors.GPU {
|
||||
|
|
|
|||
|
|
@ -41,6 +41,39 @@ func TestResourceFromProxmoxNodeIncludesTemperature(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestResourceFromHostPreservesCustomSensorMeta(t *testing.T) {
|
||||
value := 12.5
|
||||
observedAt := time.Date(2026, 7, 30, 19, 0, 0, 0, time.UTC)
|
||||
host := models.Host{
|
||||
ID: "agent-custom",
|
||||
Hostname: "edge-1",
|
||||
Sensors: models.HostSensorSummary{
|
||||
Custom: []models.HostCustomSensorMetric{{
|
||||
ID: "queue_depth",
|
||||
Name: "Queue depth",
|
||||
Unit: "items",
|
||||
Value: &value,
|
||||
Status: "warning",
|
||||
ObservedAt: observedAt,
|
||||
AlertOnError: true,
|
||||
}},
|
||||
},
|
||||
}
|
||||
|
||||
resource, _ := resourceFromHost(host)
|
||||
if resource.Agent == nil || resource.Agent.Sensors == nil || len(resource.Agent.Sensors.Custom) != 1 {
|
||||
t.Fatalf("custom sensor projection = %+v", resource.Agent)
|
||||
}
|
||||
custom := resource.Agent.Sensors.Custom[0]
|
||||
if custom.ID != "queue_depth" || custom.Value == nil || *custom.Value != 12.5 || custom.ObservedAt != observedAt {
|
||||
t.Fatalf("custom sensor projection = %+v", custom)
|
||||
}
|
||||
value = 99
|
||||
if *custom.Value != 12.5 {
|
||||
t.Fatal("custom sensor projection aliases the model value")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResourceFromProxmoxNodeUsesDisplayNameWithoutChangingRawNodeIdentity(t *testing.T) {
|
||||
node := models.Node{
|
||||
ID: "pve-cluster-node01",
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import (
|
|||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
||||
)
|
||||
|
|
@ -65,6 +66,37 @@ func TestHostSMARTMetaCarriesSizeBytesJSONContract(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestHostCustomSensorMetaJSONContract(t *testing.T) {
|
||||
value := 12.5
|
||||
observedAt := "2026-07-30T19:00:00Z"
|
||||
payload := HostSensorMeta{
|
||||
Custom: []HostCustomSensorMetric{{
|
||||
ID: "queue_depth",
|
||||
Name: "Queue depth",
|
||||
Unit: "items",
|
||||
Value: &value,
|
||||
Status: "warning",
|
||||
ObservedAt: time.Date(2026, 7, 30, 19, 0, 0, 0, time.UTC),
|
||||
}},
|
||||
}
|
||||
|
||||
data, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal HostSensorMeta: %v", err)
|
||||
}
|
||||
for _, expected := range []string{
|
||||
`"custom":[`,
|
||||
`"id":"queue_depth"`,
|
||||
`"value":12.5`,
|
||||
`"status":"warning"`,
|
||||
`"observedAt":"` + observedAt + `"`,
|
||||
} {
|
||||
if !strings.Contains(string(data), expected) {
|
||||
t.Fatalf("custom sensor JSON missing %s: %s", expected, data)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSMARTMetaJSONPreservesReportedZeroAndOmitsAbsentFields(t *testing.T) {
|
||||
zeroInt := 0
|
||||
zeroInt64 := int64(0)
|
||||
|
|
|
|||
|
|
@ -822,12 +822,25 @@ func cloneHostSensorMeta(in *HostSensorMeta) *HostSensorMeta {
|
|||
out.FanRPM = cloneStringFloat64Map(in.FanRPM)
|
||||
out.PowerWatts = cloneStringFloat64Map(in.PowerWatts)
|
||||
out.Additional = cloneStringFloat64Map(in.Additional)
|
||||
out.Custom = cloneHostCustomSensors(in.Custom)
|
||||
out.GPU = cloneHostGPUSensors(in.GPU)
|
||||
out.ThermalState = cloneHostThermalState(in.ThermalState)
|
||||
out.SMART = cloneHostSMARTMetaSlice(in.SMART)
|
||||
return &out
|
||||
}
|
||||
|
||||
func cloneHostCustomSensors(in []HostCustomSensorMetric) []HostCustomSensorMetric {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]HostCustomSensorMetric, len(in))
|
||||
for i, metric := range in {
|
||||
out[i] = metric
|
||||
out[i].Value = cloneFloat64Ptr(metric.Value)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneHostGPUSensors(in []HostGPUSensor) []HostGPUSensor {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
|
|
|
|||
|
|
@ -102,6 +102,33 @@ func TestCloneResource_MutateOriginalMetrics(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestCloneResourceIsolatesCustomSensorValues(t *testing.T) {
|
||||
value := 12.5
|
||||
original := &Resource{
|
||||
ID: "agent-custom",
|
||||
Agent: &AgentData{
|
||||
Sensors: &HostSensorMeta{
|
||||
Custom: []HostCustomSensorMetric{{
|
||||
ID: "queue_depth",
|
||||
Name: "Queue depth",
|
||||
Value: &value,
|
||||
Status: "warning",
|
||||
}},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
cloned := cloneResource(original)
|
||||
*cloned.Agent.Sensors.Custom[0].Value = 99
|
||||
if *original.Agent.Sensors.Custom[0].Value != 12.5 {
|
||||
t.Fatal("custom sensor value pointer was shared by clone")
|
||||
}
|
||||
cloned.Agent.Sensors.Custom[0].Name = "mutated"
|
||||
if original.Agent.Sensors.Custom[0].Name != "Queue depth" {
|
||||
t.Fatal("custom sensor slice was shared by clone")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloneResource_MutateIncidents(t *testing.T) {
|
||||
original := &Resource{
|
||||
ID: "r-1",
|
||||
|
|
|
|||
|
|
@ -612,11 +612,24 @@ type HostSensorMeta struct {
|
|||
FanRPM map[string]float64 `json:"fanRpm,omitempty"`
|
||||
PowerWatts map[string]float64 `json:"powerWatts,omitempty"`
|
||||
Additional map[string]float64 `json:"additional,omitempty"`
|
||||
Custom []HostCustomSensorMetric `json:"custom,omitempty"`
|
||||
GPU []HostGPUSensor `json:"gpu,omitempty"`
|
||||
ThermalState *HostThermalState `json:"thermalState,omitempty"`
|
||||
SMART []HostSMARTMeta `json:"smart,omitempty"`
|
||||
}
|
||||
|
||||
type HostCustomSensorMetric struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Unit string `json:"unit,omitempty"`
|
||||
Value *float64 `json:"value,omitempty"`
|
||||
Status string `json:"status"`
|
||||
ObservedAt time.Time `json:"observedAt"`
|
||||
Error string `json:"error,omitempty"`
|
||||
AlertOnError bool `json:"alertOnError,omitempty"`
|
||||
Stale bool `json:"stale,omitempty"`
|
||||
}
|
||||
|
||||
type HostGPUSensor struct {
|
||||
ID string `json:"id,omitempty"`
|
||||
Name string `json:"name,omitempty"`
|
||||
|
|
|
|||
|
|
@ -258,11 +258,34 @@ type Sensors struct {
|
|||
FanRPM map[string]float64 `json:"fanRpm,omitempty"`
|
||||
PowerWatts map[string]float64 `json:"powerWatts,omitempty"` // Power consumption (e.g., cpu_package, dram)
|
||||
Additional map[string]float64 `json:"additional,omitempty"`
|
||||
Custom []CustomSensorMetric `json:"custom,omitempty"`
|
||||
GPU []GPUSensor `json:"gpu,omitempty"`
|
||||
ThermalState *ThermalState `json:"thermalState,omitempty"`
|
||||
SMART []DiskSMART `json:"smart,omitempty"` // S.M.A.R.T. disk data
|
||||
}
|
||||
|
||||
const (
|
||||
CustomSensorStatusOK = "ok"
|
||||
CustomSensorStatusWarning = "warning"
|
||||
CustomSensorStatusCritical = "critical"
|
||||
CustomSensorStatusError = "error"
|
||||
)
|
||||
|
||||
// CustomSensorMetric is a bounded numeric reading produced by an explicitly
|
||||
// configured local executable. The agent, not the server, owns execution and
|
||||
// threshold evaluation.
|
||||
type CustomSensorMetric struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Unit string `json:"unit,omitempty"`
|
||||
Value *float64 `json:"value,omitempty"`
|
||||
Status string `json:"status"`
|
||||
ObservedAt time.Time `json:"observedAt"`
|
||||
Error string `json:"error,omitempty"`
|
||||
AlertOnError bool `json:"alertOnError,omitempty"`
|
||||
Stale bool `json:"stale,omitempty"`
|
||||
}
|
||||
|
||||
// GPUSensor captures direct GPU telemetry reported by a local host agent.
|
||||
type GPUSensor struct {
|
||||
ID string `json:"id,omitempty"`
|
||||
|
|
|
|||
|
|
@ -246,6 +246,8 @@ func TestNetworkInterface_Fields(t *testing.T) {
|
|||
|
||||
func TestSensors_Fields(t *testing.T) {
|
||||
thermalWarningLevel := 0
|
||||
customValue := 7.0
|
||||
observedAt := time.Date(2026, 7, 30, 19, 0, 0, 0, time.UTC)
|
||||
sensors := Sensors{
|
||||
TemperatureCelsius: map[string]float64{
|
||||
"cpu_package": 45.0,
|
||||
|
|
@ -258,6 +260,15 @@ func TestSensors_Fields(t *testing.T) {
|
|||
Additional: map[string]float64{
|
||||
"voltage": 12.0,
|
||||
},
|
||||
Custom: []CustomSensorMetric{{
|
||||
ID: "pending_updates",
|
||||
Name: "Pending updates",
|
||||
Unit: "count",
|
||||
Value: &customValue,
|
||||
Status: CustomSensorStatusWarning,
|
||||
ObservedAt: observedAt,
|
||||
AlertOnError: true,
|
||||
}},
|
||||
GPU: []GPUSensor{
|
||||
{
|
||||
ID: "0",
|
||||
|
|
@ -284,6 +295,9 @@ func TestSensors_Fields(t *testing.T) {
|
|||
if sensors.FanRPM["cpu_fan"] != 1200.0 {
|
||||
t.Errorf("cpu_fan RPM = %f, want 1200.0", sensors.FanRPM["cpu_fan"])
|
||||
}
|
||||
if len(sensors.Custom) != 1 || sensors.Custom[0].Value == nil || *sensors.Custom[0].Value != 7 {
|
||||
t.Fatalf("custom sensors = %+v, want one typed reading", sensors.Custom)
|
||||
}
|
||||
if len(sensors.GPU) != 1 {
|
||||
t.Fatalf("GPU count = %d, want 1", len(sensors.GPU))
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue