mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-07-09 16:00:59 +00:00
feat: finalize swarm service monitoring (#598)
This commit is contained in:
rcourtman
parent
8e83eaf823
commit
68ce8e7520
45 changed files with 2823 additions and 1660 deletions
2
VERSION
2
VERSION
|
|
@ -1 +1 @@
|
|||
4.25.0
|
||||
4.26.0
|
||||
|
|
|
|||
|
|
@ -66,6 +66,10 @@ func loadConfig() dockeragent.Config {
|
|||
envNoAutoUpdate := strings.TrimSpace(os.Getenv("PULSE_NO_AUTO_UPDATE"))
|
||||
envTargets := strings.TrimSpace(os.Getenv("PULSE_TARGETS"))
|
||||
envContainerStates := strings.TrimSpace(os.Getenv("PULSE_CONTAINER_STATES"))
|
||||
envSwarmScope := strings.TrimSpace(os.Getenv("PULSE_SWARM_SCOPE"))
|
||||
envSwarmServices := strings.TrimSpace(os.Getenv("PULSE_SWARM_SERVICES"))
|
||||
envSwarmTasks := strings.TrimSpace(os.Getenv("PULSE_SWARM_TASKS"))
|
||||
envIncludeContainers := strings.TrimSpace(os.Getenv("PULSE_INCLUDE_CONTAINERS"))
|
||||
|
||||
defaultInterval := 30 * time.Second
|
||||
if envInterval != "" {
|
||||
|
|
@ -74,6 +78,26 @@ func loadConfig() dockeragent.Config {
|
|||
}
|
||||
}
|
||||
|
||||
swarmScopeDefault := envSwarmScope
|
||||
if swarmScopeDefault == "" {
|
||||
swarmScopeDefault = "node"
|
||||
}
|
||||
|
||||
includeServicesDefault := true
|
||||
if envSwarmServices != "" {
|
||||
includeServicesDefault = parseBool(envSwarmServices)
|
||||
}
|
||||
|
||||
includeTasksDefault := true
|
||||
if envSwarmTasks != "" {
|
||||
includeTasksDefault = parseBool(envSwarmTasks)
|
||||
}
|
||||
|
||||
includeContainersDefault := true
|
||||
if envIncludeContainers != "" {
|
||||
includeContainersDefault = parseBool(envIncludeContainers)
|
||||
}
|
||||
|
||||
urlFlag := flag.String("url", envURL, "Pulse server URL (e.g. http://pulse:7655)")
|
||||
tokenFlag := flag.String("token", envToken, "Pulse API token (required)")
|
||||
intervalFlag := flag.Duration("interval", defaultInterval, "Reporting interval (e.g. 30s)")
|
||||
|
|
@ -85,6 +109,10 @@ func loadConfig() dockeragent.Config {
|
|||
flag.Var(&targetFlags, "target", "Pulse target in url|token[|insecure] format. Repeat to send to multiple Pulse instances")
|
||||
var containerStateFlags stringFlagList
|
||||
flag.Var(&containerStateFlags, "container-state", "Only include containers whose status matches this value (repeat to allow multiple). Allowed values: created,running,restarting,removing,paused,exited,dead.")
|
||||
swarmScopeFlag := flag.String("swarm-scope", strings.ToLower(strings.TrimSpace(swarmScopeDefault)), "Swarm data scope to collect: node, cluster, or auto")
|
||||
includeServicesFlag := flag.Bool("swarm-services", includeServicesDefault, "Include Swarm service summaries in reports")
|
||||
includeTasksFlag := flag.Bool("swarm-tasks", includeTasksDefault, "Include Swarm tasks in reports")
|
||||
includeContainersFlag := flag.Bool("include-containers", includeContainersDefault, "Include per-container metrics in reports")
|
||||
|
||||
flag.Parse()
|
||||
|
||||
|
|
@ -146,6 +174,10 @@ func loadConfig() dockeragent.Config {
|
|||
DisableAutoUpdate: *noAutoUpdateFlag,
|
||||
Targets: targets,
|
||||
ContainerStates: containerStates,
|
||||
SwarmScope: strings.ToLower(strings.TrimSpace(*swarmScopeFlag)),
|
||||
IncludeServices: *includeServicesFlag,
|
||||
IncludeTasks: *includeTasksFlag,
|
||||
IncludeContainers: *includeContainersFlag,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -2,8 +2,8 @@ apiVersion: v2
|
|||
name: pulse
|
||||
description: Helm chart for deploying the Pulse hub and optional Docker monitoring agent.
|
||||
type: application
|
||||
version: 4.25.0
|
||||
appVersion: "4.25.0"
|
||||
version: 4.26.0
|
||||
appVersion: "4.26.0"
|
||||
icon: https://raw.githubusercontent.com/rcourtman/Pulse/main/docs/images/pulse-logo.svg
|
||||
keywords:
|
||||
- monitoring
|
||||
|
|
|
|||
|
|
@ -224,7 +224,11 @@ PROXY_AUTH_LOGOUT_URL=/logout # URL for SSO logout
|
|||
"dockerDefaults": {
|
||||
"cpu": { "trigger": 75, "clear": 60 },
|
||||
"restartCount": 3,
|
||||
"restartWindow": 300
|
||||
"restartWindow": 300,
|
||||
"memoryWarnPct": 90,
|
||||
"memoryCriticalPct": 95,
|
||||
"serviceWarnGapPercent": 10,
|
||||
"serviceCriticalGapPercent": 50
|
||||
},
|
||||
"dockerIgnoredContainerPrefixes": [
|
||||
"runner-",
|
||||
|
|
@ -239,7 +243,7 @@ PROXY_AUTH_LOGOUT_URL=/logout # URL for SSO logout
|
|||
"guest": { "disk": 120, "networkOut": 240 }
|
||||
},
|
||||
"overrides": {
|
||||
"delly.lan/qemu/101": {
|
||||
"example.lan/qemu/101": {
|
||||
"memory": { "trigger": 92, "clear": 80 },
|
||||
"networkOut": -1,
|
||||
"poweredOffSeverity": "warning"
|
||||
|
|
@ -280,6 +284,7 @@ PROXY_AUTH_LOGOUT_URL=/logout # URL for SSO logout
|
|||
- `timeThresholds` apply a grace period before an alert fires; `metricTimeThresholds` allow per-metric overrides (e.g., delay network alerts longer than CPU).
|
||||
- `overrides` are indexed by the stable resource ID returned from `/api/state` (VMs: `instance/qemu/vmid`, containers: `instance/lxc/ctid`, nodes: `instance/node`).
|
||||
- `dockerIgnoredContainerPrefixes` lets you silence state/metric/restart alerts for ephemeral containers whose names or IDs share a common, case-insensitive prefix. The Docker tab in the UI keeps this list in sync.
|
||||
- Swarm service alerts track missing replicas: `serviceWarnGapPercent` defines when a warning fires, and `serviceCriticalGapPercent` must be greater than or equal to the warning gap (Pulse automatically clamps the critical value upward if an older client submits something smaller).
|
||||
- Quiet hours, escalation, deduplication, and restart loop detection are all managed here, and the UI keeps the JSON in sync automatically.
|
||||
|
||||
> Tip: Back up `alerts.json` alongside `.env` during exports. Restoring it preserves all overrides, quiet-hour schedules, and webhook routing.
|
||||
|
|
|
|||
|
|
@ -88,6 +88,20 @@ The binary reads standard Docker environment variables. If you already use TLS-s
|
|||
|
||||
High churn environments can flood Pulse with noise from short-lived tasks. Restrict the agent to the container states you care about by repeating `--container-state` (for example, `--container-state running --container-state paused`) or by exporting `PULSE_CONTAINER_STATES=running,paused`. Allowed values match Docker’s status filter: `created`, `running`, `restarting`, `removing`, `paused`, `exited`, and `dead`. If no values are provided the agent reports every container, mirroring the previous behaviour.
|
||||
|
||||
### Swarm-aware reporting
|
||||
|
||||
The agent now recognises Docker Swarm roles. Managers query the Swarm control plane for service and task metadata, while workers fall back to the labels present on local containers. The **Settings → Docker Agents** view surfaces role, scope, service counts, and updates per host so you can spot noisy stacks or unhealthy rollouts at a glance.
|
||||
|
||||
Use the new flags to tune the payload:
|
||||
|
||||
- `--swarm-scope` / `PULSE_SWARM_SCOPE` chooses between node-only and cluster-wide aggregation (`auto` switches based on the node’s role).
|
||||
- `--swarm-services` and `--swarm-tasks` disable service or task blocks if you only need a subset of data.
|
||||
- `--include-containers` removes per-container metrics when service-level reporting is sufficient (note that workers need container data to derive task info).
|
||||
|
||||
If a manager cannot reach the Swarm API the agent automatically falls back to node scope so updates keep flowing.
|
||||
|
||||
Adjust warning and critical replica gaps (or disable service alerts entirely) under **Alerts → Thresholds → Docker** in the Pulse UI.
|
||||
|
||||
### Multiple Pulse instances
|
||||
|
||||
A single `pulse-docker-agent` process can now serve any number of Pulse backends. Each target entry keeps its own API token and TLS preference, and Pulse de-duplicates reports using the shared agent ID / machine ID. This avoids running duplicate agents on busy Docker hosts.
|
||||
|
|
@ -139,6 +153,10 @@ docker run -d \
|
|||
| `--target`, `PULSE_TARGETS` | One or more `url|token[|insecure]` entries to fan-out reports to multiple Pulse servers. Separate entries with `;` or repeat the flag. | — |
|
||||
| `--interval`, `PULSE_INTERVAL` | Reporting cadence (supports `30s`, `1m`, etc.). | `30s` |
|
||||
| `--container-state`, `PULSE_CONTAINER_STATES` | Limit reports to specific Docker statuses (`created`, `running`, `restarting`, `removing`, `paused`, `exited`, `dead`). Separate multiple values with commas/semicolons or repeat the flag. | — |
|
||||
| `--swarm-scope`, `PULSE_SWARM_SCOPE` | Swarm data scope: `node`, `cluster`, or `auto` (auto picks cluster on managers, node on workers). | `node` |
|
||||
| `--swarm-services`, `PULSE_SWARM_SERVICES` | Include Swarm service summaries in reports. | `true` |
|
||||
| `--swarm-tasks`, `PULSE_SWARM_TASKS` | Include individual Swarm tasks in reports. | `true` |
|
||||
| `--include-containers`, `PULSE_INCLUDE_CONTAINERS` | Include per-container metrics (disable when only Swarm data is needed). | `true` |
|
||||
| `--hostname`, `PULSE_HOSTNAME` | Override host name reported to Pulse. | Docker info / OS hostname |
|
||||
| `--agent-id`, `PULSE_AGENT_ID` | Stable ID for the agent (useful for clustering). | Docker engine ID / machine-id |
|
||||
| `--insecure`, `PULSE_INSECURE_SKIP_VERIFY` | Skip TLS cert validation (unsafe). | `false` |
|
||||
|
|
|
|||
|
|
@ -1,12 +1,54 @@
|
|||
# Upcoming Release Highlights
|
||||
# Pulse v4.26.0
|
||||
|
||||
## Improvements
|
||||
## What's Changed
|
||||
### New
|
||||
- Standalone host agents now ship with guided Linux, macOS, and Windows installers that stream registration status back to Pulse, generate scoped commands from **Settings → Agents**, and feed host metrics into alerts alongside Proxmox and Docker.
|
||||
- Alert thresholds gained host-level overrides, connectivity toggles, and snapshot size guardrails so you can tune offline behaviour per host while keeping a global policy for other resources.
|
||||
- API tokens now support fine-grained scopes with a redesigned manager that previews command templates, highlights unused credentials, and makes revocation a single click.
|
||||
- Proxmox replication jobs surface in a dedicated **Settings → Hosts → Replication** view with API plumbing to track task health and bubble failures into the monitoring pipeline.
|
||||
- Docker Swarm environments now receive service/task-aware reporting with configurable scope, plus a Docker settings view that highlights manager/worker roles, stack health, rollout status, and service alert thresholds.
|
||||
|
||||
- Host agent installs now report back to Pulse in real time. The **Settings → Agents → Host agents** view surfaces the lookup status, highlights the matching row, and refreshes the connection badge without reloading.
|
||||
- Host agents participate in the alert system alongside Proxmox and Docker sources. Offline detection, thresholds, and overrides live in **Settings → Alerts** and flow through the existing notification channels.
|
||||
### Improvements
|
||||
- Dashboard loads and drawer links respond faster thanks to cached guest metadata, reduced polling allocations, and inline URL editing that no longer flashes on WebSocket updates.
|
||||
- Settings navigation is reorganized with dedicated Docker and Hosts sections, richer filters, and platform icons that make agent onboarding and discovery workflows clearer.
|
||||
- LXC guests now report dynamic interface IPs, configuration metadata, and queue metrics so alerting, discovery, and drawers stay accurate even during rapid container churn.
|
||||
- Notifications consolidate into a consistent toast system, with clearer feedback during agent setup, token generation, and background job state changes.
|
||||
|
||||
## Documentation
|
||||
### Bug Fixes
|
||||
- Enforced explicit node naming and respected custom Proxmox ports so cluster discovery, overrides, and disk monitoring defaults remain intact after edits.
|
||||
- Hardened setup-token flows and checksum handling in the installers to prevent stale credentials and guarantee the correct binaries are fetched.
|
||||
- Treated 501 responses from the Proxmox API as non-fatal during failover, restored FreeBSD disk counter parsing, and stopped guest link icons from re-triggering animations on updates.
|
||||
- Preserved inline editor state across WebSocket refreshes and ensured Docker host identifiers stay collision-safe in mixed environments.
|
||||
|
||||
- Updated the host agent guide with the new lookup workflow, token-free commands, and alert integration notes.
|
||||
## Installation
|
||||
- **Install or upgrade with the helper script**
|
||||
```bash
|
||||
curl -sL https://github.com/rcourtman/Pulse/releases/latest/download/install.sh | bash
|
||||
```
|
||||
- **Binary upgrade on systemd hosts**
|
||||
```bash
|
||||
sudo systemctl stop pulse
|
||||
curl -fsSL https://github.com/rcourtman/Pulse/releases/download/v4.26.0/pulse-v4.26.0-linux-amd64.tar.gz \
|
||||
| sudo tar -xz -C /opt/pulse --strip-components=1
|
||||
sudo systemctl start pulse
|
||||
```
|
||||
- **Docker update**
|
||||
```bash
|
||||
docker pull rcourtman/pulse:v4.26.0
|
||||
docker stop pulse || true
|
||||
docker rm pulse || true
|
||||
docker run -d --name pulse --restart unless-stopped -p 7655:7655 rcourtman/pulse:v4.26.0
|
||||
```
|
||||
- **Helm upgrade**
|
||||
```bash
|
||||
helm upgrade --install pulse oci://ghcr.io/rcourtman/pulse-chart \
|
||||
--version 4.26.0 \
|
||||
--namespace pulse --create-namespace
|
||||
```
|
||||
|
||||
_(Replace this stub with per-version notes when tagging the release.)_
|
||||
## Downloads
|
||||
- Multi-arch Linux tarballs (amd64/arm64/armv7)
|
||||
- Standalone sensor proxy binaries
|
||||
- Helm chart archive (pulse-4.26.0-helm.tgz)
|
||||
- SHA256 checksums (checksums.txt)
|
||||
- Docker tags: rcourtman/pulse:v4.26.0, :4.26, :4, :latest
|
||||
|
|
|
|||
|
|
@ -285,7 +285,7 @@ pulse_proxy_rpc_requests_total{method="get_temperature",result="success"}
|
|||
pulse_proxy_rpc_requests_total{method="ensure_cluster_keys",result="unauthorized"}
|
||||
|
||||
# SSH request latency
|
||||
pulse_proxy_ssh_latency_seconds{node="delly"}
|
||||
pulse_proxy_ssh_latency_seconds{node="example-node"}
|
||||
|
||||
# Active connections
|
||||
pulse_proxy_queue_depth
|
||||
|
|
@ -417,7 +417,7 @@ curl -s --unix-socket /run/pulse-sensor-proxy/pulse-sensor-proxy.sock \
|
|||
-X POST -d '{"method":"get_status"}' | jq
|
||||
|
||||
# 5. Check SSH connectivity
|
||||
ssh root@delly "sensors -j"
|
||||
ssh root@example-node "sensors -j"
|
||||
|
||||
# 6. Inspect adaptive polling for temperature pollers
|
||||
curl -s http://localhost:7655/api/monitoring/scheduler/health \
|
||||
|
|
|
|||
|
|
@ -1,98 +0,0 @@
|
|||
# API Token Scope Design Brief
|
||||
|
||||
## Objective
|
||||
Introduce scoped API tokens so administrators can grant the minimum necessary permissions to each integration (Docker agent, host agent, future platform agents, automation scripts, etc.). This replaces today’s “all-or-nothing” tokens and provides safer rotation/revocation paths.
|
||||
|
||||
## Security Rationale
|
||||
- Agent and automation tokens are often deployed on hosts or third-party services we do not fully trust. If one leaks today, the attacker inherits full administrator powers (issuing new tokens, mutating settings, triggering installs/updates, etc.).
|
||||
- Constraining tokens to the minimal scope limits the blast radius: a compromised reporting agent can only submit telemetry, not reconfigure Pulse or other integrations.
|
||||
- Customers operating in regulated or multi-team environments increasingly ask for auditable least-privilege controls. Scopes give us the primitives to surface warnings on over-privileged tokens and eventually add rotation workflows.
|
||||
- The feature still has to earn adoption, so pair the technical work with UI nudges and reporting that highlight “Full access” tokens and encourage admins to narrow permissions.
|
||||
|
||||
## Requirements Overview
|
||||
|
||||
1. **Token Model Changes**
|
||||
- Each API token record stores a list of scopes (strings) or a bitmask. Recommended canonical strings:
|
||||
- `monitoring:read`
|
||||
- `monitoring:write`
|
||||
- `docker:report`
|
||||
- `docker:manage`
|
||||
- `host-agent:report`
|
||||
- `settings:read`
|
||||
- `settings:write`
|
||||
- `*` (legacy full-access sentinel; backend accepts for migration/edit flows but the UI should not expose it)
|
||||
- Existing tokens must remain valid. Treat missing scopes as `["*"]` (full access) until the admin edits them.
|
||||
|
||||
2. **Persistence & Migration**
|
||||
- Extend the token persistence layer (currently BoltDB JSON) to include `scopes: []string`.
|
||||
- On startup, detect tokens without the new field and default to `["*"]`.
|
||||
- Expose the complete scope list when returning token metadata (internal API used by Settings UI).
|
||||
|
||||
3. **Middleware Enforcement**
|
||||
- Add a helper `RequireScope(scope string)` that checks the request’s token record for `scope` or `*`.
|
||||
- Apply the helper according to the table below:
|
||||
|
||||
| Endpoint (or group) | HTTP verbs | Required scope(s) | Notes |
|
||||
|-------------------------------------------------|-------------------------------------|-----------------------------|------------------------------------------------------|
|
||||
| `/api/agents/docker/report` | `POST` | `docker:report` | Docker agent heartbeat payloads |
|
||||
| `/api/agents/docker/commands/*` | `POST` | `docker:manage` (optional) | If we expose command ack/management over tokens |
|
||||
| `/api/agents/docker/hosts/*` | `DELETE`, `PUT`, `POST` | `docker:manage` | Admin actions for Docker hosts |
|
||||
| `/api/agents/host/report` | `POST` | `host-agent:report` | Host agent reporting |
|
||||
| `/api/state` | `GET` | `monitoring:read` | General state polling (if token-authenticated) |
|
||||
| `/api/alerts/*` | `GET` | `monitoring:read` | Alerts reading APIs |
|
||||
| `/api/alerts/*` (mutations) | `POST`, `PUT`, `DELETE` | `monitoring:write` | Acknowledge, silence, etc. |
|
||||
| `/api/settings/*` | `GET` | `settings:read` | Settings reads via API |
|
||||
| `/api/settings/*` | `POST`, `PUT`, `DELETE`, `PATCH` | `settings:write` | Any settings mutation |
|
||||
| `/api/security/tokens*` | all verbs | _n/a (session only)_ | Leave browser-session only; do not allow API tokens yet |
|
||||
| `/api/install/*`, `/api/updates/*` (mutations) | `POST`, `PUT` | `settings:write` | Sensitive operational endpoints |
|
||||
|
||||
(More endpoints can be added as required; start with the rows above and expand during implementation.)
|
||||
- Maintain compatibility for admin sessions (browser login) which continue to bypass token checks.
|
||||
|
||||
4. **Token Generation API**
|
||||
- Update `POST /api/security/tokens` to accept a `scopes` array.
|
||||
- Validation rules:
|
||||
- Reject unknown scope identifiers (except the `*` sentinel described above).
|
||||
- If the array is omitted (legacy callers), default to `['*']` (full access) to preserve backward compatibility.
|
||||
- If the array is provided but empty, reject with a 400 (“select at least one scope or delete the token”).
|
||||
- Reject mixed arrays that contain both `'*'` and explicit scopes; if the UI submits such a payload, return a 400 with guidance (“either all scopes or full access”).
|
||||
- Include the scope list in the response payload so the UI can display it.
|
||||
|
||||
5. **UI/UX Adjustments**
|
||||
- **Settings → Security** panel:
|
||||
- When generating or editing a token, show a multi-select with friendly labels (“Docker agent reporting”, “Host agent reporting”, etc.).
|
||||
- Display the scope summary in the token list (e.g. badges).
|
||||
- For legacy tokens (implicit `*`), show “Full access” and allow editing to reduce scope.
|
||||
- **Docker Agents / Host Agents screens:**
|
||||
- When requesting a token:
|
||||
- Docker: pre-select both `docker:report` (always) and `docker:manage` if the user needs lifecycle commands (hide manage behind a toggle if desired).
|
||||
- Host agent: pre-select `host-agent:report`.
|
||||
- Warn if the stored token lacks the required scope (fallback to showing `<api-token>` placeholder).
|
||||
|
||||
6. **Testing**
|
||||
- Unit tests covering:
|
||||
- Scope parsing/migration
|
||||
- Middleware checks (token with/without required scope)
|
||||
- Integration tests for agent endpoints verifying 403 on missing scope.
|
||||
|
||||
7. **Documentation**
|
||||
- Update `README.md` and relevant docs (e.g. `docs/CONFIGURATION.md`, `docs/HOST_AGENT.md`, Docker docs) to explain scoped tokens.
|
||||
- Provide an upgrade note for existing installations (“legacy tokens default to full access; edit to restrict scope”).
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
- Use constants for scope strings to avoid typos throughout the codebase.
|
||||
- Token middleware already retrieves `APITokenRecord`; that struct should grow a `Scopes []string` field with helper methods (`HasScope`).
|
||||
- For future extensibility, keep the scope checks granular but simple (string equality) rather than regex matching.
|
||||
- Ensure the Settings UI gracefully handles lack of admin privileges (disable scope selection, show hint).
|
||||
- Update agent commands (Docker/Host) to mention required scope in their description.
|
||||
- Guardrails: the backend should never auto-insert `*` once a scoped token exists, and any admin edit that clears all scopes should surface a clear “delete token or assign scopes” decision.
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
- Scoped tokens persisted and surfaced via API.
|
||||
- Middleware rejects tokens missing required scope.
|
||||
- UI can create, edit, and display scoped tokens; agent panels auto-fill only when valid.
|
||||
- Documentation updated; existing tokens remain functional without manual migration.
|
||||
|
||||
Once implemented delete this doc.
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
# Home Assistant Battery Automation Notes (2025-10-10)
|
||||
|
||||
- Work performed on delly host, LXC VMID 101 (Home Assistant). Pulse codebase unaffected.
|
||||
- Root cause: malformed `automations.yaml` caused five restarts between 00:23–00:33 BST.
|
||||
- Fixes applied:
|
||||
- Restored automation backups.
|
||||
- Replaced `pyscript.solis_set_charge_current` calls with `number.set_value` targeting `number.solis_rhi_time_charging_charge_current`.
|
||||
- Removed invalid `source:` attributes from `number.set_value` actions.
|
||||
- Validation:
|
||||
- Triggered key automations (`automation.free_electricity_session_maximum_charging`, `automation.intelligent_dispatch_solis_battery_charging`, `automation.emergency_low_soc_protection`, EV guard hold/refresh/release).
|
||||
- Observed expected charge-current adjustments (100 A car slots/free session, 25 A emergency, 5 A guard) and matching system logs.
|
||||
- Defaults restored: overnight slot `23:30–05:30`.
|
||||
- Backups: `/var/lib/docker/volumes/hass_config/_data/automations.yaml.codex_source_cleanup_20251010_090205` (do not delete).
|
||||
- Testing helpers: API token stored at `/tmp/ha_token.txt` inside VM; use `pct exec 101 -- bash -lc '...'` for commands.
|
||||
- Final state: charge current reset to 25 A; EV guard sensor `off`.
|
||||
|
|
@ -1,256 +0,0 @@
|
|||
# Mock Mode Development Guide
|
||||
|
||||
Mock mode allows you to develop and test Pulse without requiring real Proxmox infrastructure. It generates realistic mock data including nodes, VMs, containers, storage, backups, and alerts.
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Toggling Mock Mode
|
||||
|
||||
During hot-dev mode (`scripts/hot-dev.sh`), use these npm commands to toggle mock mode:
|
||||
|
||||
```bash
|
||||
# Enable mock mode
|
||||
npm run mock:on
|
||||
|
||||
# Disable mock mode (use real infrastructure)
|
||||
npm run mock:off
|
||||
|
||||
# Check current status
|
||||
npm run mock:status
|
||||
|
||||
# Edit mock configuration
|
||||
npm run mock:edit
|
||||
```
|
||||
|
||||
The backend will **automatically reload** when you toggle mock mode - no manual restarts needed!
|
||||
|
||||
### Configuration
|
||||
|
||||
Mock mode is configured via `mock.env` in the project root. This file is **tracked in the repository** with sensible defaults, so mock mode works out of the box for all developers.
|
||||
|
||||
**Default configuration (mock.env):**
|
||||
```bash
|
||||
PULSE_MOCK_MODE=false # Disabled by default
|
||||
PULSE_MOCK_NODES=7
|
||||
PULSE_MOCK_VMS_PER_NODE=5
|
||||
PULSE_MOCK_LXCS_PER_NODE=8
|
||||
PULSE_MOCK_DOCKER_HOSTS=3
|
||||
PULSE_MOCK_DOCKER_CONTAINERS=12
|
||||
PULSE_MOCK_RANDOM_METRICS=true
|
||||
PULSE_MOCK_STOPPED_PERCENT=20
|
||||
```
|
||||
|
||||
**Local overrides (not tracked):**
|
||||
Create `mock.env.local` for personal settings that won't be committed:
|
||||
```bash
|
||||
# mock.env.local - your personal settings
|
||||
PULSE_MOCK_MODE=true # Always start in mock mode
|
||||
PULSE_MOCK_NODES=3 # Fewer nodes for faster startup
|
||||
```
|
||||
|
||||
The `.local` file overrides values from `mock.env`, and is gitignored to keep your personal preferences private.
|
||||
|
||||
**Configuration options:**
|
||||
|
||||
- `PULSE_MOCK_MODE`: Enable/disable mock mode (`true`/`false`)
|
||||
- `PULSE_MOCK_NODES`: Number of nodes to generate (default: 7)
|
||||
- `PULSE_MOCK_VMS_PER_NODE`: Average VMs per node (default: 5)
|
||||
- `PULSE_MOCK_LXCS_PER_NODE`: Average containers per node (default: 8)
|
||||
- `PULSE_MOCK_DOCKER_HOSTS`: Number of Docker hosts to generate (default: 3)
|
||||
- `PULSE_MOCK_DOCKER_CONTAINERS`: Average containers per Docker host (default: 12)
|
||||
- `PULSE_MOCK_RANDOM_METRICS`: Enable metric fluctuations (`true`/`false`)
|
||||
- `PULSE_MOCK_STOPPED_PERCENT`: Percentage of guests in stopped state (default: 20)
|
||||
|
||||
### Data Isolation
|
||||
|
||||
Hot-dev mode now isolates mock data from production credentials automatically:
|
||||
|
||||
- **Mock mode:** data lives in `/opt/pulse/tmp/mock-data`
|
||||
- **Production mode:** data lives in `/etc/pulse`
|
||||
|
||||
The toggle script exports `PULSE_DATA_DIR` before launching the backend, creates the temporary directory if needed, and cleans up on exit. This guarantees mock credentials never overwrite your real cluster configuration and makes it safe to flip between datasets repeatedly during a session.
|
||||
|
||||
## Hot-Dev Workflow
|
||||
|
||||
Hot reload now runs as a long-lived systemd service so it is always ready when you log in.
|
||||
|
||||
1. **Check the hot-dev service:**
|
||||
```bash
|
||||
systemctl status pulse-hot-dev
|
||||
```
|
||||
The service is enabled by default; use `sudo systemctl restart pulse-hot-dev` if you need a clean rebuild.
|
||||
|
||||
2. **Toggle mock mode as needed:**
|
||||
```bash
|
||||
npm run mock:on # Backend auto-reloads with mock data
|
||||
npm run mock:off # Backend auto-reloads with real data
|
||||
```
|
||||
|
||||
3. **Edit mock configuration:**
|
||||
```bash
|
||||
npm run mock:edit # Opens mock.env in your editor
|
||||
# Save and exit - backend auto-reloads!
|
||||
```
|
||||
|
||||
4. **Frontend changes:** Just save your files - Vite hot-reloads instantly
|
||||
|
||||
**No port changes. No manual restarts. Everything just works!**
|
||||
|
||||
> **Note:** The legacy `pulse-backend.service` is intentionally disabled on this dev box. All backend/API traffic comes from the hot-dev service, so you never need to run the production binary locally.
|
||||
|
||||
### Default credentials
|
||||
|
||||
Authentication is enabled for the dev stack so security-focused features behave exactly like production. Use the shared credentials below when the UI prompts for a login:
|
||||
|
||||
```
|
||||
Username: dev
|
||||
Password: dev
|
||||
```
|
||||
|
||||
You can change them at any time by editing `.env` at the repo root (the backend watcher loads it automatically on restart).
|
||||
|
||||
|
||||
## Mock Data Generation
|
||||
|
||||
Mock mode generates:
|
||||
|
||||
- **Nodes**: Mix of clustered and standalone nodes
|
||||
- **Cluster**: First 5 nodes form `mock-cluster`, rest are standalone
|
||||
- **VMs & Containers**: Realistic distribution with various states
|
||||
- **Storage**: Local, ZFS, PBS, and shared NFS storage
|
||||
- **Backups**: Both PVE and PBS backups with realistic metadata
|
||||
- **Mail Gateway**: PMG instances with mail throughput, spam/virus totals, and quarantine counts
|
||||
- **Alerts**: CPU, memory, disk, and connectivity alerts
|
||||
- **Metrics**: Live-updating metrics every 2 seconds
|
||||
|
||||
### Node Characteristics
|
||||
|
||||
- **Clustered nodes** (`pve1`-`pve5`): Part of `mock-cluster`
|
||||
- **Standalone nodes** (`standalone1`, etc.): Independent instances
|
||||
- **Offline nodes**: `pve3` is always offline to test error handling
|
||||
- **Host URLs**: Each node has `Host` field set (e.g., `https://pve1.local:8006`)
|
||||
- **Cluster fields**: `IsClusterMember` and `ClusterName` properly set
|
||||
|
||||
## API Behavior in Mock Mode
|
||||
|
||||
### Fast, Cached Responses
|
||||
|
||||
In mock mode, `/api/state` returns **cached data instantly** - no locks, no delays, no timeouts. The mock data is stored in memory and updated every 2 seconds with realistic metric fluctuations.
|
||||
|
||||
### WebSocket Updates
|
||||
|
||||
The WebSocket connection receives updates every 2 seconds with changing metrics, just like production.
|
||||
|
||||
### Dashboard Grouping
|
||||
|
||||
Mock nodes include all required fields for proper dashboard grouping:
|
||||
- `isClusterMember`: Boolean indicating cluster membership
|
||||
- `clusterName`: Name of the cluster (e.g., "mock-cluster")
|
||||
- `host`: Full node URL (e.g., "https://pve1.local:8006")
|
||||
|
||||
## Demo Server Usage
|
||||
|
||||
On the demo server, mock mode works the same way:
|
||||
|
||||
### Systemd Service
|
||||
|
||||
If using systemd (`pulse-dev` service):
|
||||
|
||||
```bash
|
||||
# Toggle mock mode (restarts service)
|
||||
sudo /opt/pulse/scripts/toggle-mock.sh on
|
||||
sudo /opt/pulse/scripts/toggle-mock.sh off
|
||||
|
||||
# Check status
|
||||
/opt/pulse/scripts/toggle-mock.sh status
|
||||
```
|
||||
|
||||
### Manual Mode
|
||||
|
||||
If running the backend manually:
|
||||
|
||||
```bash
|
||||
# Edit mock.env
|
||||
nano /opt/pulse/mock.env
|
||||
|
||||
# The file watcher will detect changes and auto-reload the backend
|
||||
# within 5 seconds (or immediately if fsnotify is working)
|
||||
```
|
||||
|
||||
## File Watcher Details
|
||||
|
||||
The backend watches `mock.env` using:
|
||||
|
||||
1. **Primary**: `fsnotify` (instant notification on file changes)
|
||||
2. **Fallback**: Polling every 5 seconds (if fsnotify fails)
|
||||
|
||||
When `mock.env` changes:
|
||||
- Environment variables are updated
|
||||
- Monitor is reloaded with new configuration
|
||||
- New mock data is generated
|
||||
- WebSocket clients receive updated state
|
||||
|
||||
**No manual process restarts required!**
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Mock mode not updating
|
||||
|
||||
1. Check that mock.env exists: `ls -la /opt/pulse/mock.env`
|
||||
2. Check file watcher logs: Look for "Detected mock.env file change" in logs
|
||||
3. Verify environment variables: `env | grep PULSE_MOCK`
|
||||
4. Try touching the file: `touch /opt/pulse/mock.env`
|
||||
|
||||
### Backend not reloading
|
||||
|
||||
1. Ensure the `pulse-hot-dev` systemd service is active
|
||||
2. Check for errors in backend logs
|
||||
3. Verify file watcher started successfully
|
||||
4. Fall back to manual restart if needed
|
||||
|
||||
### Missing cluster grouping
|
||||
|
||||
1. Verify mock data includes `isClusterMember` and `clusterName`
|
||||
2. Check API response: `curl http://localhost:7656/api/state | jq '.nodes[0]'`
|
||||
3. Ensure frontend is receiving WebSocket updates
|
||||
|
||||
## Implementation Details
|
||||
|
||||
### Backend Components
|
||||
|
||||
- **Config Watcher** (`internal/config/watcher.go`): Watches both `.env` and `mock.env`
|
||||
- **Mock Integration** (`internal/mock/integration.go`): Manages mock state and updates
|
||||
- **Mock Generator** (`internal/mock/generator.go`): Generates realistic mock data
|
||||
- **Monitor** (`internal/monitoring/monitor.go`): Returns cached mock data when enabled
|
||||
|
||||
### Auto-Reload Flow
|
||||
|
||||
1. User runs `npm run mock:on` (or edits `mock.env`)
|
||||
2. `toggle-mock.sh` updates `mock.env` and touches the file
|
||||
3. Config watcher detects file change (via fsnotify or polling)
|
||||
4. Watcher triggers reload callback
|
||||
5. ReloadableMonitor reloads with fresh config
|
||||
6. New monitor instance starts with updated mock mode
|
||||
7. WebSocket broadcasts new state to connected clients
|
||||
|
||||
### Performance
|
||||
|
||||
- Mock data generation: < 100ms for 7 nodes with 90+ guests
|
||||
- State snapshot: Instant (returns cached data)
|
||||
- Memory usage: ~50MB additional for mock data
|
||||
- Update interval: 2 seconds for metric fluctuations
|
||||
|
||||
## Best Practices
|
||||
|
||||
1. **Use mock mode for frontend development** - Fast, predictable data
|
||||
2. **Test with real data before PRs** - Ensure real infrastructure works
|
||||
3. **Adjust mock config to test edge cases** - High load, many nodes, etc.
|
||||
4. **Use mock.env.local for personal settings** - Your preferences won't be committed
|
||||
5. **Keep mock.env defaults reasonable** - Other developers will use them
|
||||
6. **Document any mock data assumptions** - Help other developers
|
||||
|
||||
## See Also
|
||||
|
||||
- [CONFIGURATION.md](../CONFIGURATION.md) - Production configuration
|
||||
- [TROUBLESHOOTING.md](../TROUBLESHOOTING.md) - Common issues
|
||||
- [API.md](../API.md) - API documentation
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
# Pi-hole Nebula Sync Notes
|
||||
|
||||
- Primary Pi-hole: delly CT 114 (192.168.0.102)
|
||||
Secondary: minipc CT 202 (192.168.0.101)
|
||||
Virtual IP: 192.168.0.100
|
||||
- Runs on the delly host (not inside containers).
|
||||
- Binary: `/usr/local/bin/nebula-sync`; wrapper script: `/usr/local/bin/pihole-sync.sh`.
|
||||
- Cron job (`root@delly`): `*/30 * * * *` → logs written to `/var/log/nebula-sync.log`.
|
||||
- Credentials: `/root/.pihole-sync-credentials` on delly (chmod 600). Request the password from the user if needed.
|
||||
- Both Pi-holes require `app_sudo = true` inside `/etc/pihole/pihole.toml`.
|
||||
- When debugging, coordinate with the user before touching production Pi-hole instances.
|
||||
|
|
@ -1,27 +0,0 @@
|
|||
# mock.env.local.example
|
||||
# Copy this to /opt/pulse/mock.env.local for your personal mock mode settings
|
||||
# The .local file is gitignored and will override values from mock.env
|
||||
#
|
||||
# Usage:
|
||||
# cp docs/development/mock.env.local.example mock.env.local
|
||||
# # Edit mock.env.local with your preferences
|
||||
# npm run mock:on
|
||||
|
||||
# Example: Always start in mock mode for frontend development
|
||||
PULSE_MOCK_MODE=true
|
||||
|
||||
# Example: Use fewer nodes for faster startup
|
||||
PULSE_MOCK_NODES=3
|
||||
|
||||
# Example: More VMs for testing high-density scenarios
|
||||
# PULSE_MOCK_VMS_PER_NODE=10
|
||||
|
||||
# Example: Adjust Docker coverage
|
||||
# PULSE_MOCK_DOCKER_HOSTS=2
|
||||
# PULSE_MOCK_DOCKER_CONTAINERS=6
|
||||
|
||||
# Example: Disable metric fluctuations for consistent testing
|
||||
# PULSE_MOCK_RANDOM_METRICS=false
|
||||
|
||||
# Example: All guests running (no stopped VMs/containers)
|
||||
# PULSE_MOCK_STOPPED_PERCENT=0
|
||||
|
|
@ -1,82 +0,0 @@
|
|||
# Frontend UI Style Guide
|
||||
|
||||
This project now ships a handful of shared primitives to keep typography and form layouts consistent. The snippets below show the preferred usage.
|
||||
|
||||
## Section headers
|
||||
|
||||
Use `SectionHeader` for any inline card titles, modal headings, or sub-section titles instead of ad-hoc `<h2>`/`<h3>` elements.
|
||||
|
||||
```tsx
|
||||
import { SectionHeader } from '@/components/shared/SectionHeader';
|
||||
|
||||
<SectionHeader
|
||||
label="Overview"
|
||||
title="Cluster health"
|
||||
description="Key metrics across every node"
|
||||
size="sm" // sm | md | lg (defaults to md)
|
||||
align="left" // left | center (defaults to left)
|
||||
/>
|
||||
```
|
||||
|
||||
Pass `titleClass`/`descriptionClass` when you need to tweak color or emphasis without rebuilding the layout.
|
||||
|
||||
## Empty states
|
||||
|
||||
Whenever a panel needs to show a loading, error, or "no data" treatment, render `EmptyState` inside a `Card`.
|
||||
|
||||
```tsx
|
||||
import { Card } from '@/components/shared/Card';
|
||||
import { EmptyState } from '@/components/shared/EmptyState';
|
||||
|
||||
<Card padding="lg" tone="info">
|
||||
<EmptyState
|
||||
align="center" // center | left (defaults to center)
|
||||
tone="info" // default | info | success | warning | danger
|
||||
icon={<MyIcon class="h-12 w-12 text-blue-400" />}
|
||||
title="No backups yet"
|
||||
description="Run your first job or adjust the filters to see activity."
|
||||
actions={(
|
||||
<Button onClick={openScheduler}>Open Scheduler</Button>
|
||||
)}
|
||||
/>
|
||||
</Card>
|
||||
```
|
||||
|
||||
Icons and actions are optional; omit them when not needed.
|
||||
|
||||
## Form helpers
|
||||
|
||||
Shared form styles live in `@/components/shared/Form`. Import the helpers and apply them to each field container, label, and control for a uniform look.
|
||||
|
||||
```tsx
|
||||
import { formField, labelClass, controlClass, formHelpText, formCheckbox } from '@/components/shared/Form';
|
||||
|
||||
<div class={formField}>
|
||||
<label class={labelClass('flex items-center gap-2')}>
|
||||
Host URL <span class="text-red-500">*</span>
|
||||
</label>
|
||||
<input
|
||||
type="url"
|
||||
placeholder="https://cluster.example.com:8006"
|
||||
class={controlClass('px-2 py-1.5 font-mono')}
|
||||
/>
|
||||
<p class={`${formHelpText} mt-1`}>
|
||||
Use HTTPS on port 8006 for Proxmox VE and 8007 for PBS.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<label class="flex items-center gap-2 text-sm text-gray-700 dark:text-gray-300">
|
||||
<input type="checkbox" class={formCheckbox} />
|
||||
Enable this integration
|
||||
</label>
|
||||
```
|
||||
|
||||
Helper summary:
|
||||
|
||||
- `formField`: wraps a label + control stack.
|
||||
- `labelClass(extra?)`: base typography for labels, with optional extra classes.
|
||||
- `controlClass(extra?)`: base input styling; append sizing tweaks (`px-2 py-1.5`) as needed.
|
||||
- `formHelpText`: small secondary text (validation notes, hints).
|
||||
- `formCheckbox`: shared checkbox styling for toggles inside copy-heavy forms.
|
||||
|
||||
Stick to these helpers when building new settings panels, modals, or detail cards. If a component needs a variant that the helpers do not cover, extend them in `Form.ts` so the convention remains centralized.
|
||||
|
|
@ -1,61 +0,0 @@
|
|||
# Installer v2 Quick Reference
|
||||
|
||||
## Opt-In / Opt-Out
|
||||
|
||||
```bash
|
||||
# Use the new installer
|
||||
export PULSE_INSTALLER_V2=1
|
||||
curl -fsSL https://download.pulse.example/install-docker-agent.sh | bash -s -- [flags]
|
||||
|
||||
# Force the legacy installer
|
||||
export PULSE_INSTALLER_V2=0
|
||||
curl -fsSL https://download.pulse.example/install-docker-agent.sh | bash -s -- [flags]
|
||||
```
|
||||
|
||||
## Common Flags
|
||||
|
||||
- `--url <https://pulse.example>` — Primary Pulse server URL
|
||||
- `--token <api-token>` — API token for enrollment
|
||||
- `--target <url|token[|insecure]>` — Additional targets (repeatable)
|
||||
- `--interval <duration>` — Poll interval (default `30s`)
|
||||
- `--dry-run` — Show actions without applying changes
|
||||
- `--uninstall` — Remove agent binary, systemd unit, and startup hooks
|
||||
|
||||
## Examples
|
||||
|
||||
```bash
|
||||
# Preview installation without changes
|
||||
export PULSE_INSTALLER_V2=1
|
||||
curl -fsSL https://download.pulse.example/install-docker-agent.sh | bash -s -- \
|
||||
--dry-run \
|
||||
--url https://pulse.example \
|
||||
--token <api-token>
|
||||
|
||||
# Install with two targets and custom interval
|
||||
export PULSE_INSTALLER_V2=1
|
||||
curl -fsSL https://download.pulse.example/install-docker-agent.sh | bash -s -- \
|
||||
--url https://pulse-primary \
|
||||
--token <api-token> \
|
||||
--target https://pulse-dr|<dr-token> \
|
||||
--target https://pulse-edge|<edge-token>|true \
|
||||
--interval 15s
|
||||
|
||||
# Uninstall
|
||||
curl -fsSL https://download.pulse.example/install-docker-agent.sh | bash -s -- --uninstall
|
||||
```
|
||||
|
||||
## Verification
|
||||
|
||||
- Binary path: `/usr/local/bin/pulse-docker-agent`
|
||||
- Systemd unit: `/etc/systemd/system/pulse-docker-agent.service`
|
||||
- Logs: `journalctl -u pulse-docker-agent -f`
|
||||
|
||||
## Rollback
|
||||
|
||||
```bash
|
||||
# Force legacy installer
|
||||
export PULSE_INSTALLER_V2=0
|
||||
curl -fsSL https://download.pulse.example/install-docker-agent.sh | bash -s -- ...
|
||||
```
|
||||
|
||||
Contact support or the Pulse engineering team if issues arise during rollout.
|
||||
|
|
@ -1,260 +0,0 @@
|
|||
# Adaptive Polling Management Endpoints (Future Enhancement)
|
||||
|
||||
## Status: DEFERRED
|
||||
|
||||
**Decision Date:** 2025-10-20
|
||||
**Re-evaluated:** v4.24.0 GA release
|
||||
**Current Status:** Not implemented in v4.24.0
|
||||
|
||||
## Overview
|
||||
|
||||
Manual circuit breaker and dead-letter queue (DLQ) management endpoints are **not included in v4.24.0**. The read-only scheduler health API (`/api/monitoring/scheduler/health`) provides full visibility, and automatic recovery mechanisms have proven sufficient during testing and early production rollouts.
|
||||
|
||||
---
|
||||
|
||||
## What's Available in v4.24.0
|
||||
|
||||
### Read-Only Scheduler Health API
|
||||
|
||||
**Endpoint:** `GET /api/monitoring/scheduler/health`
|
||||
|
||||
Provides complete visibility into:
|
||||
- Queue depth and task distribution
|
||||
- Circuit breaker states per instance
|
||||
- Dead-letter queue contents and retry schedules
|
||||
- Per-instance staleness tracking
|
||||
- Failure streaks and error categorization
|
||||
|
||||
**Documentation:** See [Scheduler Health API](../api/SCHEDULER_HEALTH.md) for complete reference.
|
||||
|
||||
### Existing Management Options
|
||||
|
||||
**v4.24.0 operators can:**
|
||||
|
||||
1. **Toggle adaptive polling** (no restart required)
|
||||
- Via UI: **Settings → System → Monitoring**
|
||||
- Via API: Update `system.json` with `adaptivePollingEnabled: false`
|
||||
|
||||
2. **Service restart** (clears transient state)
|
||||
```bash
|
||||
# Systemd
|
||||
sudo systemctl restart pulse
|
||||
|
||||
# Docker
|
||||
docker restart pulse
|
||||
|
||||
# LXC
|
||||
pct restart <ctid>
|
||||
```
|
||||
- Clears all circuit breakers
|
||||
- Resets DLQ (tasks re-queued with fresh state)
|
||||
- Useful for recovering from stuck states
|
||||
|
||||
3. **Version rollback** (if broader issues)
|
||||
- Via UI: **Settings → System → Updates → Restore previous version**
|
||||
- Via CLI: `pulse config rollback`
|
||||
- Documented in [Operations Runbook](ADAPTIVE_POLLING_ROLLOUT.md)
|
||||
|
||||
4. **Per-instance configuration fixes**
|
||||
- Update node credentials if authentication failures cause DLQ entries
|
||||
- Adjust network/firewall if connectivity issues trigger breakers
|
||||
- Fix underlying infrastructure problems
|
||||
|
||||
---
|
||||
|
||||
## Why Endpoints Are Deferred
|
||||
|
||||
### Test Results Demonstrate Sufficient Automation
|
||||
|
||||
**Integration testing** (55 seconds, 12 instances):
|
||||
- Circuit breakers opened and closed automatically
|
||||
- Transient failures recovered without intervention
|
||||
- Permanent failures correctly routed to DLQ
|
||||
|
||||
**Soak testing** (2-240 minutes, 80 instances):
|
||||
- Heap: 2.3MB → 3.1MB (healthy growth)
|
||||
- Goroutines: 16 → 6 (no leak)
|
||||
- No scenarios requiring manual intervention
|
||||
|
||||
**Production rollout** (v4.24.0):
|
||||
- Automatic recovery working as designed
|
||||
- Service restart sufficient for edge cases
|
||||
- No operator requests for manual controls
|
||||
|
||||
### Implementation Cost vs. Benefit
|
||||
|
||||
**Would require:**
|
||||
- Authentication and RBAC integration
|
||||
- Comprehensive audit logging
|
||||
- UI integration in Settings → System → Monitoring
|
||||
- Additional testing and maintenance burden
|
||||
|
||||
**Current workarounds proven effective:**
|
||||
- Adaptive polling toggle (immediate, no restart)
|
||||
- Service restart (clears all state in < 30 seconds)
|
||||
- Version rollback (if systematic issues)
|
||||
|
||||
---
|
||||
|
||||
## Future Implementation Plan
|
||||
|
||||
### Proposed Endpoints (When Needed)
|
||||
|
||||
If production usage reveals operational gaps, implement:
|
||||
|
||||
#### 1. Reset Circuit Breaker
|
||||
```
|
||||
POST /api/monitoring/breakers/{key}/reset
|
||||
Authorization: Required (session or API token)
|
||||
```
|
||||
|
||||
**Request:**
|
||||
```json
|
||||
{
|
||||
"reason": "Manual reset after infrastructure fix"
|
||||
}
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"key": "pve::pve-node1",
|
||||
"previousState": "open",
|
||||
"newState": "closed",
|
||||
"resetBy": "admin",
|
||||
"resetAt": "2025-10-20T15:30:00Z"
|
||||
}
|
||||
```
|
||||
|
||||
**Use case:** Immediately retry a specific instance after fixing underlying issue (e.g., restored network connectivity)
|
||||
|
||||
#### 2. Retry All DLQ Tasks
|
||||
```
|
||||
POST /api/monitoring/dlq/retry
|
||||
Authorization: Required (session or API token)
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"tasksRetried": 5,
|
||||
"keys": ["pve::pve-node1", "pbs::backup-server"]
|
||||
}
|
||||
```
|
||||
|
||||
**Use case:** Bulk retry after fixing widespread issue (e.g., certificate renewal)
|
||||
|
||||
#### 3. Retry Specific DLQ Task
|
||||
```
|
||||
POST /api/monitoring/dlq/{key}/retry
|
||||
Authorization: Required (session or API token)
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"key": "pve::pve-node1",
|
||||
"previousRetryCount": 5,
|
||||
"scheduledFor": "2025-10-20T15:35:00Z"
|
||||
}
|
||||
```
|
||||
|
||||
**Use case:** Targeted retry of single instance
|
||||
|
||||
#### 4. Remove from DLQ
|
||||
```
|
||||
DELETE /api/monitoring/dlq/{key}
|
||||
Authorization: Required (session or API token)
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"key": "pve::decomissioned-node",
|
||||
"reason": "Instance permanently decommissioned"
|
||||
}
|
||||
```
|
||||
|
||||
**Use case:** Remove decommissioned instances from DLQ
|
||||
|
||||
### Security Requirements
|
||||
|
||||
All management endpoints would require:
|
||||
- **Authentication:** Valid session cookie or API token
|
||||
- **RBAC:** Admin-level permissions
|
||||
- **Audit logging:** Every action logged with:
|
||||
- Operator username/IP
|
||||
- Instance key affected
|
||||
- Reason provided
|
||||
- Timestamp
|
||||
- Previous and new states
|
||||
- **Rate limiting:** Prevent abuse (e.g., 10 requests/minute)
|
||||
|
||||
---
|
||||
|
||||
## Re-evaluation Criteria
|
||||
|
||||
**Implement management endpoints if:**
|
||||
|
||||
1. **Operator demand:** >3 requests in first 60 days of v4.24.0 deployment
|
||||
2. **Service restart frequency:** >5 restarts per week due to stuck breakers/DLQ
|
||||
3. **Incident impact:** Manual controls would have prevented or accelerated recovery from >1 production incident
|
||||
4. **Feedback from operations runbook:** [ADAPTIVE_POLLING_ROLLOUT.md](ADAPTIVE_POLLING_ROLLOUT.md) troubleshooting inadequate
|
||||
|
||||
**Don't implement if:**
|
||||
- Current workarounds remain effective
|
||||
- Automatic recovery continues to handle 99%+ of scenarios
|
||||
- No clear operational pain points emerge
|
||||
|
||||
---
|
||||
|
||||
## Monitoring Current State
|
||||
|
||||
### Check Circuit Breakers
|
||||
```bash
|
||||
curl -s http://<host>:7655/api/monitoring/scheduler/health \
|
||||
| jq '.instances[] | select(.breaker.state != "closed") | {key, state: .breaker.state, since: .breaker.since}'
|
||||
```
|
||||
|
||||
### Check Dead-Letter Queue
|
||||
```bash
|
||||
curl -s http://<host>:7655/api/monitoring/scheduler/health \
|
||||
| jq '.instances[] | select(.deadLetter.present) | {key, reason: .deadLetter.reason, retryCount: .deadLetter.retryCount, nextRetry: .deadLetter.nextRetry}'
|
||||
```
|
||||
|
||||
### Track Recovery Times
|
||||
```bash
|
||||
# Monitor breaker state changes
|
||||
journalctl -u pulse | grep -E "circuit breaker|dead-letter"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Feedback & Requests
|
||||
|
||||
If you encounter scenarios where manual management endpoints would be valuable:
|
||||
|
||||
1. **Document the use case**
|
||||
- What problem occurred?
|
||||
- Why wasn't automatic recovery sufficient?
|
||||
- How would manual control have helped?
|
||||
|
||||
2. **File an issue**
|
||||
- [GitHub Issues](https://github.com/rcourtman/Pulse/issues)
|
||||
- Include: scheduler health API output, logs, timeline
|
||||
|
||||
3. **Track frequency**
|
||||
- If the pattern recurs >3 times, escalate for implementation
|
||||
|
||||
---
|
||||
|
||||
## Related Documentation
|
||||
|
||||
- [Scheduler Health API](../api/SCHEDULER_HEALTH.md) - Complete API reference
|
||||
- [Operations Runbook](ADAPTIVE_POLLING_ROLLOUT.md) - Steady-state operations and troubleshooting
|
||||
- [Adaptive Polling Architecture](../monitoring/ADAPTIVE_POLLING.md) - Technical details
|
||||
- [Configuration Guide](../CONFIGURATION.md) - Adaptive polling settings
|
||||
|
|
@ -1,279 +0,0 @@
|
|||
# Adaptive Polling Operations Runbook
|
||||
|
||||
**GA in v4.24.0 - Enabled by Default**
|
||||
|
||||
This runbook guides operators managing the adaptive polling scheduler in production. Adaptive polling is enabled by default in v4.24.0 and can be toggled via **Settings → System → Monitoring** (no restart) or environment variables (restart required).
|
||||
|
||||
Follow these operational procedures for steady-state monitoring, troubleshooting, and rollback scenarios.
|
||||
|
||||
---
|
||||
|
||||
## 1. Preparation (Before v4.24.0 Deployment)
|
||||
|
||||
**For new deployments or upgrades to v4.24.0:**
|
||||
|
||||
1. **Monitoring readiness**
|
||||
- Review available scrape series in [Prometheus Metrics](../monitoring/PROMETHEUS_METRICS.md).
|
||||
- Set up Grafana dashboard with:
|
||||
- Instance gauges: `pulse_monitor_poll_queue_depth`, `pulse_monitor_poll_staleness_seconds`, `pulse_monitor_poll_last_success_timestamp`
|
||||
- **Per-node coverage:** `pulse_monitor_node_poll_staleness_seconds`, `pulse_monitor_node_poll_errors_total`, `pulse_monitor_node_poll_duration_seconds`
|
||||
- Scheduler health: `pulse_scheduler_queue_depth`, `pulse_scheduler_queue_due_soon`, `pulse_scheduler_dead_letter_depth`, `pulse_scheduler_breaker_state`, and `pulse_scheduler_breaker_failure_count`
|
||||
- Diagnostics cache sanity: `increase(pulse_diagnostics_cache_misses_total[5m])` vs `pulse_diagnostics_cache_hits_total`
|
||||
- HTTP SLA: `rate(pulse_http_request_errors_total{status_class="server_error"}[5m])`
|
||||
- Continue trending `pulse_monitor_poll_total` / `pulse_monitor_poll_errors_total` for throughput
|
||||
- Configure alerts (see §4)
|
||||
|
||||
2. **Baseline metrics**
|
||||
- Record pre-upgrade metrics if upgrading from < v4.24.0:
|
||||
- Typical polling frequency
|
||||
- Average response times
|
||||
- Current alert volumes
|
||||
- These help assess adaptive polling impact
|
||||
|
||||
3. **Rollback readiness**
|
||||
- Verify update rollback workflow: **Settings → System → Updates → Restore previous version**
|
||||
- Test rollback in staging environment
|
||||
- Confirm `/api/monitoring/scheduler/health` accessible
|
||||
- Document emergency disable procedure (see §5)
|
||||
|
||||
4. **Configuration review**
|
||||
- Review `system.json` or environment variables for adaptive polling tunables:
|
||||
- `ADAPTIVE_POLLING_BASE_INTERVAL` (default: 10s)
|
||||
- `ADAPTIVE_POLLING_MIN_INTERVAL` (default: 5s)
|
||||
- `ADAPTIVE_POLLING_MAX_INTERVAL` (default: 5m)
|
||||
- Adjust if needed for your environment (e.g., high-frequency monitoring)
|
||||
|
||||
---
|
||||
|
||||
## 2. Post-Deployment Verification (v4.24.0+)
|
||||
|
||||
**Adaptive polling is enabled by default. Verify it's working correctly:**
|
||||
|
||||
1. **Check scheduler health**
|
||||
```bash
|
||||
curl -s http://<host>:7655/api/monitoring/scheduler/health | jq
|
||||
```
|
||||
|
||||
**Expected response:**
|
||||
- `"enabled": true`
|
||||
- `queue.depth` reasonable (< instances × 1.5)
|
||||
- `deadLetter.count` = 0 (or only known failing instances)
|
||||
- `instances[]` array populated with your nodes
|
||||
- No `breaker.state` stuck in `open` (except known issues)
|
||||
|
||||
2. **Verify UI access**
|
||||
- Navigate to **Settings → System → Monitoring**
|
||||
- Confirm "Adaptive Polling" toggle is ON
|
||||
- Review queue depth and recent poll status
|
||||
|
||||
3. **Check Grafana metrics** (if configured)
|
||||
- `pulse_monitor_poll_queue_depth` shows reasonable values
|
||||
- `pulse_monitor_poll_staleness_seconds` < 60s for healthy instances
|
||||
- `pulse_monitor_poll_errors_total` not rapidly increasing
|
||||
- `pulse_monitor_poll_last_success_timestamp` updating regularly
|
||||
|
||||
4. **Monitor update history**
|
||||
- Check **Settings → System → Updates** for update entry
|
||||
- Verify rollback button is available
|
||||
- Confirm update status shows "completed"
|
||||
|
||||
**Via API:**
|
||||
```bash
|
||||
curl -s http://<host>:7655/api/updates/history | jq '.entries[0]'
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 3. Steady-State Operations
|
||||
|
||||
**Ongoing monitoring and SLO checks:**
|
||||
|
||||
1. **Daily health checks**
|
||||
- Review scheduler health dashboard or API endpoint
|
||||
- Check for:
|
||||
- Queue depth < 50 (alert if > 50 for 10+ minutes)
|
||||
- Instance staleness < 60s for healthy instances, < 120s for critical instances
|
||||
- **Per-node staleness** `pulse_monitor_node_poll_staleness_seconds` < 120s
|
||||
- DLQ count stable (not growing)
|
||||
- Circuit breakers mostly `closed`
|
||||
- Diagnostics cache misses roughly follow hits (`increase(pulse_diagnostics_cache_misses_total[10m])` in line with hits)
|
||||
- HTTP error rate (`rate(pulse_http_request_errors_total{status_class="server_error"}[5m])`) near zero
|
||||
|
||||
2. **Weekly reviews**
|
||||
- Analyze trends in Grafana:
|
||||
- Poll success rates
|
||||
- Average queue depth over time
|
||||
- Circuit breaker trip frequency
|
||||
- Dead-letter queue patterns
|
||||
- Document any recurring issues
|
||||
|
||||
3. **SLO targets**
|
||||
- **Queue depth**: < 1.5× instance count (< 50 typical)
|
||||
- **Staleness**: < 60s for healthy instances, < 120s for critical instances
|
||||
- **Poll success rate**: > 99% for healthy infrastructure
|
||||
- **DLQ growth**: < 5% per week (excluding known failures)
|
||||
- **Circuit breaker recovery**: < 5 minutes for transient failures
|
||||
|
||||
4. **Log correlation**
|
||||
- Cross-reference scheduler health with update history
|
||||
- Check `/api/updates/history` for rollback events correlated with scheduler issues
|
||||
- Review audit logs for adaptive polling configuration changes
|
||||
|
||||
---
|
||||
|
||||
## 4. Grafana & Alert Configuration
|
||||
|
||||
1. **Dashboard panels**
|
||||
- **Queue Depth**: `pulse_monitor_poll_queue_depth`
|
||||
- Use single-stat with alert if > 1.5× active instances for > 10 min
|
||||
- **Instance & Node Staleness**: combine `pulse_monitor_poll_staleness_seconds` and `pulse_monitor_node_poll_staleness_seconds`
|
||||
- Alert threshold: > 60 s for > 5 min (excluding known failing instances)
|
||||
- **Polling Throughput**: rate of `pulse_monitor_poll_total{result="success"}` vs `result="error"`
|
||||
- **Per-node errors**: table or graph of `pulse_monitor_node_poll_errors_total` to spot noisy nodes
|
||||
- **Scheduler Health**: panels for `pulse_scheduler_queue_depth`, `pulse_scheduler_queue_due_soon`, `pulse_scheduler_dead_letter_depth`, `pulse_scheduler_breaker_state`, `pulse_scheduler_breaker_failure_count`
|
||||
- **Diagnostics Cache**: compare `increase(pulse_diagnostics_cache_hits_total[5m])` vs misses so spikes stand out
|
||||
- **HTTP SLA**: `rate(pulse_http_request_errors_total{status_class="server_error"}[5m])`
|
||||
- **Last Success Timestamp** (v4.24.0+): `pulse_monitor_poll_last_success_timestamp` to detect polling gaps
|
||||
|
||||
2. **Alerts**
|
||||
- Queue depth > threshold for >10 min (Warning), >20 min (Critical)
|
||||
- Instance or node staleness > 60 s for >5 min (Critical)
|
||||
- Dead-letter count increase > N (based on baseline) triggers Warning
|
||||
- Any breaker stuck in `open` for >10 min (from `pulse_scheduler_breaker_state`) triggers Critical
|
||||
- Queue wait > 5 s (95th percentile on `pulse_scheduler_queue_wait_seconds`) triggers Warning
|
||||
- Permanent failures (`pulse_monitor_poll_errors_total{category="permanent"}`) trigger immediate Critical
|
||||
- Diagnostics refresh duration > 20 s alongside miss spikes should page engineering (`pulse_diagnostics_refresh_duration_seconds`)
|
||||
|
||||
3. **Notification routing**
|
||||
- Ensure alerts route to on-call + feature owner
|
||||
|
||||
---
|
||||
|
||||
## 5. Rollback Procedures
|
||||
|
||||
### Option A: Disable Adaptive Polling (Keep v4.24.0)
|
||||
|
||||
**If adaptive polling causes issues but you want to keep v4.24.0:**
|
||||
|
||||
1. **Via UI (No restart required)**
|
||||
- Navigate to **Settings → System → Monitoring**
|
||||
- Toggle "Adaptive Polling" OFF
|
||||
- Changes apply immediately
|
||||
|
||||
2. **Via environment variables (Restart required)**
|
||||
```bash
|
||||
# Systemd
|
||||
sudo systemctl edit pulse
|
||||
# Add:
|
||||
[Service]
|
||||
Environment="ADAPTIVE_POLLING_ENABLED=false"
|
||||
|
||||
# Then restart
|
||||
sudo systemctl restart pulse
|
||||
```
|
||||
|
||||
3. **Verification**
|
||||
```bash
|
||||
curl -s http://<host>:7655/api/monitoring/scheduler/health | jq '.enabled'
|
||||
# Should return: false
|
||||
```
|
||||
|
||||
### Option B: Full Version Rollback
|
||||
|
||||
**If v4.24.0 causes broader issues:**
|
||||
|
||||
1. **Via UI**
|
||||
- Navigate to **Settings → System → Updates**
|
||||
- Click **"Restore previous version"**
|
||||
- Confirm rollback
|
||||
- Pulse restarts automatically with previous version
|
||||
|
||||
2. **Via CLI**
|
||||
```bash
|
||||
# Systemd installations
|
||||
sudo /opt/pulse/pulse config rollback
|
||||
|
||||
# LXC containers
|
||||
pct exec <ctid> -- bash -c "cd /opt/pulse && ./pulse config rollback"
|
||||
```
|
||||
|
||||
3. **Verification**
|
||||
```bash
|
||||
# Check version
|
||||
curl -s http://<host>:7655/api/version | jq '.version'
|
||||
|
||||
# Check update history
|
||||
curl -s http://<host>:7655/api/updates/history | jq '.entries[0]'
|
||||
# Should show action="rollback", status="completed"
|
||||
```
|
||||
|
||||
4. **Post-rollback**
|
||||
- Verify rollback logged in update history
|
||||
- Check journal: `journalctl -u pulse | grep rollback`
|
||||
- Monitor for 15-30 minutes to ensure stability
|
||||
- Document rollback reason and notify stakeholders
|
||||
|
||||
---
|
||||
|
||||
## 6. Troubleshooting
|
||||
|
||||
| Symptom | Possible Cause | Action |
|
||||
|---------|----------------|--------|
|
||||
| Queue depth remains high (> 2× usual) | Insufficient workers, hidden breaker, misconfigured flag | Check scheduler health API for breaker states; consider increasing workers or reverting flag |
|
||||
| Staleness spikes across many instances | Backend API slowdown or connectivity issues | Inspect backend logs, network health; revert flag if duration > 15 min |
|
||||
| Dead-letter count climbs rapidly | Downstream API failures | Investigate specific instances via scheduler health API; fix credential/connectivity issues or rollback |
|
||||
| Circuit breakers stuck half-open/open | Persistent transient failures | Review error logs, ensure backoff/rate limits not starving retries; rollback if unresolved quickly |
|
||||
| Grafana panels flatline | Metrics exporter or job issue | Ensure Prometheus scraping working; verify service restarted with flag |
|
||||
|
||||
### Accessing Scheduler Health API
|
||||
|
||||
```bash
|
||||
curl -s http://<host>:7655/api/monitoring/scheduler/health | jq
|
||||
```
|
||||
|
||||
Key sections to inspect:
|
||||
|
||||
- `queue.depth`, `queue.perType`
|
||||
- `instances[].pollStatus` (success/failure streaks and last error)
|
||||
- `instances[].breaker` (current breaker state, retry windows)
|
||||
- `instances[].deadLetter` (reason, retry counts, schedules)
|
||||
- `staleness` (normalized freshness score)
|
||||
|
||||
Common queries:
|
||||
|
||||
**Instances with errors:**
|
||||
```bash
|
||||
curl -s http://<host>:7655/api/monitoring/scheduler/health \
|
||||
| jq '.instances[] | select(.pollStatus.lastError != null) | {key, lastError: .pollStatus.lastError}'
|
||||
```
|
||||
|
||||
**Current dead-letter entries:**
|
||||
```bash
|
||||
curl -s http://<host>:7655/api/monitoring/scheduler/health \
|
||||
| jq '.instances[] | select(.deadLetter.present) | {key, reason: .deadLetter.reason, retryCount: .deadLetter.retryCount}'
|
||||
```
|
||||
|
||||
**Breakers not closed:**
|
||||
```bash
|
||||
curl -s http://<host>:7655/api/monitoring/scheduler/health \
|
||||
| jq '.instances[] | select(.breaker.state != "closed") | {key, breaker: .breaker}'
|
||||
```
|
||||
|
||||
### When to Roll Back
|
||||
|
||||
Rollback immediately if any of the following occurs:
|
||||
- Queue depth > 3× baseline for > 15 min
|
||||
- Staleness > 120 s on majority of instances
|
||||
- Dead-letter count doubles without clear cause
|
||||
- Customer-facing alerts or latency regressions attributed to adaptive polling
|
||||
|
||||
Document the incident and notify stakeholders after rollback.
|
||||
|
||||
---
|
||||
|
||||
## 7. Related Documentation
|
||||
|
||||
- [Scheduler Health API](../api/SCHEDULER_HEALTH.md) - Complete API reference
|
||||
- [Adaptive Polling Architecture](../monitoring/ADAPTIVE_POLLING.md) - Technical details
|
||||
- [Management Endpoints](ADAPTIVE_POLLING_MANAGEMENT_ENDPOINTS.md) - Circuit breaker/DLQ controls
|
||||
- [Configuration Guide](../CONFIGURATION.md) - Adaptive polling settings
|
||||
|
|
@ -1,163 +0,0 @@
|
|||
# Pulse Sensor Proxy Audit Log Rotation
|
||||
|
||||
The sensor proxy writes a tamper-evident audit trail to
|
||||
`/var/log/pulse/sensor-proxy/audit.log`. Every entry includes the SHA-256 hash
|
||||
of the previous entry, so any modification becomes obvious. Because the process
|
||||
keeps the file open and maintains the running hash in memory, rotation requires
|
||||
special handling.
|
||||
|
||||
## Rotation Strategy
|
||||
|
||||
Use `logrotate` to rotate the file once it reaches 100 MB. After each rotation,
|
||||
restart the proxy so it opens a new file and starts a fresh hash chain.
|
||||
|
||||
Create `/etc/logrotate.d/pulse-sensor-proxy` with the following contents:
|
||||
|
||||
```conf
|
||||
/var/log/pulse/sensor-proxy/audit.log {
|
||||
daily
|
||||
size 100M
|
||||
rotate 90
|
||||
compress
|
||||
delaycompress
|
||||
missingok
|
||||
notifempty
|
||||
create 0640 pulse pulse
|
||||
sharedscripts
|
||||
postrotate
|
||||
systemctl restart pulse-sensor-proxy.service >/dev/null 2>&1 || true
|
||||
endscript
|
||||
}
|
||||
```
|
||||
|
||||
### Why a Restart Is Mandatory
|
||||
|
||||
`copytruncate` and similar tricks break the chain integrity. Restarting the
|
||||
service ensures:
|
||||
|
||||
1. The proxy releases the old file descriptor.
|
||||
2. A new hash chain starts at sequence 1 with an all-zero `prev_hash`.
|
||||
|
||||
If the proxy is not restarted, it will continue writing to the renamed file and
|
||||
the rotation will have no effect.
|
||||
|
||||
### Chain Continuity Across Rotations
|
||||
|
||||
Each rotated log (`audit.log.1.gz`, `audit.log.2.gz`, …) is self-contained. To
|
||||
prove continuity between files:
|
||||
|
||||
1. After each rotation, record the final `event_hash` from the rotated file (for
|
||||
example, store it in the filename or a checksum manifest).
|
||||
2. When reviewing logs, verify the `prev_hash` of the first entry in the new
|
||||
file is the zero hash, and reconcile the recorded final hash from the prior
|
||||
file to show no entries were removed.
|
||||
|
||||
Maintaining this “final hash ledger” allows auditors to stitch the rotated files
|
||||
together chronologically while preserving the tamper-evident guarantees.
|
||||
|
||||
### Permissions
|
||||
|
||||
Adjust the `create` directive to match the user and group that run the sensor
|
||||
proxy. The example assumes both user and group are `pulse`.
|
||||
|
||||
---
|
||||
|
||||
## Post-Rotation Health Checks (v4.24.0+)
|
||||
|
||||
**After rotating audit logs and restarting pulse-sensor-proxy, verify adaptive polling health:**
|
||||
|
||||
### 1. Check Scheduler Health
|
||||
|
||||
```bash
|
||||
curl -s http://localhost:7655/api/monitoring/scheduler/health | jq
|
||||
```
|
||||
|
||||
**Verify:**
|
||||
- Temperature proxy pollers appear in `instances[]` array
|
||||
- `pollStatus.lastSuccess` is recent (within last 60 seconds)
|
||||
- No new entries in `deadLetter` queue for proxy instances
|
||||
- `breaker.state` is `closed` for proxy nodes
|
||||
|
||||
**Example check for proxy instances:**
|
||||
```bash
|
||||
curl -s http://localhost:7655/api/monitoring/scheduler/health \
|
||||
| jq '.instances[] | select(.type == "proxy" or .connection | contains("proxy")) | {key, lastSuccess: .pollStatus.lastSuccess, breaker: .breaker.state}'
|
||||
```
|
||||
|
||||
### 2. Monitor Metrics (10-15 minutes)
|
||||
|
||||
Watch these metrics to ensure proxy restart didn't cause issues:
|
||||
|
||||
```bash
|
||||
# Queue depth should remain stable
|
||||
curl -s http://localhost:7655/api/monitoring/scheduler/health | jq '.queue.depth'
|
||||
|
||||
# Check staleness for proxy instances
|
||||
curl -s http://localhost:7655/api/monitoring/scheduler/health \
|
||||
| jq '.instances[] | select(.type == "proxy") | {key, staleness: .pollStatus.lastSuccess}'
|
||||
```
|
||||
|
||||
**Expected behavior:**
|
||||
- Queue depth: No significant spike (< 10 temporary increase acceptable)
|
||||
- Staleness: Proxy instances show fresh polls within 30-60 seconds
|
||||
- No circuit breaker trips for proxy instances
|
||||
- No new DLQ entries
|
||||
|
||||
### 3. Cross-Reference Audit Logs
|
||||
|
||||
**Link rotation events with scheduler health for security review:**
|
||||
|
||||
```bash
|
||||
# Check Pulse audit log for rotation timing
|
||||
journalctl -u pulse-sensor-proxy --since "10 minutes ago" | grep -E "restart|rotation"
|
||||
|
||||
# Check update history for any concurrent events
|
||||
curl -s http://localhost:7655/api/updates/history?limit=5 | jq '.entries[] | {action, timestamp, status}'
|
||||
```
|
||||
|
||||
**Why this matters:**
|
||||
- Security auditors can correlate proxy restarts with scheduler behavior
|
||||
- Update rollbacks may be concurrent with log rotations
|
||||
- Rollback metadata (new in v4.24.0) provides full operational context
|
||||
- Ensures restart didn't mask polling failures or breaker trips
|
||||
|
||||
### 4. Troubleshooting Rotation Issues
|
||||
|
||||
**If proxy instances don't rejoin queue:**
|
||||
|
||||
1. **Check service status**
|
||||
```bash
|
||||
systemctl status pulse-sensor-proxy
|
||||
```
|
||||
|
||||
2. **Verify scheduler sees the proxy**
|
||||
```bash
|
||||
curl -s http://localhost:7655/api/monitoring/scheduler/health \
|
||||
| jq '.instances[] | select(.type == "proxy")'
|
||||
```
|
||||
|
||||
3. **Check for circuit breakers**
|
||||
```bash
|
||||
curl -s http://localhost:7655/api/monitoring/scheduler/health \
|
||||
| jq '.instances[] | select(.breaker.state != "closed") | {key, state: .breaker.state, retryAt: .breaker.retryAt}'
|
||||
```
|
||||
|
||||
4. **Review logs for errors**
|
||||
```bash
|
||||
journalctl -u pulse-sensor-proxy -n 50
|
||||
journalctl -u pulse | grep -E "proxy|temperature"
|
||||
```
|
||||
|
||||
**Recovery actions:**
|
||||
- If breakers are stuck: Restart main Pulse service (`systemctl restart pulse`)
|
||||
- If DLQ entries persist: Check proxy credentials and network connectivity
|
||||
- If polling doesn't resume: Verify proxy configuration in **Settings → Sensors**
|
||||
|
||||
---
|
||||
|
||||
## Related Documentation
|
||||
|
||||
- [Scheduler Health API](../api/SCHEDULER_HEALTH.md) - Complete API reference
|
||||
- [Adaptive Polling Operations](ADAPTIVE_POLLING_ROLLOUT.md) - Health monitoring procedures
|
||||
- [Pulse Sensor Proxy Hardening](../security/pulse-sensor-proxy-hardening.md) - Security configuration
|
||||
- [Temperature Monitoring Security](../TEMPERATURE_MONITORING_SECURITY.md) - Proxy-specific security notes
|
||||
|
|
@ -1,105 +0,0 @@
|
|||
# Pulse Sensor Proxy Runbook
|
||||
|
||||
## Quick Reference
|
||||
- Binary: `/opt/pulse/sensor-proxy/bin/pulse-sensor-proxy`
|
||||
- Unit: `pulse-sensor-proxy.service`
|
||||
- Logs: `/var/log/pulse/sensor-proxy/proxy.log`
|
||||
- Audit trail: `/var/log/pulse/sensor-proxy/audit.log` (hash chained, forwarded via rsyslog)
|
||||
- Metrics: `http://127.0.0.1:9127/metrics` (set `PULSE_SENSOR_PROXY_METRICS_ADDR` to change/disable)
|
||||
- Limiters: 1 request/sec per UID (burst 5), per-UID concurrency 2, global concurrency 8, 2 s penalty on validation failures
|
||||
|
||||
## Monitoring Alerts & Response
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant Backend
|
||||
participant Proxy
|
||||
participant Node
|
||||
|
||||
Backend->>Proxy: get_temperature
|
||||
Proxy->>Proxy: Check rate limit
|
||||
Proxy->>Node: SSH sensors -j
|
||||
Node->>Proxy: JSON response
|
||||
Proxy->>Backend: Temperature data
|
||||
```
|
||||
|
||||
### Rate Limit Hits (`pulse_proxy_limiter_rejections_total`)
|
||||
1. Check audit log entries tagged `limiter.rejection` for offending UID.
|
||||
2. Confirm workload legitimacy; if expected, consider increasing limits via config override.
|
||||
3. If malicious, block source process/user and inspect Pulse audit logs.
|
||||
|
||||
### Penalty Events (`pulse_proxy_limiter_penalties_total`)
|
||||
1. Review corresponding validation failures in audit log (`command.validation_failed`).
|
||||
2. If repeated invalid JSON/unknown methods, inspect caller code for regressions or intrusion attempts.
|
||||
|
||||
### Audit Log Forwarder Down
|
||||
1. `journalctl -u rsyslog` to confirm transmission errors.
|
||||
2. Ensure `/etc/pulse/log-forwarding` certs valid & remote host reachable.
|
||||
3. Forwarding queue stored locally in `/var/log/pulse/sensor-proxy/forwarding.log`; ship manually if outage exceeds 1 hour.
|
||||
|
||||
### Proxy Health Endpoint Fails
|
||||
1. `systemctl status pulse-sensor-proxy`
|
||||
2. Check `/var/log/pulse/sensor-proxy/proxy.log` for panic or limiter exhaustion.
|
||||
3. Inspect `/var/log/pulse/sensor-proxy/audit.log` for recent privileged method denials.
|
||||
|
||||
## Standard Procedures
|
||||
### Restart Proxy Safely
|
||||
```bash
|
||||
sudo systemctl stop pulse-sensor-proxy
|
||||
sudo apparmor_parser -r /etc/apparmor.d/pulse-sensor-proxy # if updating policy
|
||||
sudo systemctl start pulse-sensor-proxy
|
||||
```
|
||||
Verify:
|
||||
```bash
|
||||
# Metrics endpoint exposes proxy build/health
|
||||
curl -s http://127.0.0.1:9127/metrics | grep pulse_proxy_build_info
|
||||
|
||||
# Ensure adaptive polling sees the proxy again
|
||||
curl -s http://localhost:7655/api/monitoring/scheduler/health \
|
||||
| jq '.instances[] | select(.key | contains("temperature")) | {key, pollStatus}'
|
||||
```
|
||||
Temperature instances should show recent `lastSuccess` timestamps with no DLQ entries.
|
||||
|
||||
### Rotate SSH Keys
|
||||
1. Run `scripts/secure-sensor-files.sh` to regenerate keys (ensure environment locked down).
|
||||
2. Use RPC `ensure_cluster_keys` to distribute new public key.
|
||||
3. Confirm nodes accept `ssh` from proxy host.
|
||||
4. Confirm the scheduler clears any temporary breakers/dlq entries:
|
||||
```bash
|
||||
curl -s http://localhost:7655/api/monitoring/scheduler/health \
|
||||
| jq '.instances[] | select(.key | contains("temperature")) | {key, breaker: .breaker.state, deadLetter: .deadLetter.present}'
|
||||
```
|
||||
Expect `breaker.state=="closed"` and `deadLetter.present==false` for all proxy-driven pollers.
|
||||
|
||||
### Rate Limit Tuning
|
||||
|
||||
| Profile | Nodes | `per_peer_interval_ms` | `per_peer_burst` | Notes |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| Default | ≤5 | 1000 | 5 | Shipped with commit 46b8b8d; no action needed for single host clusters. |
|
||||
| Medium | 6–10 | 500 | 10 | Doubles throughput; monitor `pulse_proxy_limiter_rejects_total`. |
|
||||
| Large | 11–20 | 250 | 20 | Confirm proxy CPU stays below 70 % and audit logs remain clean. |
|
||||
| XL | 21–40 | 150 | 30 | Requires high-trust environment; ensure UID filters are locked down. |
|
||||
|
||||
**Procedure:**
|
||||
1. Edit `/etc/pulse-sensor-proxy/config.yaml` and set the desired profile values under `rate_limit`.
|
||||
2. Restart the service:
|
||||
```bash
|
||||
sudo systemctl restart pulse-sensor-proxy
|
||||
```
|
||||
3. Validate:
|
||||
```bash
|
||||
curl -s http://127.0.0.1:9127/metrics \
|
||||
| grep pulse_proxy_limiter_rejects_total
|
||||
```
|
||||
The counter should stop incrementing during steady-state polling.
|
||||
4. Record the change in the operations log and review audit entries for unexpected callers.
|
||||
|
||||
## Incident Handling
|
||||
- **Unauthorized Command Attempt:** audit log shows `command.validation_failed` and limiter penalties; capture correlation ID, check Pulse side for compromised container.
|
||||
- **Excessive Temperature Failures:** refer to `pulse_proxy_ssh_requests_total{result="error"}`; validate network ACLs and node health; escalate to Proxmox team if nodes unreachable.
|
||||
- **Log Tampering Suspected:** verify audit hash chain by replaying `eventHash` values; compare with remote log store (immutable). Trigger security response if mismatch.
|
||||
|
||||
## Postmortem Checklist
|
||||
- Timeline: command audit entries, limiter stats, rsyslog queue depth.
|
||||
- Verify AppArmor/seccomp status (`aa-status`, `systemctl show pulse-sensor-proxy -p AppArmorProfile`).
|
||||
- Ensure firewall ACLs match `docs/security/pulse-sensor-proxy-network.md`.
|
||||
|
|
@ -1,208 +0,0 @@
|
|||
# Script Library Guide
|
||||
|
||||
## 1. Introduction
|
||||
|
||||
The script library system standardises helper functions used across Pulse
|
||||
installers. It reduces duplication, improves testability, and makes it easier to
|
||||
roll out fixes across the installer fleet. Use the shared libraries when:
|
||||
|
||||
- Multiple scripts need the same functionality (logging, HTTP, systemd, etc.).
|
||||
- You are refactoring legacy scripts to adopt the v2 pattern.
|
||||
- New features require reusable helpers (e.g., additional service management).
|
||||
|
||||
## 2. Architecture Overview
|
||||
|
||||
```
|
||||
scripts/
|
||||
├── lib/ # Shared library modules
|
||||
│ ├── common.sh # Core utilities
|
||||
│ ├── systemd.sh # Service management
|
||||
│ ├── http.sh # HTTP/API operations
|
||||
│ └── README.md # API documentation
|
||||
├── tests/ # Test suites
|
||||
│ ├── run.sh # Test runner
|
||||
│ ├── test-*.sh # Smoke tests
|
||||
│ └── integration/ # Integration tests
|
||||
├── bundle.sh # Bundler tool
|
||||
├── bundle.manifest # Bundle configuration
|
||||
└── install-*.sh # Installer scripts
|
||||
|
||||
dist/ # Generated bundled scripts
|
||||
└── install-*.sh # Ready for distribution
|
||||
```
|
||||
|
||||
### Development & Bundling Workflow
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
Code[Write Code<br/>scripts/lib]
|
||||
Test[Run Tests]
|
||||
Bundle[Bundle<br/>make bundle-scripts]
|
||||
Dist[Distribute<br/>dist/*.sh]
|
||||
|
||||
Code --> Test
|
||||
Test --> Bundle
|
||||
Bundle --> Dist
|
||||
```
|
||||
|
||||
This workflow emphasizes the library's modular design: develop reusable modules in `scripts/lib`, test thoroughly, bundle for distribution, and validate bundled artifacts before release.
|
||||
|
||||
## 3. Using the Library in Your Script
|
||||
|
||||
```bash
|
||||
#!/usr/bin/env bash
|
||||
# Example installer using shared libraries
|
||||
|
||||
LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}" )" && pwd)/lib"
|
||||
if [[ -f "$LIB_DIR/common.sh" ]]; then
|
||||
source "$LIB_DIR/common.sh"
|
||||
source "$LIB_DIR/systemd.sh"
|
||||
source "$LIB_DIR/http.sh"
|
||||
fi
|
||||
|
||||
common::init "$@"
|
||||
|
||||
main() {
|
||||
common::ensure_root --allow-sudo --args "$@"
|
||||
common::log_info "Starting installation..."
|
||||
# ...script logic...
|
||||
}
|
||||
|
||||
main "$@"
|
||||
```
|
||||
|
||||
## 4. Common Migration Patterns
|
||||
|
||||
**Logging**
|
||||
```bash
|
||||
# Before
|
||||
echo "[INFO] Installing..."
|
||||
|
||||
# After
|
||||
common::log_info "Installing..."
|
||||
```
|
||||
|
||||
**Privilege Escalation**
|
||||
```bash
|
||||
# Before
|
||||
if [[ $EUID -ne 0 ]]; then
|
||||
echo "Must run as root"; exit 1
|
||||
fi
|
||||
|
||||
# After
|
||||
common::ensure_root --allow-sudo --args "$@"
|
||||
```
|
||||
|
||||
**Downloads**
|
||||
```bash
|
||||
# Before
|
||||
curl -o file.tar.gz http://example.com/file.tar.gz
|
||||
|
||||
# After
|
||||
http::download --url http://example.com/file.tar.gz --output file.tar.gz
|
||||
```
|
||||
|
||||
**Systemd Unit Creation**
|
||||
```bash
|
||||
# Before
|
||||
cat > /etc/systemd/system/my.service <<'EOF'
|
||||
...
|
||||
systemctl daemon-reload
|
||||
systemctl enable my.service
|
||||
systemctl start my.service
|
||||
|
||||
# After
|
||||
systemd::create_service /etc/systemd/system/my.service <<'EOF'
|
||||
...
|
||||
EOF
|
||||
systemd::enable_and_start "my.service"
|
||||
```
|
||||
|
||||
## 5. Creating New Library Modules
|
||||
|
||||
Create a new module when functionality is reused across scripts or complex
|
||||
enough to warrant dedicated helpers. Template:
|
||||
|
||||
```bash
|
||||
#!/usr/bin/env bash
|
||||
# Module: mymodule.sh
|
||||
|
||||
mymodule::do_thing() {
|
||||
local arg="$1"
|
||||
# implementation
|
||||
}
|
||||
```
|
||||
|
||||
Add the module to `scripts/lib`, document exported functions in
|
||||
`scripts/lib/README.md`, and update `scripts/bundle.manifest` for any bundles
|
||||
that need it.
|
||||
|
||||
## 6. Testing Requirements
|
||||
|
||||
Every migrated script must include:
|
||||
|
||||
1. Smoke test (`scripts/tests/test-<script>.sh`) covering syntax and core flows.
|
||||
2. Integration test (when the script modifies system state or talks to services).
|
||||
3. Successful execution of `scripts/tests/run.sh` and relevant integration tests.
|
||||
|
||||
Example smoke test:
|
||||
|
||||
```bash
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT="scripts/my-script.sh"
|
||||
bash -n "$SCRIPT"
|
||||
output="$($SCRIPT --dry-run 2>&1)"
|
||||
[[ -n "$output" ]] || exit 1
|
||||
echo "All my-script tests passed"
|
||||
```
|
||||
|
||||
## 7. Bundling for Distribution
|
||||
|
||||
1. Update `scripts/bundle.manifest` with the module order.
|
||||
2. Run `make bundle-scripts` or `bash scripts/bundle.sh`.
|
||||
3. Validate outputs: `bash -n dist/my-installer.sh` and `dist/my-installer.sh --dry-run`.
|
||||
|
||||
## 8. Code Style Guidelines
|
||||
|
||||
- Namespace exported functions (`module::function`).
|
||||
- Use `common::` helpers whenever applicable.
|
||||
- Quote variables, prefer `[[ ]]` over `[ ]`.
|
||||
- Keep shellcheck clean (`make lint-scripts`).
|
||||
- Internal helpers can use `_module::` or nested `local` functions.
|
||||
|
||||
## 9. Migration Checklist
|
||||
|
||||
- [ ] Create v2 script alongside legacy version.
|
||||
- [ ] Source shared modules and call `common::init`.
|
||||
- [ ] Replace manual logging/privilege escalation with library calls.
|
||||
- [ ] Extract reusable helpers into modules.
|
||||
- [ ] Add `--dry-run` support.
|
||||
- [ ] Write/update smoke and integration tests.
|
||||
- [ ] Update bundle manifest and regenerate bundles.
|
||||
- [ ] Validate bundled artifacts.
|
||||
- [ ] Refresh documentation and release notes.
|
||||
- [ ] Provide before/after metrics in PR.
|
||||
|
||||
## 10. Common Pitfalls
|
||||
|
||||
- Modifying shared modules for a single script — create script-specific helpers.
|
||||
- Forgetting to update bundle manifest or regenerate bundles.
|
||||
- Skipping tests (smoke/integration) before submitting PRs.
|
||||
- Hardcoding paths; prefer variables and configurable directories.
|
||||
- Breaking backwards compatibility without a rollout plan.
|
||||
|
||||
## 11. Examples
|
||||
|
||||
- `scripts/install-docker-agent-v2.sh` — complete migration example.
|
||||
- `scripts/lib/README.md` — full API reference.
|
||||
- `scripts/tests/test-docker-agent-v2.sh` — smoke test pattern.
|
||||
- `scripts/tests/integration/test-docker-agent-install.sh` — integration setup.
|
||||
|
||||
## 12. Getting Help
|
||||
|
||||
- Review existing modules in `scripts/lib` before adding new helpers.
|
||||
- Run `scripts/tests/run.sh` (smoke) and relevant integration tests.
|
||||
- Use `make lint-scripts` to catch style issues early.
|
||||
- Ask in GitHub Discussions or internal Slack with context & logs when blocked.
|
||||
|
|
@ -1,5 +1,6 @@
|
|||
import { createSignal, createMemo, Show, For, onMount, onCleanup, createEffect } from 'solid-js';
|
||||
import { useNavigate, useLocation } from '@solidjs/router';
|
||||
import Toggle from '@/components/shared/Toggle';
|
||||
|
||||
// Workaround for eslint false-positive when `For` is used only in JSX
|
||||
const __ensureForUsage = For;
|
||||
|
|
@ -184,6 +185,8 @@ interface ThresholdsTableProps {
|
|||
restartWindow: number;
|
||||
memoryWarnPct: number;
|
||||
memoryCriticalPct: number;
|
||||
serviceWarnGapPercent: number;
|
||||
serviceCriticalGapPercent: number;
|
||||
};
|
||||
setDockerDefaults: (
|
||||
value:
|
||||
|
|
@ -194,6 +197,8 @@ interface ThresholdsTableProps {
|
|||
restartWindow: number;
|
||||
memoryWarnPct: number;
|
||||
memoryCriticalPct: number;
|
||||
serviceWarnGapPercent: number;
|
||||
serviceCriticalGapPercent: number;
|
||||
}
|
||||
| ((prev: {
|
||||
cpu: number;
|
||||
|
|
@ -202,6 +207,8 @@ interface ThresholdsTableProps {
|
|||
restartWindow: number;
|
||||
memoryWarnPct: number;
|
||||
memoryCriticalPct: number;
|
||||
serviceWarnGapPercent: number;
|
||||
serviceCriticalGapPercent: number;
|
||||
}) => {
|
||||
cpu: number;
|
||||
memory: number;
|
||||
|
|
@ -209,6 +216,8 @@ interface ThresholdsTableProps {
|
|||
restartWindow: number;
|
||||
memoryWarnPct: number;
|
||||
memoryCriticalPct: number;
|
||||
serviceWarnGapPercent: number;
|
||||
serviceCriticalGapPercent: number;
|
||||
}),
|
||||
) => void;
|
||||
dockerIgnoredPrefixes: () => string[];
|
||||
|
|
@ -263,6 +272,8 @@ interface ThresholdsTableProps {
|
|||
setDisableAllPMG: (value: boolean) => void;
|
||||
disableAllDockerHosts: () => boolean;
|
||||
setDisableAllDockerHosts: (value: boolean) => void;
|
||||
disableAllDockerServices: () => boolean;
|
||||
setDisableAllDockerServices: (value: boolean) => void;
|
||||
disableAllDockerContainers: () => boolean;
|
||||
setDisableAllDockerContainers: (value: boolean) => void;
|
||||
// Global disable offline alerts flags
|
||||
|
|
@ -291,20 +302,32 @@ export function ThresholdsTable(props: ThresholdsTableProps) {
|
|||
const [editingThresholds, setEditingThresholds] = createSignal<
|
||||
Record<string, number | undefined>
|
||||
>({});
|
||||
const [activeTab, setActiveTab] = createSignal<'proxmox' | 'pmg' | 'docker'>('proxmox');
|
||||
const [activeTab, setActiveTab] = createSignal<'proxmox' | 'pmg' | 'hosts' | 'docker'>('proxmox');
|
||||
let searchInputRef: HTMLInputElement | undefined;
|
||||
const [dockerIgnoredInput, setDockerIgnoredInput] = createSignal(
|
||||
props.dockerIgnoredPrefixes().join('\n'),
|
||||
);
|
||||
const serviceWarnInputId = 'docker-service-warn-gap';
|
||||
const serviceCriticalInputId = 'docker-service-critical-gap';
|
||||
|
||||
createEffect(() => {
|
||||
setDockerIgnoredInput(props.dockerIgnoredPrefixes().join('\n'));
|
||||
});
|
||||
|
||||
const serviceGapValidationMessage = createMemo(() => {
|
||||
const warn = Number(props.dockerDefaults.serviceWarnGapPercent ?? 0);
|
||||
const crit = Number(props.dockerDefaults.serviceCriticalGapPercent ?? 0);
|
||||
if (crit > 0 && warn > crit) {
|
||||
return 'Critical gap must be greater than or equal to the warning gap when enabled.';
|
||||
}
|
||||
return '';
|
||||
});
|
||||
|
||||
// Determine active tab from URL
|
||||
const getActiveTabFromRoute = (): 'proxmox' | 'pmg' | 'docker' => {
|
||||
const getActiveTabFromRoute = (): 'proxmox' | 'pmg' | 'hosts' | 'docker' => {
|
||||
const path = location.pathname;
|
||||
if (path.includes('/thresholds/docker')) return 'docker';
|
||||
if (path.includes('/thresholds/hosts')) return 'hosts';
|
||||
if (path.includes('/thresholds/mail-gateway')) return 'pmg';
|
||||
return 'proxmox'; // default
|
||||
};
|
||||
|
|
@ -324,10 +347,11 @@ export function ThresholdsTable(props: ThresholdsTableProps) {
|
|||
}
|
||||
});
|
||||
|
||||
const handleTabClick = (tab: 'proxmox' | 'pmg' | 'docker') => {
|
||||
const handleTabClick = (tab: 'proxmox' | 'pmg' | 'hosts' | 'docker') => {
|
||||
const tabRoutes = {
|
||||
proxmox: '/alerts/thresholds/proxmox',
|
||||
pmg: '/alerts/thresholds/mail-gateway',
|
||||
hosts: '/alerts/thresholds/hosts',
|
||||
docker: '/alerts/thresholds/docker',
|
||||
};
|
||||
navigate(tabRoutes[tab]);
|
||||
|
|
@ -648,8 +672,10 @@ export function ThresholdsTable(props: ThresholdsTableProps) {
|
|||
name,
|
||||
displayName: name,
|
||||
rawName: name,
|
||||
type: 'hostAgent',
|
||||
type: 'hostAgent' as const,
|
||||
resourceType: 'Host Agent',
|
||||
node: '',
|
||||
instance: '',
|
||||
status: 'unknown',
|
||||
hasOverride: true,
|
||||
disableConnectivity: override.disableConnectivity || false,
|
||||
|
|
@ -1417,7 +1443,7 @@ export function ThresholdsTable(props: ThresholdsTableProps) {
|
|||
label: 'Host Agents',
|
||||
total: props.hosts?.length ?? 0,
|
||||
overrides: countOverrides(hostAgentsWithOverrides()),
|
||||
tab: 'docker' as const,
|
||||
tab: 'hosts' as const,
|
||||
},
|
||||
{
|
||||
key: 'storage' as const,
|
||||
|
|
@ -2166,6 +2192,17 @@ export function ThresholdsTable(props: ThresholdsTableProps) {
|
|||
>
|
||||
Mail Gateway
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => handleTabClick('hosts')}
|
||||
class={`py-3 px-1 border-b-2 font-medium text-sm transition-colors cursor-pointer ${
|
||||
activeTab() === 'hosts'
|
||||
? 'border-blue-500 text-blue-600 dark:text-blue-400'
|
||||
: 'border-transparent text-gray-500 hover:text-gray-700 hover:border-gray-300 dark:text-gray-400 dark:hover:text-gray-300'
|
||||
}`}
|
||||
>
|
||||
Host Agents
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => handleTabClick('docker')}
|
||||
|
|
@ -2610,7 +2647,7 @@ export function ThresholdsTable(props: ThresholdsTableProps) {
|
|||
</Show>
|
||||
</Show>
|
||||
|
||||
<Show when={activeTab() === 'docker'}>
|
||||
<Show when={activeTab() === 'hosts'}>
|
||||
<Show when={hasSection('hostAgents')}>
|
||||
<div ref={registerSection('hostAgents')} class="scroll-mt-24">
|
||||
<ResourceTable
|
||||
|
|
@ -2635,9 +2672,7 @@ export function ThresholdsTable(props: ThresholdsTableProps) {
|
|||
setGlobalDefaults={props.setHostDefaults}
|
||||
setHasUnsavedChanges={props.setHasUnsavedChanges}
|
||||
globalDisableFlag={props.disableAllHosts}
|
||||
onToggleGlobalDisable={() =>
|
||||
props.setDisableAllHosts(!props.disableAllHosts())
|
||||
}
|
||||
onToggleGlobalDisable={() => props.setDisableAllHosts(!props.disableAllHosts())}
|
||||
globalDisableOfflineFlag={props.disableAllHostsOffline}
|
||||
onToggleGlobalDisableOffline={() =>
|
||||
props.setDisableAllHostsOffline(!props.disableAllHostsOffline())
|
||||
|
|
@ -2647,6 +2682,9 @@ export function ThresholdsTable(props: ThresholdsTableProps) {
|
|||
/>
|
||||
</div>
|
||||
</Show>
|
||||
</Show>
|
||||
|
||||
<Show when={activeTab() === 'docker'}>
|
||||
<div class="mb-6 rounded-lg border border-gray-200 bg-white p-5 shadow-sm dark:border-gray-700 dark:bg-gray-900">
|
||||
<div class="flex flex-col gap-3 sm:flex-row sm:items-center sm:justify-between">
|
||||
<div>
|
||||
|
|
@ -2665,16 +2703,101 @@ export function ThresholdsTable(props: ThresholdsTableProps) {
|
|||
onClick={handleResetDockerIgnored}
|
||||
>
|
||||
Reset
|
||||
</button>
|
||||
</Show>
|
||||
</button>
|
||||
</Show>
|
||||
</div>
|
||||
<textarea
|
||||
value={dockerIgnoredInput()}
|
||||
onInput={(event) => handleDockerIgnoredChange(event.currentTarget.value)}
|
||||
placeholder="runner-"
|
||||
rows={4}
|
||||
class="mt-4 w-full rounded-md border border-gray-300 bg-white p-3 text-sm text-gray-900 shadow-sm focus:border-sky-500 focus:outline-none focus:ring-2 focus:ring-sky-200 dark:border-gray-700 dark:bg-gray-800 dark:text-gray-100 dark:focus:border-sky-400 dark:focus:ring-sky-600/40"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div class="mb-6 rounded-lg border border-gray-200 bg-white p-5 shadow-sm dark:border-gray-700 dark:bg-gray-900">
|
||||
<div class="flex flex-col gap-3 sm:flex-row sm:items-center sm:justify-between">
|
||||
<div>
|
||||
<h3 class="text-sm font-semibold text-gray-900 dark:text-gray-100">Swarm service alerts</h3>
|
||||
<p class="mt-1 text-xs text-gray-600 dark:text-gray-400">
|
||||
Pulse raises alerts when running replicas fall behind the desired count or a rollout gets stuck. Adjust the gap thresholds below or disable service alerts entirely.
|
||||
</p>
|
||||
</div>
|
||||
<Toggle
|
||||
checked={!props.disableAllDockerServices()}
|
||||
onToggle={() => {
|
||||
props.setDisableAllDockerServices(!props.disableAllDockerServices());
|
||||
props.setHasUnsavedChanges(true);
|
||||
}}
|
||||
label={<span class="text-sm font-medium text-gray-900 dark:text-gray-100">Alerts</span>}
|
||||
description={<span class="text-xs text-gray-500 dark:text-gray-400">Toggle Swarm service replica monitoring</span>}
|
||||
size="sm"
|
||||
/>
|
||||
</div>
|
||||
<textarea
|
||||
value={dockerIgnoredInput()}
|
||||
onInput={(event) => handleDockerIgnoredChange(event.currentTarget.value)}
|
||||
placeholder="runner-"
|
||||
rows={4}
|
||||
class="mt-4 w-full rounded-md border border-gray-300 bg-white p-3 text-sm text-gray-900 shadow-sm focus:border-sky-500 focus:outline-none focus:ring-2 focus:ring-sky-200 dark:border-gray-700 dark:bg-gray-800 dark:text-gray-100 dark:focus:border-sky-400 dark:focus:ring-sky-600/40"
|
||||
/>
|
||||
|
||||
<div class="mt-4 grid gap-4 sm:grid-cols-2">
|
||||
<div>
|
||||
<label
|
||||
for={serviceWarnInputId}
|
||||
class="text-xs font-medium uppercase tracking-wide text-gray-600 dark:text-gray-400"
|
||||
>
|
||||
Warning gap %
|
||||
</label>
|
||||
<input
|
||||
type="number"
|
||||
min="0"
|
||||
max="100"
|
||||
id={serviceWarnInputId}
|
||||
value={props.dockerDefaults.serviceWarnGapPercent}
|
||||
onInput={(event) => {
|
||||
const value = Number(event.currentTarget.value);
|
||||
const normalized = Number.isFinite(value) ? Math.max(0, Math.min(100, value)) : 0;
|
||||
props.setDockerDefaults((prev) => ({
|
||||
...prev,
|
||||
serviceWarnGapPercent: normalized,
|
||||
}));
|
||||
props.setHasUnsavedChanges(true);
|
||||
}}
|
||||
class="mt-1 w-full rounded-md border border-gray-300 bg-white p-2 text-sm text-gray-900 shadow-sm focus:border-sky-500 focus:outline-none focus:ring-2 focus:ring-sky-200 dark:border-gray-700 dark:bg-gray-800 dark:text-gray-100 dark:focus:border-sky-400 dark:focus:ring-sky-600/40"
|
||||
/>
|
||||
<p class="mt-1 text-xs text-gray-500 dark:text-gray-400">
|
||||
Convert to warning when at least this percentage of replicas are missing.
|
||||
</p>
|
||||
</div>
|
||||
<div>
|
||||
<label
|
||||
for={serviceCriticalInputId}
|
||||
class="text-xs font-medium uppercase tracking-wide text-gray-600 dark:text-gray-400"
|
||||
>
|
||||
Critical gap %
|
||||
</label>
|
||||
<input
|
||||
type="number"
|
||||
min="0"
|
||||
max="100"
|
||||
id={serviceCriticalInputId}
|
||||
value={props.dockerDefaults.serviceCriticalGapPercent}
|
||||
onInput={(event) => {
|
||||
const value = Number(event.currentTarget.value);
|
||||
const normalized = Number.isFinite(value) ? Math.max(0, Math.min(100, value)) : 0;
|
||||
props.setDockerDefaults((prev) => ({
|
||||
...prev,
|
||||
serviceCriticalGapPercent: normalized,
|
||||
}));
|
||||
props.setHasUnsavedChanges(true);
|
||||
}}
|
||||
class="mt-1 w-full rounded-md border border-gray-300 bg-white p-2 text-sm text-gray-900 shadow-sm focus:border-sky-500 focus:outline-none focus:ring-2 focus:ring-sky-200 dark:border-gray-700 dark:bg-gray-800 dark:text-gray-100 dark:focus:border-sky-400 dark:focus:ring-sky-600/40"
|
||||
/>
|
||||
<p class="mt-1 text-xs text-gray-500 dark:text-gray-400">
|
||||
Raise a critical alert when the missing replica gap meets or exceeds this value.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
{serviceGapValidationMessage() && (
|
||||
<p class="mt-1.5 text-xs font-medium text-red-600 dark:text-red-400">
|
||||
{serviceGapValidationMessage()}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<Show when={hasSection('dockerHosts')}>
|
||||
|
|
|
|||
|
|
@ -1,13 +1,16 @@
|
|||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { render, fireEvent, screen, cleanup } from '@solidjs/testing-library';
|
||||
import { createSignal } from 'solid-js';
|
||||
|
||||
import { ThresholdsTable, normalizeDockerIgnoredInput } from '../ThresholdsTable';
|
||||
import type { PMGThresholdDefaults, SnapshotAlertConfig, BackupAlertConfig } from '@/types/alerts';
|
||||
import type { Host } from '@/types/api';
|
||||
|
||||
let mockPathname = '/alerts/thresholds/docker';
|
||||
|
||||
vi.mock('@solidjs/router', () => ({
|
||||
useNavigate: () => vi.fn(),
|
||||
useLocation: () => ({ pathname: '/alerts/thresholds/docker' }),
|
||||
useLocation: () => ({ pathname: mockPathname }),
|
||||
}));
|
||||
|
||||
vi.mock('../ResourceTable', () => ({
|
||||
|
|
@ -22,6 +25,10 @@ afterEach(() => {
|
|||
cleanup();
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
mockPathname = '/alerts/thresholds/docker';
|
||||
});
|
||||
|
||||
const DEFAULT_PMG_THRESHOLDS: PMGThresholdDefaults = {
|
||||
queueTotalWarning: 100,
|
||||
queueTotalCritical: 200,
|
||||
|
|
@ -48,6 +55,8 @@ const DEFAULT_DOCKER_DEFAULTS = {
|
|||
restartWindow: 300,
|
||||
memoryWarnPct: 90,
|
||||
memoryCriticalPct: 95,
|
||||
serviceWarnGapPercent: 10,
|
||||
serviceCriticalGapPercent: 50,
|
||||
};
|
||||
|
||||
const baseProps = () => ({
|
||||
|
|
@ -87,7 +96,7 @@ const baseProps = () => ({
|
|||
factoryGuestDefaults: {},
|
||||
factoryNodeDefaults: {},
|
||||
factoryHostDefaults: { cpu: 80, memory: 85, disk: 90 },
|
||||
factoryDockerDefaults: {},
|
||||
factoryDockerDefaults: DEFAULT_DOCKER_DEFAULTS,
|
||||
factoryStorageDefault: 85,
|
||||
backupDefaults: () => ({ enabled: false, warningDays: 7, criticalDays: 14 }),
|
||||
setBackupDefaults: vi.fn(),
|
||||
|
|
@ -128,6 +137,8 @@ const baseProps = () => ({
|
|||
setDisableAllPMG: vi.fn(),
|
||||
disableAllDockerHosts: () => false,
|
||||
setDisableAllDockerHosts: vi.fn(),
|
||||
disableAllDockerServices: () => false,
|
||||
setDisableAllDockerServices: vi.fn(),
|
||||
disableAllDockerContainers: () => false,
|
||||
setDisableAllDockerContainers: vi.fn(),
|
||||
disableAllNodesOffline: () => false,
|
||||
|
|
@ -147,6 +158,7 @@ const baseProps = () => ({
|
|||
const renderThresholdsTable = (options?: {
|
||||
initialPrefixes?: string[];
|
||||
includeReset?: boolean;
|
||||
hosts?: Host[];
|
||||
}) => {
|
||||
let setDockerIgnoredPrefixesMock!: ReturnType<typeof vi.fn>;
|
||||
let resetDockerIgnoredPrefixesMock: ReturnType<typeof vi.fn> | undefined;
|
||||
|
|
@ -156,6 +168,7 @@ const renderThresholdsTable = (options?: {
|
|||
const result = render(() => {
|
||||
const [prefixes, setPrefixes] = createSignal(options?.initialPrefixes ?? []);
|
||||
getPrefixes = prefixes;
|
||||
const [dockerDefaults, setDockerDefaultsState] = createSignal({ ...DEFAULT_DOCKER_DEFAULTS });
|
||||
|
||||
setHasUnsavedChangesMock = vi.fn();
|
||||
|
||||
|
|
@ -170,14 +183,33 @@ const renderThresholdsTable = (options?: {
|
|||
setPrefixes([]);
|
||||
});
|
||||
|
||||
const base = baseProps();
|
||||
|
||||
const props = {
|
||||
...baseProps(),
|
||||
...base,
|
||||
hosts: options?.hosts ?? base.hosts,
|
||||
dockerIgnoredPrefixes: () => prefixes(),
|
||||
setDockerIgnoredPrefixes: (value: string[] | ((prev: string[]) => string[])) => {
|
||||
const next = typeof value === 'function' ? value(prefixes()) : value;
|
||||
setDockerIgnoredPrefixesMock(next);
|
||||
setPrefixes(next);
|
||||
},
|
||||
get dockerDefaults() {
|
||||
return dockerDefaults();
|
||||
},
|
||||
setDockerDefaults: (
|
||||
value:
|
||||
| typeof DEFAULT_DOCKER_DEFAULTS
|
||||
| ((
|
||||
prev: typeof DEFAULT_DOCKER_DEFAULTS,
|
||||
) => typeof DEFAULT_DOCKER_DEFAULTS),
|
||||
) => {
|
||||
const next =
|
||||
typeof value === 'function'
|
||||
? value(dockerDefaults())
|
||||
: { ...value };
|
||||
setDockerDefaultsState(next);
|
||||
},
|
||||
setHasUnsavedChanges: (value: boolean) => {
|
||||
setHasUnsavedChangesMock(value);
|
||||
},
|
||||
|
|
@ -210,6 +242,29 @@ describe('normalizeDockerIgnoredInput', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('ThresholdsTable hosts tab', () => {
|
||||
it('renders host agents table when hosts tab is active', () => {
|
||||
mockPathname = '/alerts/thresholds/hosts';
|
||||
const host: Host = {
|
||||
id: 'host-1',
|
||||
hostname: 'host-1.local',
|
||||
displayName: 'Host 1',
|
||||
memory: {
|
||||
total: 1024,
|
||||
used: 512,
|
||||
free: 512,
|
||||
usage: 50,
|
||||
},
|
||||
status: 'online',
|
||||
lastSeen: 1,
|
||||
};
|
||||
|
||||
renderThresholdsTable({ includeReset: false, hosts: [host] });
|
||||
|
||||
expect(screen.getByTestId('resource-table-Host Agents')).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
||||
describe('ThresholdsTable docker ignored prefixes', () => {
|
||||
it('updates prefixes when textarea is edited', () => {
|
||||
const { setDockerIgnoredPrefixesMock, setHasUnsavedChangesMock, getPrefixes } =
|
||||
|
|
@ -245,3 +300,19 @@ describe('ThresholdsTable docker ignored prefixes', () => {
|
|||
expect(setHasUnsavedChangesMock).toHaveBeenCalledWith(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('ThresholdsTable service gap validation', () => {
|
||||
it('shows a validation message when the critical gap falls below the warning gap', () => {
|
||||
renderThresholdsTable({ includeReset: false });
|
||||
|
||||
const warnInput = screen.getByLabelText('Warning gap %') as HTMLInputElement;
|
||||
const critInput = screen.getByLabelText('Critical gap %') as HTMLInputElement;
|
||||
|
||||
fireEvent.input(warnInput, { target: { value: '40' } });
|
||||
fireEvent.input(critInput, { target: { value: '20' } });
|
||||
|
||||
expect(
|
||||
screen.getByText(/critical gap must be greater than or equal to the warning gap/i),
|
||||
).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -7,6 +7,8 @@ import { createSearchHistoryManager } from '@/utils/searchHistory';
|
|||
interface HostsFilterProps {
|
||||
search: () => string;
|
||||
setSearch: (value: string) => void;
|
||||
statusFilter: () => 'all' | 'online' | 'degraded' | 'offline';
|
||||
setStatusFilter: (value: 'all' | 'online' | 'degraded' | 'offline') => void;
|
||||
searchInputRef?: (el: HTMLInputElement) => void;
|
||||
onReset?: () => void;
|
||||
activeHostName?: string;
|
||||
|
|
@ -85,11 +87,13 @@ export const HostsFilter: Component<HostsFilterProps> = (props) => {
|
|||
const hasActiveFilters = createMemo(
|
||||
() =>
|
||||
props.search().trim() !== '' ||
|
||||
Boolean(props.activeHostName),
|
||||
Boolean(props.activeHostName) ||
|
||||
props.statusFilter() !== 'all',
|
||||
);
|
||||
|
||||
const handleReset = () => {
|
||||
props.setSearch('');
|
||||
props.setStatusFilter('all');
|
||||
props.onClearHost?.();
|
||||
props.onReset?.();
|
||||
closeHistory();
|
||||
|
|
@ -291,6 +295,69 @@ export const HostsFilter: Component<HostsFilterProps> = (props) => {
|
|||
</div>
|
||||
</div>
|
||||
|
||||
<div class="flex flex-wrap items-center gap-2">
|
||||
<div class="inline-flex rounded-lg bg-gray-100 p-0.5 dark:bg-gray-700">
|
||||
<button
|
||||
type="button"
|
||||
aria-pressed={props.statusFilter() === 'all'}
|
||||
onClick={() => props.setStatusFilter('all')}
|
||||
class={`px-2.5 py-1 text-xs font-medium rounded-md transition-all ${
|
||||
props.statusFilter() === 'all'
|
||||
? 'bg-white dark:bg-gray-800 text-gray-900 dark:text-gray-100 shadow-sm'
|
||||
: 'text-gray-600 dark:text-gray-400 hover:text-gray-900 dark:hover:text-gray-100'
|
||||
}`}
|
||||
title="Show all hosts"
|
||||
>
|
||||
All
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
aria-pressed={props.statusFilter() === 'online'}
|
||||
onClick={() =>
|
||||
props.setStatusFilter(props.statusFilter() === 'online' ? 'all' : 'online')
|
||||
}
|
||||
class={`px-2.5 py-1 text-xs font-medium rounded-md transition-all ${
|
||||
props.statusFilter() === 'online'
|
||||
? 'bg-white dark:bg-gray-800 text-green-600 dark:text-green-400 shadow-sm'
|
||||
: 'text-gray-600 dark:text-gray-400 hover:text-gray-900 dark:hover:text-gray-100'
|
||||
}`}
|
||||
title="Show online hosts only"
|
||||
>
|
||||
Online
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
aria-pressed={props.statusFilter() === 'degraded'}
|
||||
onClick={() =>
|
||||
props.setStatusFilter(props.statusFilter() === 'degraded' ? 'all' : 'degraded')
|
||||
}
|
||||
class={`px-2.5 py-1 text-xs font-medium rounded-md transition-all ${
|
||||
props.statusFilter() === 'degraded'
|
||||
? 'bg-white dark:bg-gray-800 text-amber-600 dark:text-amber-400 shadow-sm'
|
||||
: 'text-gray-600 dark:text-gray-400 hover:text-gray-900 dark:hover:text-gray-100'
|
||||
}`}
|
||||
title="Show degraded hosts only"
|
||||
>
|
||||
Degraded
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
aria-pressed={props.statusFilter() === 'offline'}
|
||||
onClick={() =>
|
||||
props.setStatusFilter(props.statusFilter() === 'offline' ? 'all' : 'offline')
|
||||
}
|
||||
class={`px-2.5 py-1 text-xs font-medium rounded-md transition-all ${
|
||||
props.statusFilter() === 'offline'
|
||||
? 'bg-white dark:bg-gray-800 text-red-600 dark:text-red-400 shadow-sm'
|
||||
: 'text-gray-600 dark:text-gray-400 hover:text-gray-900 dark:hover:text-gray-100'
|
||||
}`}
|
||||
title="Show offline hosts only"
|
||||
>
|
||||
Offline
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="flex flex-wrap items-center gap-2">
|
||||
<Show when={props.activeHostName}>
|
||||
<div class="flex items-center gap-1 rounded-full bg-blue-50 px-2 py-1 text-xs font-medium text-blue-700 dark:bg-blue-900/40 dark:text-blue-200">
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ export const HostsOverview: Component<HostsOverviewProps> = (props) => {
|
|||
const navigate = useNavigate();
|
||||
const wsContext = useWebSocket();
|
||||
const [search, setSearch] = createSignal('');
|
||||
const [statusFilter, setStatusFilter] = createSignal<'all' | 'online' | 'degraded' | 'offline'>('all');
|
||||
|
||||
// Keyboard listener to auto-focus search
|
||||
let searchInputRef: HTMLInputElement | undefined;
|
||||
|
|
@ -102,8 +103,15 @@ export const HostsOverview: Component<HostsOverviewProps> = (props) => {
|
|||
);
|
||||
};
|
||||
|
||||
const matchesStatus = (host: Host) => {
|
||||
const filter = statusFilter();
|
||||
if (filter === 'all') return true;
|
||||
const normalized = (host.status || '').toLowerCase();
|
||||
return normalized === filter;
|
||||
};
|
||||
|
||||
const filteredHosts = createMemo(() => {
|
||||
return sortedHosts().filter(matchesSearch);
|
||||
return sortedHosts().filter((host) => matchesSearch(host) && matchesStatus(host));
|
||||
});
|
||||
|
||||
return (
|
||||
|
|
@ -154,6 +162,8 @@ export const HostsOverview: Component<HostsOverviewProps> = (props) => {
|
|||
<HostsFilter
|
||||
search={search}
|
||||
setSearch={setSearch}
|
||||
statusFilter={statusFilter}
|
||||
setStatusFilter={setStatusFilter}
|
||||
onReset={() => setSearch('')}
|
||||
searchInputRef={(el) => (searchInputRef = el)}
|
||||
/>
|
||||
|
|
|
|||
|
|
@ -43,6 +43,20 @@ export const DockerAgents: Component = () => {
|
|||
const [latestRecord, setLatestRecord] = createSignal<APITokenRecord | null>(null);
|
||||
const [tokenName, setTokenName] = createSignal('');
|
||||
|
||||
const [expandedHosts, setExpandedHosts] = createSignal<Set<string>>(new Set());
|
||||
const isHostExpanded = (id: string) => expandedHosts().has(id);
|
||||
const toggleHostExpanded = (id: string) => {
|
||||
setExpandedHosts((prev) => {
|
||||
const next = new Set(prev);
|
||||
if (next.has(id)) {
|
||||
next.delete(id);
|
||||
} else {
|
||||
next.add(id);
|
||||
}
|
||||
return next;
|
||||
});
|
||||
};
|
||||
|
||||
const pulseUrl = () => {
|
||||
if (typeof window !== 'undefined') {
|
||||
const protocol = window.location.protocol;
|
||||
|
|
@ -865,6 +879,24 @@ WantedBy=multi-user.target`;
|
|||
const commandInProgress = commandStatus === 'queued' || commandStatus === 'dispatched' || commandStatus === 'acknowledged';
|
||||
const commandFailed = commandStatus === 'failed';
|
||||
const commandCompleted = commandStatus === 'completed';
|
||||
const services = host.services ?? [];
|
||||
const serviceCount = services.length;
|
||||
const taskCount = host.tasks?.length ?? 0;
|
||||
const swarm = host.swarm;
|
||||
const expanded = isHostExpanded(host.id);
|
||||
const unhealthyServiceCount = services.filter(service => {
|
||||
const desired = service.desiredTasks ?? 0;
|
||||
const running = service.runningTasks ?? 0;
|
||||
return desired > 0 && running < desired;
|
||||
}).length;
|
||||
const missingReplicaCount = services.reduce((total, service) => {
|
||||
const desired = service.desiredTasks ?? 0;
|
||||
const running = service.runningTasks ?? 0;
|
||||
if (desired <= 0 || running >= desired) {
|
||||
return total;
|
||||
}
|
||||
return total + (desired - running);
|
||||
}, 0);
|
||||
const offlineActionLabel = commandFailed
|
||||
? 'Force remove host'
|
||||
: host.pendingUninstall
|
||||
|
|
@ -872,20 +904,66 @@ WantedBy=multi-user.target`;
|
|||
: 'Remove offline host';
|
||||
|
||||
return (
|
||||
<tr class={`${isOnline ? 'bg-white dark:bg-gray-900' : 'bg-gray-50 dark:bg-gray-800/50 opacity-60'}`}>
|
||||
<td class="py-3 px-4">
|
||||
<div class="font-medium text-gray-900 dark:text-gray-100">{host.displayName}</div>
|
||||
<div class="text-xs text-gray-500 dark:text-gray-400">{host.hostname}</div>
|
||||
<Show when={host.os || host.architecture}>
|
||||
<div class="text-xs text-gray-400 dark:text-gray-500 mt-1">
|
||||
{host.os}
|
||||
<Show when={host.os && host.architecture}>
|
||||
<span class="mx-1">•</span>
|
||||
</Show>
|
||||
{host.architecture}
|
||||
</div>
|
||||
</Show>
|
||||
</td>
|
||||
<>
|
||||
<tr class={`${isOnline ? 'bg-white dark:bg-gray-900' : 'bg-gray-50 dark:bg-gray-800/50 opacity-60'}`}>
|
||||
<td class="py-3 px-4">
|
||||
<div class="font-medium text-gray-900 dark:text-gray-100">{host.displayName}</div>
|
||||
<div class="text-xs text-gray-500 dark:text-gray-400">{host.hostname}</div>
|
||||
<Show when={host.os || host.architecture}>
|
||||
<div class="text-xs text-gray-400 dark:text-gray-500 mt-1">
|
||||
{host.os}
|
||||
<Show when={host.os && host.architecture}>
|
||||
<span class="mx-1">•</span>
|
||||
</Show>
|
||||
{host.architecture}
|
||||
</div>
|
||||
</Show>
|
||||
<Show when={swarm || serviceCount > 0}>
|
||||
<div class="mt-2 flex flex-wrap items-center gap-2 text-xs text-gray-500 dark:text-gray-400">
|
||||
<Show when={swarm}>
|
||||
<span
|
||||
class={`inline-flex items-center gap-1 px-2 py-0.5 rounded-full text-[10px] font-semibold uppercase tracking-wide ${
|
||||
swarm?.nodeRole === 'manager'
|
||||
? 'bg-blue-100 text-blue-700 dark:bg-blue-900/50 dark:text-blue-200'
|
||||
: 'bg-gray-200 text-gray-700 dark:bg-gray-700 dark:text-gray-200'
|
||||
}`}
|
||||
>
|
||||
{swarm?.nodeRole ?? 'unknown'}
|
||||
</span>
|
||||
<span>scope: {swarm?.scope ?? 'node'}</span>
|
||||
</Show>
|
||||
<span>services: {serviceCount}</span>
|
||||
<span>tasks: {taskCount}</span>
|
||||
<Show when={serviceCount > 0}>
|
||||
<span
|
||||
class={`inline-flex items-center gap-1 rounded-full px-2 py-0.5 text-[10px] font-semibold uppercase tracking-wide ${
|
||||
unhealthyServiceCount > 0
|
||||
? 'bg-red-100 text-red-700 dark:bg-red-900/40 dark:text-red-300'
|
||||
: 'bg-emerald-100 text-emerald-700 dark:bg-emerald-900/40 dark:text-emerald-200'
|
||||
}`}
|
||||
title={
|
||||
unhealthyServiceCount > 0
|
||||
? `${unhealthyServiceCount} service(s) missing ${missingReplicaCount} replica(s)`
|
||||
: 'All reported services are meeting their desired replica count'
|
||||
}
|
||||
>
|
||||
{unhealthyServiceCount > 0
|
||||
? `${unhealthyServiceCount} degraded`
|
||||
: 'All healthy'}
|
||||
</span>
|
||||
</Show>
|
||||
</div>
|
||||
</Show>
|
||||
<Show when={swarm || serviceCount > 0 || taskCount > 0}>
|
||||
<button
|
||||
type="button"
|
||||
class="mt-2 text-xs font-semibold text-blue-600 hover:text-blue-700 focus:outline-none disabled:opacity-50"
|
||||
onClick={() => toggleHostExpanded(host.id)}
|
||||
>
|
||||
{expanded ? 'Hide details' : 'Show details'}
|
||||
</button>
|
||||
</Show>
|
||||
</td>
|
||||
<td class="py-3 px-4">
|
||||
<div class="flex items-center gap-2">
|
||||
<span
|
||||
|
|
@ -1007,8 +1085,173 @@ WantedBy=multi-user.target`;
|
|||
</Show>
|
||||
</td>
|
||||
</tr>
|
||||
);
|
||||
}}
|
||||
<Show when={expanded}>
|
||||
<tr class="bg-gray-50 dark:bg-gray-800/40">
|
||||
<td colSpan={7} class="px-4 py-4">
|
||||
<div class="space-y-4 text-sm">
|
||||
<Show when={swarm}>
|
||||
<div class="rounded-lg border border-sky-100 bg-sky-50 px-3 py-2 text-xs text-sky-900 dark:border-sky-800 dark:bg-sky-900/40 dark:text-sky-100">
|
||||
<div class="flex flex-wrap gap-3">
|
||||
<div><span class="font-semibold">Role:</span> {swarm?.nodeRole ?? 'unknown'}</div>
|
||||
<div><span class="font-semibold">Scope:</span> {swarm?.scope ?? 'node'}</div>
|
||||
<Show when={swarm?.clusterName || swarm?.clusterId}>
|
||||
<div><span class="font-semibold">Cluster:</span> {swarm?.clusterName || swarm?.clusterId}</div>
|
||||
</Show>
|
||||
<Show when={swarm?.error}>
|
||||
<div class="text-red-600 dark:text-red-300"><span class="font-semibold">Error:</span> {swarm?.error}</div>
|
||||
</Show>
|
||||
</div>
|
||||
</div>
|
||||
</Show>
|
||||
|
||||
<Show when={serviceCount > 0}>
|
||||
<div class="space-y-2">
|
||||
<h4 class="text-sm font-semibold text-gray-900 dark:text-gray-100">Services ({serviceCount})</h4>
|
||||
<div class="grid gap-3 md:grid-cols-2 xl:grid-cols-3">
|
||||
<For each={host.services}>
|
||||
{(service) => {
|
||||
const desired = service.desiredTasks ?? 0;
|
||||
const running = service.runningTasks ?? 0;
|
||||
const completed = service.completedTasks;
|
||||
const missing = desired > running ? desired - running : 0;
|
||||
const statusLabel =
|
||||
desired <= 0
|
||||
? 'Idle'
|
||||
: missing === 0
|
||||
? 'Healthy'
|
||||
: missing >= desired
|
||||
? 'Down'
|
||||
: 'Degraded';
|
||||
const statusColor =
|
||||
missing === 0
|
||||
? 'bg-emerald-100 text-emerald-700 dark:bg-emerald-900/40 dark:text-emerald-200'
|
||||
: 'bg-red-100 text-red-700 dark:bg-red-900/40 dark:text-red-300';
|
||||
|
||||
return (
|
||||
<div class="rounded-lg border border-gray-200 bg-white p-3 shadow-sm dark:border-gray-700 dark:bg-gray-900">
|
||||
<div class="flex items-center justify-between gap-2">
|
||||
<div class="font-semibold text-gray-900 dark:text-gray-100 truncate">{service.name || service.id}</div>
|
||||
<div class="flex items-center gap-2">
|
||||
<Show when={service.mode}>
|
||||
<span class="text-[10px] uppercase tracking-wide text-gray-500 dark:text-gray-400">
|
||||
{service.mode}
|
||||
</span>
|
||||
</Show>
|
||||
<span
|
||||
class={`inline-flex items-center rounded-full px-2 py-0.5 text-[10px] font-semibold uppercase tracking-wide ${statusColor}`}
|
||||
>
|
||||
{statusLabel}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="mt-2 space-y-1 text-xs text-gray-500 dark:text-gray-400">
|
||||
<div>
|
||||
Tasks: {running} / {desired}
|
||||
<Show when={missing > 0}>
|
||||
<span class="ml-1 text-red-600 dark:text-red-300">(-{missing})</span>
|
||||
</Show>
|
||||
</div>
|
||||
<Show when={completed !== undefined}>
|
||||
<div>Completed: {completed}</div>
|
||||
</Show>
|
||||
<Show when={service.stack}>
|
||||
<div>Stack: {service.stack}</div>
|
||||
</Show>
|
||||
</div>
|
||||
<Show when={service.image}>
|
||||
<div class="mt-2 text-xs text-gray-500 dark:text-gray-400 truncate">{service.image}</div>
|
||||
</Show>
|
||||
<Show when={(service.endpointPorts?.length ?? 0) > 0}>
|
||||
<div class="mt-2 text-xs text-gray-500 dark:text-gray-400">
|
||||
Ports:{' '}
|
||||
<For each={service.endpointPorts}>
|
||||
{(port, index) => (
|
||||
<span>
|
||||
{index() > 0 && ', '}
|
||||
{port.publishedPort ?? '—'} → {port.targetPort ?? '—'} {port.protocol ? port.protocol.toUpperCase() : ''}
|
||||
</span>
|
||||
)}
|
||||
</For>
|
||||
</div>
|
||||
</Show>
|
||||
<Show when={service.updateStatus?.state || service.updateStatus?.message}>
|
||||
<div class="mt-2 text-xs text-gray-500 dark:text-gray-400">
|
||||
Update: {service.updateStatus?.state || 'pending'}
|
||||
<Show when={service.updateStatus?.message}>
|
||||
<span class="ml-1">({service.updateStatus?.message})</span>
|
||||
</Show>
|
||||
</div>
|
||||
</Show>
|
||||
</div>
|
||||
);
|
||||
}}
|
||||
</For>
|
||||
</div>
|
||||
</div>
|
||||
</Show>
|
||||
|
||||
<Show when={taskCount > 0}>
|
||||
<div class="space-y-2">
|
||||
<h4 class="text-sm font-semibold text-gray-900 dark:text-gray-100">Tasks ({taskCount})</h4>
|
||||
<div class="overflow-x-auto rounded-lg border border-gray-200 dark:border-gray-700">
|
||||
<table class="min-w-full text-xs">
|
||||
<thead class="bg-gray-50 text-gray-600 dark:bg-gray-800 dark:text-gray-300">
|
||||
<tr>
|
||||
<th class="px-3 py-2 text-left font-semibold">Service</th>
|
||||
<th class="px-3 py-2 text-left font-semibold">Slot</th>
|
||||
<th class="px-3 py-2 text-left font-semibold">State</th>
|
||||
<th class="px-3 py-2 text-left font-semibold">Container</th>
|
||||
<th class="px-3 py-2 text-left font-semibold">Updated</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody class="divide-y divide-gray-200 dark:divide-gray-700">
|
||||
<For each={host.tasks}>
|
||||
{(task) => (
|
||||
<tr class="bg-white dark:bg-gray-900">
|
||||
<td class="px-3 py-2 text-gray-700 dark:text-gray-300">{task.serviceName || task.serviceId || '—'}</td>
|
||||
<td class="px-3 py-2 text-gray-700 dark:text-gray-300">{task.slot ?? '—'}</td>
|
||||
<td class="px-3 py-2">
|
||||
<span
|
||||
class={`inline-flex items-center px-2 py-0.5 rounded-full text-[10px] font-semibold ${
|
||||
task.currentState?.toLowerCase() === 'running'
|
||||
? 'bg-green-100 text-green-700 dark:bg-green-900/40 dark:text-green-300'
|
||||
: 'bg-gray-200 text-gray-700 dark:bg-gray-700 dark:text-gray-200'
|
||||
}`}
|
||||
>
|
||||
{task.currentState ?? 'unknown'}
|
||||
</span>
|
||||
</td>
|
||||
<td class="px-3 py-2 text-gray-700 dark:text-gray-300">
|
||||
{task.containerName || task.containerId?.slice(0, 12) || '—'}
|
||||
</td>
|
||||
<td class="px-3 py-2 text-gray-500 dark:text-gray-400">
|
||||
{task.updatedAt
|
||||
? formatRelativeTime(task.updatedAt)
|
||||
: task.startedAt
|
||||
? formatRelativeTime(task.startedAt)
|
||||
: '—'}
|
||||
</td>
|
||||
</tr>
|
||||
)}
|
||||
</For>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</Show>
|
||||
|
||||
<Show when={serviceCount === 0 && taskCount === 0}>
|
||||
<div class="text-xs text-gray-500 dark:text-gray-400">
|
||||
No Swarm services or tasks reported yet.
|
||||
</div>
|
||||
</Show>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
</Show>
|
||||
</>
|
||||
);
|
||||
}}
|
||||
</For>
|
||||
</tbody>
|
||||
</table>
|
||||
|
|
|
|||
|
|
@ -700,7 +700,6 @@ sudo systemctl daemon-reload`;
|
|||
const [isDeleting, setIsDeleting] = createSignal(false);
|
||||
const tokenRevokedAt = host.tokenRevokedAt;
|
||||
const tokenRevoked = typeof tokenRevokedAt === 'number';
|
||||
const tokenRevokedRelative = tokenRevokedAt ? formatRelativeTime(tokenRevokedAt) : '';
|
||||
const lastSeenMs = host.lastSeen ? new Date(host.lastSeen).getTime() : null;
|
||||
const expectedIntervalMs =
|
||||
(host.intervalSeconds && host.intervalSeconds > 0 ? host.intervalSeconds : 30) * 1000;
|
||||
|
|
|
|||
|
|
@ -108,6 +108,7 @@ interface DestinationsRef {
|
|||
type OverrideType =
|
||||
| 'guest'
|
||||
| 'node'
|
||||
| 'hostAgent'
|
||||
| 'storage'
|
||||
| 'pbs'
|
||||
| 'pmg'
|
||||
|
|
@ -123,7 +124,7 @@ interface Override {
|
|||
node?: string; // Node name (for guests and storage), undefined for nodes themselves
|
||||
instance?: string;
|
||||
disabled?: boolean; // Completely disable alerts for this guest/storage
|
||||
disableConnectivity?: boolean; // For nodes - disable offline/connectivity alerts
|
||||
disableConnectivity?: boolean; // For nodes/hosts - disable offline/connectivity alerts
|
||||
poweredOffSeverity?: 'warning' | 'critical';
|
||||
thresholds: {
|
||||
cpu?: number;
|
||||
|
|
@ -134,6 +135,7 @@ interface Override {
|
|||
networkIn?: number;
|
||||
networkOut?: number;
|
||||
usage?: number; // For storage devices
|
||||
temperature?: number;
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -817,6 +819,26 @@ const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
|
|||
}
|
||||
|
||||
if (config.dockerDefaults) {
|
||||
const normalizeGap = (value: unknown, fallback: number) => {
|
||||
const numeric = Number(value);
|
||||
if (!Number.isFinite(numeric)) {
|
||||
return fallback;
|
||||
}
|
||||
return Math.max(0, Math.min(100, numeric));
|
||||
};
|
||||
|
||||
const serviceWarnGap = normalizeGap(
|
||||
config.dockerDefaults.serviceWarnGapPercent,
|
||||
FACTORY_DOCKER_DEFAULTS.serviceWarnGapPercent,
|
||||
);
|
||||
let serviceCriticalGap = normalizeGap(
|
||||
config.dockerDefaults.serviceCriticalGapPercent,
|
||||
FACTORY_DOCKER_DEFAULTS.serviceCriticalGapPercent,
|
||||
);
|
||||
if (serviceCriticalGap > 0 && serviceWarnGap > serviceCriticalGap) {
|
||||
serviceCriticalGap = serviceWarnGap;
|
||||
}
|
||||
|
||||
setDockerDefaults({
|
||||
cpu: getTriggerValue(config.dockerDefaults.cpu) ?? 80,
|
||||
memory: getTriggerValue(config.dockerDefaults.memory) ?? 85,
|
||||
|
|
@ -824,7 +846,11 @@ const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
|
|||
restartWindow: config.dockerDefaults.restartWindow ?? 300,
|
||||
memoryWarnPct: config.dockerDefaults.memoryWarnPct ?? 90,
|
||||
memoryCriticalPct: config.dockerDefaults.memoryCriticalPct ?? 95,
|
||||
serviceWarnGapPercent: serviceWarnGap,
|
||||
serviceCriticalGapPercent: serviceCriticalGap,
|
||||
});
|
||||
} else {
|
||||
setDockerDefaults({ ...FACTORY_DOCKER_DEFAULTS });
|
||||
}
|
||||
setDockerIgnoredPrefixes(config.dockerIgnoredContainerPrefixes ?? []);
|
||||
|
||||
|
|
@ -928,6 +954,7 @@ const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
|
|||
setDisableAllPBS(config.disableAllPBS ?? false);
|
||||
setDisableAllPMG(config.disableAllPMG ?? false);
|
||||
setDisableAllDockerHosts(config.disableAllDockerHosts ?? false);
|
||||
setDisableAllDockerServices(config.disableAllDockerServices ?? false);
|
||||
setDisableAllDockerContainers(config.disableAllDockerContainers ?? false);
|
||||
|
||||
// Load global disable offline alerts flags
|
||||
|
|
@ -1176,6 +1203,8 @@ const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
|
|||
restartWindow: 300,
|
||||
memoryWarnPct: 90,
|
||||
memoryCriticalPct: 95,
|
||||
serviceWarnGapPercent: 10,
|
||||
serviceCriticalGapPercent: 50,
|
||||
};
|
||||
|
||||
const FACTORY_STORAGE_DEFAULT = 85;
|
||||
|
|
@ -1284,6 +1313,7 @@ const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
|
|||
const [disableAllPBS, setDisableAllPBS] = createSignal(false);
|
||||
const [disableAllPMG, setDisableAllPMG] = createSignal(false);
|
||||
const [disableAllDockerHosts, setDisableAllDockerHosts] = createSignal(false);
|
||||
const [disableAllDockerServices, setDisableAllDockerServices] = createSignal(false);
|
||||
const [disableAllDockerContainers, setDisableAllDockerContainers] = createSignal(false);
|
||||
|
||||
// Global disable offline alerts flags
|
||||
|
|
@ -1425,6 +1455,17 @@ const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
|
|||
? Math.max(normalizedBackupCritical, normalizedBackupWarning)
|
||||
: normalizedBackupWarning;
|
||||
|
||||
const dockerDefaultsValue = dockerDefaults();
|
||||
if (
|
||||
dockerDefaultsValue.serviceCriticalGapPercent > 0 &&
|
||||
dockerDefaultsValue.serviceWarnGapPercent > dockerDefaultsValue.serviceCriticalGapPercent
|
||||
) {
|
||||
showError(
|
||||
'Swarm service critical gap must be greater than or equal to the warning gap when enabled.',
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const alertConfig = {
|
||||
enabled: true,
|
||||
// Global disable flags per resource type
|
||||
|
|
@ -1436,6 +1477,7 @@ const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
|
|||
disableAllPMG: disableAllPMG(),
|
||||
disableAllDockerHosts: disableAllDockerHosts(),
|
||||
disableAllDockerContainers: disableAllDockerContainers(),
|
||||
disableAllDockerServices: disableAllDockerServices(),
|
||||
// Global disable offline alerts flags
|
||||
disableAllNodesOffline: disableAllNodesOffline(),
|
||||
disableAllGuestsOffline: disableAllGuestsOffline(),
|
||||
|
|
@ -1466,12 +1508,14 @@ const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
|
|||
disk: createHysteresisThreshold(hostDefaults().disk),
|
||||
},
|
||||
dockerDefaults: {
|
||||
cpu: createHysteresisThreshold(dockerDefaults().cpu),
|
||||
memory: createHysteresisThreshold(dockerDefaults().memory),
|
||||
restartCount: dockerDefaults().restartCount,
|
||||
restartWindow: dockerDefaults().restartWindow,
|
||||
memoryWarnPct: dockerDefaults().memoryWarnPct,
|
||||
memoryCriticalPct: dockerDefaults().memoryCriticalPct,
|
||||
cpu: createHysteresisThreshold(dockerDefaultsValue.cpu),
|
||||
memory: createHysteresisThreshold(dockerDefaultsValue.memory),
|
||||
restartCount: dockerDefaultsValue.restartCount,
|
||||
restartWindow: dockerDefaultsValue.restartWindow,
|
||||
memoryWarnPct: dockerDefaultsValue.memoryWarnPct,
|
||||
memoryCriticalPct: dockerDefaultsValue.memoryCriticalPct,
|
||||
serviceWarnGapPercent: dockerDefaultsValue.serviceWarnGapPercent,
|
||||
serviceCriticalGapPercent: dockerDefaultsValue.serviceCriticalGapPercent,
|
||||
},
|
||||
dockerIgnoredContainerPrefixes: dockerIgnoredPrefixes()
|
||||
.map((prefix) => prefix.trim())
|
||||
|
|
@ -1767,10 +1811,12 @@ const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
|
|||
setDisableAllPBS={setDisableAllPBS}
|
||||
disableAllPMG={disableAllPMG}
|
||||
setDisableAllPMG={setDisableAllPMG}
|
||||
disableAllDockerHosts={disableAllDockerHosts}
|
||||
setDisableAllDockerHosts={setDisableAllDockerHosts}
|
||||
disableAllDockerContainers={disableAllDockerContainers}
|
||||
setDisableAllDockerContainers={setDisableAllDockerContainers}
|
||||
disableAllDockerHosts={disableAllDockerHosts}
|
||||
setDisableAllDockerHosts={setDisableAllDockerHosts}
|
||||
disableAllDockerServices={disableAllDockerServices}
|
||||
setDisableAllDockerServices={setDisableAllDockerServices}
|
||||
disableAllDockerContainers={disableAllDockerContainers}
|
||||
setDisableAllDockerContainers={setDisableAllDockerContainers}
|
||||
disableAllNodesOffline={disableAllNodesOffline}
|
||||
setDisableAllNodesOffline={setDisableAllNodesOffline}
|
||||
disableAllGuestsOffline={disableAllGuestsOffline}
|
||||
|
|
@ -2238,7 +2284,16 @@ interface ThresholdsTabProps {
|
|||
guestDefaults: () => Record<string, number | undefined>;
|
||||
nodeDefaults: () => Record<string, number | undefined>;
|
||||
hostDefaults: () => Record<string, number | undefined>;
|
||||
dockerDefaults: () => { cpu: number; memory: number; restartCount: number; restartWindow: number; memoryWarnPct: number; memoryCriticalPct: number };
|
||||
dockerDefaults: () => {
|
||||
cpu: number;
|
||||
memory: number;
|
||||
restartCount: number;
|
||||
restartWindow: number;
|
||||
memoryWarnPct: number;
|
||||
memoryCriticalPct: number;
|
||||
serviceWarnGapPercent: number;
|
||||
serviceCriticalGapPercent: number;
|
||||
};
|
||||
dockerIgnoredPrefixes: () => string[];
|
||||
storageDefault: () => number;
|
||||
timeThresholds: () => { guest: number; node: number; storage: number; pbs: number };
|
||||
|
|
@ -2271,7 +2326,36 @@ interface ThresholdsTabProps {
|
|||
| ((prev: Record<string, number | undefined>) => Record<string, number | undefined>),
|
||||
) => void;
|
||||
setDockerDefaults: (
|
||||
value: { cpu: number; memory: number; restartCount: number; restartWindow: number; memoryWarnPct: number; memoryCriticalPct: number } | ((prev: { cpu: number; memory: number; restartCount: number; restartWindow: number; memoryWarnPct: number; memoryCriticalPct: number }) => { cpu: number; memory: number; restartCount: number; restartWindow: number; memoryWarnPct: number; memoryCriticalPct: number }),
|
||||
value:
|
||||
| {
|
||||
cpu: number;
|
||||
memory: number;
|
||||
restartCount: number;
|
||||
restartWindow: number;
|
||||
memoryWarnPct: number;
|
||||
memoryCriticalPct: number;
|
||||
serviceWarnGapPercent: number;
|
||||
serviceCriticalGapPercent: number;
|
||||
}
|
||||
| ((prev: {
|
||||
cpu: number;
|
||||
memory: number;
|
||||
restartCount: number;
|
||||
restartWindow: number;
|
||||
memoryWarnPct: number;
|
||||
memoryCriticalPct: number;
|
||||
serviceWarnGapPercent: number;
|
||||
serviceCriticalGapPercent: number;
|
||||
}) => {
|
||||
cpu: number;
|
||||
memory: number;
|
||||
restartCount: number;
|
||||
restartWindow: number;
|
||||
memoryWarnPct: number;
|
||||
memoryCriticalPct: number;
|
||||
serviceWarnGapPercent: number;
|
||||
serviceCriticalGapPercent: number;
|
||||
}),
|
||||
) => void;
|
||||
setDockerIgnoredPrefixes: (value: string[] | ((prev: string[]) => string[])) => void;
|
||||
setStorageDefault: (value: number) => void;
|
||||
|
|
@ -2317,6 +2401,8 @@ interface ThresholdsTabProps {
|
|||
setDisableAllPMG: (value: boolean) => void;
|
||||
disableAllDockerHosts: () => boolean;
|
||||
setDisableAllDockerHosts: (value: boolean) => void;
|
||||
disableAllDockerServices: () => boolean;
|
||||
setDisableAllDockerServices: (value: boolean) => void;
|
||||
disableAllDockerContainers: () => boolean;
|
||||
setDisableAllDockerContainers: (value: boolean) => void;
|
||||
// Global disable offline alerts flags
|
||||
|
|
@ -2410,6 +2496,8 @@ function ThresholdsTab(props: ThresholdsTabProps) {
|
|||
setDisableAllPMG={props.setDisableAllPMG}
|
||||
disableAllDockerHosts={props.disableAllDockerHosts}
|
||||
setDisableAllDockerHosts={props.setDisableAllDockerHosts}
|
||||
disableAllDockerServices={props.disableAllDockerServices}
|
||||
setDisableAllDockerServices={props.setDisableAllDockerServices}
|
||||
disableAllDockerContainers={props.disableAllDockerContainers}
|
||||
setDisableAllDockerContainers={props.setDisableAllDockerContainers}
|
||||
disableAllNodesOffline={props.disableAllNodesOffline}
|
||||
|
|
|
|||
|
|
@ -64,6 +64,8 @@ export interface DockerThresholdConfig {
|
|||
restartWindow?: number;
|
||||
memoryWarnPct?: number;
|
||||
memoryCriticalPct?: number;
|
||||
serviceWarnGapPercent?: number;
|
||||
serviceCriticalGapPercent?: number;
|
||||
}
|
||||
|
||||
export interface PMGThresholdDefaults {
|
||||
|
|
@ -198,6 +200,7 @@ export interface AlertConfig {
|
|||
disableAllHosts?: boolean;
|
||||
disableAllDockerHosts?: boolean;
|
||||
disableAllDockerContainers?: boolean;
|
||||
disableAllDockerServices?: boolean;
|
||||
disableAllNodesOffline?: boolean;
|
||||
disableAllGuestsOffline?: boolean;
|
||||
disableAllPBSOffline?: boolean;
|
||||
|
|
|
|||
|
|
@ -135,6 +135,9 @@ export interface DockerHost {
|
|||
intervalSeconds: number;
|
||||
agentVersion?: string;
|
||||
containers: DockerContainer[];
|
||||
services?: DockerService[];
|
||||
tasks?: DockerTask[];
|
||||
swarm?: DockerSwarmInfo;
|
||||
tokenId?: string;
|
||||
tokenName?: string;
|
||||
tokenHint?: string;
|
||||
|
|
@ -161,6 +164,66 @@ export interface DockerHostCommand {
|
|||
expiresAt?: number;
|
||||
}
|
||||
|
||||
export interface DockerService {
|
||||
id: string;
|
||||
name: string;
|
||||
stack?: string;
|
||||
image?: string;
|
||||
mode?: string;
|
||||
desiredTasks?: number;
|
||||
runningTasks?: number;
|
||||
completedTasks?: number;
|
||||
labels?: Record<string, string>;
|
||||
endpointPorts?: DockerServicePort[];
|
||||
updateStatus?: DockerServiceUpdate;
|
||||
createdAt?: number;
|
||||
updatedAt?: number;
|
||||
}
|
||||
|
||||
export interface DockerServicePort {
|
||||
name?: string;
|
||||
protocol?: string;
|
||||
targetPort?: number;
|
||||
publishedPort?: number;
|
||||
publishMode?: string;
|
||||
}
|
||||
|
||||
export interface DockerServiceUpdate {
|
||||
state?: string;
|
||||
message?: string;
|
||||
completedAt?: number;
|
||||
}
|
||||
|
||||
export interface DockerTask {
|
||||
id: string;
|
||||
serviceId?: string;
|
||||
serviceName?: string;
|
||||
slot?: number;
|
||||
nodeId?: string;
|
||||
nodeName?: string;
|
||||
desiredState?: string;
|
||||
currentState?: string;
|
||||
error?: string;
|
||||
message?: string;
|
||||
containerId?: string;
|
||||
containerName?: string;
|
||||
createdAt?: number;
|
||||
updatedAt?: number;
|
||||
startedAt?: number;
|
||||
completedAt?: number;
|
||||
}
|
||||
|
||||
export interface DockerSwarmInfo {
|
||||
nodeId?: string;
|
||||
nodeRole?: string;
|
||||
localState?: string;
|
||||
controlAvailable?: boolean;
|
||||
clusterId?: string;
|
||||
clusterName?: string;
|
||||
scope?: string;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface DockerContainer {
|
||||
id: string;
|
||||
name: string;
|
||||
|
|
|
|||
|
|
@ -264,12 +264,14 @@ type CustomAlertRule struct {
|
|||
|
||||
// DockerThresholdConfig represents Docker-specific alert thresholds
|
||||
type DockerThresholdConfig struct {
|
||||
CPU HysteresisThreshold `json:"cpu"` // CPU usage % threshold (default: 80%)
|
||||
Memory HysteresisThreshold `json:"memory"` // Memory usage % threshold (default: 85%)
|
||||
RestartCount int `json:"restartCount"` // Number of restarts to trigger alert (default: 3)
|
||||
RestartWindow int `json:"restartWindow"` // Time window in seconds for restart loop detection (default: 300 = 5min)
|
||||
MemoryWarnPct int `json:"memoryWarnPct"` // Memory limit % to trigger warning (default: 90)
|
||||
MemoryCriticalPct int `json:"memoryCriticalPct"` // Memory limit % to trigger critical (default: 95)
|
||||
CPU HysteresisThreshold `json:"cpu"` // CPU usage % threshold (default: 80%)
|
||||
Memory HysteresisThreshold `json:"memory"` // Memory usage % threshold (default: 85%)
|
||||
RestartCount int `json:"restartCount"` // Number of restarts to trigger alert (default: 3)
|
||||
RestartWindow int `json:"restartWindow"` // Time window in seconds for restart loop detection (default: 300 = 5min)
|
||||
MemoryWarnPct int `json:"memoryWarnPct"` // Memory limit % to trigger warning (default: 90)
|
||||
MemoryCriticalPct int `json:"memoryCriticalPct"` // Memory limit % to trigger critical (default: 95)
|
||||
ServiceWarnGapPct int `json:"serviceWarnGapPercent"` // % of desired tasks missing to trigger warning (default: 10)
|
||||
ServiceCritGapPct int `json:"serviceCriticalGapPercent"` // % of desired tasks missing to trigger critical (default: 50)
|
||||
}
|
||||
|
||||
// PMGThresholdConfig represents Proxmox Mail Gateway-specific alert thresholds
|
||||
|
|
@ -344,6 +346,7 @@ type AlertConfig struct {
|
|||
DisableAllPMG bool `json:"disableAllPMG"` // Disable all alerts for PMG instances
|
||||
DisableAllDockerHosts bool `json:"disableAllDockerHosts"` // Disable all alerts for Docker hosts
|
||||
DisableAllDockerContainers bool `json:"disableAllDockerContainers"` // Disable all alerts for Docker containers
|
||||
DisableAllDockerServices bool `json:"disableAllDockerServices"` // Disable all alerts for Docker services
|
||||
DisableAllNodesOffline bool `json:"disableAllNodesOffline"` // Disable node offline/connectivity alerts globally
|
||||
DisableAllGuestsOffline bool `json:"disableAllGuestsOffline"` // Disable guest powered-off alerts globally
|
||||
DisableAllHostsOffline bool `json:"disableAllHostsOffline"` // Disable host agent offline alerts globally
|
||||
|
|
@ -813,6 +816,20 @@ func (m *Manager) UpdateConfig(config AlertConfig) {
|
|||
if config.DockerDefaults.MemoryCriticalPct <= 0 {
|
||||
config.DockerDefaults.MemoryCriticalPct = 95
|
||||
}
|
||||
if config.DockerDefaults.ServiceWarnGapPct <= 0 {
|
||||
config.DockerDefaults.ServiceWarnGapPct = 10
|
||||
}
|
||||
if config.DockerDefaults.ServiceCritGapPct <= 0 {
|
||||
config.DockerDefaults.ServiceCritGapPct = 50
|
||||
}
|
||||
if config.DockerDefaults.ServiceCritGapPct > 0 &&
|
||||
config.DockerDefaults.ServiceCritGapPct < config.DockerDefaults.ServiceWarnGapPct {
|
||||
log.Warn().
|
||||
Int("warnGapPercent", config.DockerDefaults.ServiceWarnGapPct).
|
||||
Int("criticalGapPercent", config.DockerDefaults.ServiceCritGapPct).
|
||||
Msg("Adjusting Docker service critical gap to match warning gap")
|
||||
config.DockerDefaults.ServiceCritGapPct = config.DockerDefaults.ServiceWarnGapPct
|
||||
}
|
||||
|
||||
// Initialize PMG defaults if missing/zero
|
||||
if config.PMGDefaults.QueueTotalWarning <= 0 {
|
||||
|
|
@ -1182,6 +1199,17 @@ func (m *Manager) applyGlobalOfflineSettingsLocked() {
|
|||
m.dockerRestartTracking = make(map[string]*dockerRestartRecord)
|
||||
m.dockerLastExitCode = make(map[string]int)
|
||||
}
|
||||
if m.config.DisableAllDockerServices {
|
||||
var serviceAlerts []string
|
||||
for alertID := range m.activeAlerts {
|
||||
if strings.HasPrefix(alertID, "docker-service-") {
|
||||
serviceAlerts = append(serviceAlerts, alertID)
|
||||
}
|
||||
}
|
||||
for _, alertID := range serviceAlerts {
|
||||
m.clearAlertNoLock(alertID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// reevaluateActiveAlertsLocked re-evaluates all active alerts against the current configuration
|
||||
|
|
@ -2582,6 +2610,57 @@ func dockerResourceID(hostID, containerID string) string {
|
|||
return fmt.Sprintf("docker:%s/%s", hostID, containerID)
|
||||
}
|
||||
|
||||
// dockerServiceDisplayName normalizes the service name for alert readability.
|
||||
func dockerServiceDisplayName(service models.DockerService) string {
|
||||
name := strings.TrimSpace(service.Name)
|
||||
if name != "" {
|
||||
return name
|
||||
}
|
||||
id := strings.TrimSpace(service.ID)
|
||||
if len(id) > 12 {
|
||||
id = id[:12]
|
||||
}
|
||||
if id == "" {
|
||||
return "service"
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
func dockerServiceResourceID(hostID, serviceID, serviceName string) string {
|
||||
hostID = strings.TrimSpace(hostID)
|
||||
id := strings.TrimSpace(serviceID)
|
||||
if id == "" {
|
||||
name := strings.TrimSpace(serviceName)
|
||||
if name == "" {
|
||||
name = "service"
|
||||
}
|
||||
builder := strings.Builder{}
|
||||
for _, r := range strings.ToLower(name) {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z':
|
||||
builder.WriteRune(r)
|
||||
case r >= '0' && r <= '9':
|
||||
builder.WriteRune(r)
|
||||
case r == '-', r == '_':
|
||||
builder.WriteRune(r)
|
||||
case r == ' ' || r == '/' || r == '\\' || r == ':' || r == '.':
|
||||
builder.WriteRune('-')
|
||||
}
|
||||
}
|
||||
id = strings.Trim(builder.String(), "-_")
|
||||
if id == "" {
|
||||
id = "service"
|
||||
}
|
||||
if len(id) > 32 {
|
||||
id = id[:32]
|
||||
}
|
||||
}
|
||||
if hostID == "" {
|
||||
return fmt.Sprintf("docker-service:%s", id)
|
||||
}
|
||||
return fmt.Sprintf("docker:%s/service/%s", hostID, id)
|
||||
}
|
||||
|
||||
func matchesDockerIgnoredPrefix(name, id string, prefixes []string) bool {
|
||||
if len(prefixes) == 0 {
|
||||
return false
|
||||
|
|
@ -2627,7 +2706,7 @@ func (m *Manager) CheckDockerHost(host models.DockerHost) {
|
|||
return
|
||||
}
|
||||
|
||||
seen := make(map[string]struct{}, len(host.Containers))
|
||||
seen := make(map[string]struct{}, len(host.Containers)+len(host.Services))
|
||||
for _, container := range host.Containers {
|
||||
containerName := dockerContainerDisplayName(container)
|
||||
resourceID := dockerResourceID(host.ID, container.ID)
|
||||
|
|
@ -2654,6 +2733,12 @@ func (m *Manager) CheckDockerHost(host models.DockerHost) {
|
|||
m.evaluateDockerContainer(host, container, resourceID)
|
||||
}
|
||||
|
||||
for _, service := range host.Services {
|
||||
resourceID := dockerServiceResourceID(host.ID, service.ID, service.Name)
|
||||
seen[resourceID] = struct{}{}
|
||||
m.evaluateDockerService(host, service, resourceID)
|
||||
}
|
||||
|
||||
m.cleanupDockerContainerAlerts(host, seen)
|
||||
}
|
||||
|
||||
|
|
@ -2750,6 +2835,151 @@ func (m *Manager) evaluateDockerContainer(host models.DockerHost, container mode
|
|||
m.checkDockerContainerMemoryLimit(host, container, resourceID, containerName, instanceName, nodeName)
|
||||
}
|
||||
|
||||
func (m *Manager) evaluateDockerService(host models.DockerHost, service models.DockerService, resourceID string) {
|
||||
m.mu.RLock()
|
||||
disableAllServices := m.config.DisableAllDockerServices
|
||||
warnPct := m.config.DockerDefaults.ServiceWarnGapPct
|
||||
critPct := m.config.DockerDefaults.ServiceCritGapPct
|
||||
overrideConfig, hasOverride := m.config.Overrides[resourceID]
|
||||
m.mu.RUnlock()
|
||||
|
||||
if disableAllServices {
|
||||
m.clearDockerServiceAlert(resourceID)
|
||||
return
|
||||
}
|
||||
if hasOverride && overrideConfig.Disabled {
|
||||
m.clearDockerServiceAlert(resourceID)
|
||||
return
|
||||
}
|
||||
|
||||
desired := service.DesiredTasks
|
||||
running := service.RunningTasks
|
||||
if desired <= 0 {
|
||||
m.clearDockerServiceAlert(resourceID)
|
||||
return
|
||||
}
|
||||
|
||||
missing := desired - running
|
||||
if missing < 0 {
|
||||
missing = 0
|
||||
}
|
||||
|
||||
percentMissing := 0.0
|
||||
if desired > 0 {
|
||||
percentMissing = (float64(missing) / float64(desired)) * 100.0
|
||||
}
|
||||
|
||||
severity := AlertLevel("")
|
||||
thresholdValue := 0.0
|
||||
if critPct > 0 && percentMissing >= float64(critPct) {
|
||||
severity = AlertLevelCritical
|
||||
thresholdValue = float64(critPct)
|
||||
} else if warnPct > 0 && percentMissing >= float64(warnPct) {
|
||||
severity = AlertLevelWarning
|
||||
thresholdValue = float64(warnPct)
|
||||
}
|
||||
|
||||
updateState := ""
|
||||
updateMessage := ""
|
||||
if service.UpdateStatus != nil {
|
||||
updateState = strings.ToLower(strings.TrimSpace(service.UpdateStatus.State))
|
||||
updateMessage = strings.TrimSpace(service.UpdateStatus.Message)
|
||||
if severity == "" {
|
||||
switch updateState {
|
||||
case "paused", "rollback_started", "rollback_paused":
|
||||
severity = AlertLevelWarning
|
||||
case "rollback_failed":
|
||||
severity = AlertLevelCritical
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if severity == "" {
|
||||
m.clearDockerServiceAlert(resourceID)
|
||||
return
|
||||
}
|
||||
|
||||
serviceName := dockerServiceDisplayName(service)
|
||||
instanceName := dockerInstanceName(host)
|
||||
nodeName := strings.TrimSpace(host.Hostname)
|
||||
|
||||
message := ""
|
||||
if missing > 0 {
|
||||
message = fmt.Sprintf("Docker service '%s' is running %d of %d desired tasks", serviceName, service.RunningTasks, service.DesiredTasks)
|
||||
} else if updateState != "" {
|
||||
message = fmt.Sprintf("Docker service '%s' update state: %s", serviceName, service.UpdateStatus.State)
|
||||
} else {
|
||||
message = fmt.Sprintf("Docker service '%s' triggered a Swarm alert", serviceName)
|
||||
}
|
||||
if updateMessage != "" {
|
||||
message = fmt.Sprintf("%s (%s)", message, updateMessage)
|
||||
}
|
||||
|
||||
metadata := map[string]interface{}{
|
||||
"resourceType": "Docker Service",
|
||||
"hostId": host.ID,
|
||||
"hostName": host.DisplayName,
|
||||
"hostHostname": host.Hostname,
|
||||
"serviceId": service.ID,
|
||||
"serviceName": service.Name,
|
||||
"stack": service.Stack,
|
||||
"mode": service.Mode,
|
||||
"desiredTasks": service.DesiredTasks,
|
||||
"runningTasks": service.RunningTasks,
|
||||
"completedTasks": service.CompletedTasks,
|
||||
"missingTasks": missing,
|
||||
"percentMissing": percentMissing,
|
||||
}
|
||||
if updateState != "" {
|
||||
metadata["updateState"] = service.UpdateStatus.State
|
||||
}
|
||||
if updateMessage != "" {
|
||||
metadata["updateMessage"] = updateMessage
|
||||
}
|
||||
if service.UpdateStatus != nil && service.UpdateStatus.CompletedAt != nil && !service.UpdateStatus.CompletedAt.IsZero() {
|
||||
metadata["updateCompletedAt"] = service.UpdateStatus.CompletedAt.UTC()
|
||||
}
|
||||
|
||||
alertID := fmt.Sprintf("docker-service-health-%s", resourceID)
|
||||
alert := &Alert{
|
||||
ID: alertID,
|
||||
Type: "docker-service-health",
|
||||
Level: severity,
|
||||
ResourceID: resourceID,
|
||||
ResourceName: serviceName,
|
||||
Node: nodeName,
|
||||
Instance: instanceName,
|
||||
Message: message,
|
||||
Value: percentMissing,
|
||||
Threshold: thresholdValue,
|
||||
StartTime: time.Now(),
|
||||
LastSeen: time.Now(),
|
||||
Metadata: metadata,
|
||||
}
|
||||
|
||||
m.mu.Lock()
|
||||
if existing, exists := m.activeAlerts[alertID]; exists && existing != nil {
|
||||
alert.StartTime = existing.StartTime
|
||||
}
|
||||
m.preserveAlertState(alertID, alert)
|
||||
m.activeAlerts[alertID] = alert
|
||||
m.recentAlerts[alertID] = alert
|
||||
m.historyManager.AddAlert(*alert)
|
||||
m.dispatchAlert(alert, severity == AlertLevelCritical)
|
||||
m.mu.Unlock()
|
||||
|
||||
log.Warn().
|
||||
Str("service", serviceName).
|
||||
Str("host", host.DisplayName).
|
||||
Float64("percentMissing", percentMissing).
|
||||
Msg("Docker service alert raised")
|
||||
}
|
||||
|
||||
func (m *Manager) clearDockerServiceAlert(resourceID string) {
|
||||
alertID := fmt.Sprintf("docker-service-health-%s", resourceID)
|
||||
m.clearAlert(alertID)
|
||||
}
|
||||
|
||||
// HandleDockerHostOnline clears offline tracking and alerts for a Docker host.
|
||||
func (m *Manager) HandleDockerHostOnline(host models.DockerHost) {
|
||||
if host.ID == "" {
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ package alerts
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
|
@ -940,6 +941,144 @@ func TestCheckDockerHostIgnoresContainersByPrefix(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestDockerServiceReplicaAlerts(t *testing.T) {
|
||||
m := NewManager()
|
||||
m.ClearActiveAlerts()
|
||||
|
||||
m.mu.RLock()
|
||||
cfg := m.config
|
||||
m.mu.RUnlock()
|
||||
cfg.Enabled = true
|
||||
m.UpdateConfig(cfg)
|
||||
|
||||
host := models.DockerHost{
|
||||
ID: "host-1",
|
||||
DisplayName: "Prod Swarm",
|
||||
Hostname: "swarm-prod",
|
||||
Services: []models.DockerService{
|
||||
{
|
||||
ID: "svc-1",
|
||||
Name: "web",
|
||||
DesiredTasks: 4,
|
||||
RunningTasks: 2,
|
||||
Mode: "replicated",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
m.CheckDockerHost(host)
|
||||
|
||||
resourceID := dockerServiceResourceID(host.ID, "svc-1", "web")
|
||||
alertID := fmt.Sprintf("docker-service-health-%s", resourceID)
|
||||
alert, exists := m.activeAlerts[alertID]
|
||||
if !exists {
|
||||
t.Fatalf("expected service alert %s to be raised", alertID)
|
||||
}
|
||||
if alert.Level != AlertLevelCritical {
|
||||
t.Fatalf("expected critical severity, got %s", alert.Level)
|
||||
}
|
||||
if missing, ok := alert.Metadata["missingTasks"].(int); !ok || missing != 2 {
|
||||
t.Fatalf("expected missingTasks metadata to be 2, got %v", alert.Metadata["missingTasks"])
|
||||
}
|
||||
|
||||
// Resolve by restoring replicas
|
||||
host.Services[0].RunningTasks = 4
|
||||
m.CheckDockerHost(host)
|
||||
|
||||
if _, exists := m.activeAlerts[alertID]; exists {
|
||||
t.Fatalf("expected service alert %s to be cleared when replicas restored", alertID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateConfigClampsDockerServiceCriticalGap(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
m := NewManager()
|
||||
|
||||
cfg := AlertConfig{
|
||||
Enabled: true,
|
||||
GuestDefaults: ThresholdConfig{},
|
||||
NodeDefaults: ThresholdConfig{},
|
||||
HostDefaults: ThresholdConfig{},
|
||||
StorageDefault: HysteresisThreshold{},
|
||||
DockerDefaults: DockerThresholdConfig{
|
||||
ServiceWarnGapPct: 35,
|
||||
ServiceCritGapPct: 20,
|
||||
},
|
||||
PMGDefaults: PMGThresholdConfig{},
|
||||
SnapshotDefaults: SnapshotAlertConfig{},
|
||||
BackupDefaults: BackupAlertConfig{},
|
||||
Overrides: make(map[string]ThresholdConfig),
|
||||
Schedule: ScheduleConfig{},
|
||||
}
|
||||
|
||||
m.UpdateConfig(cfg)
|
||||
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
|
||||
if m.config.DockerDefaults.ServiceWarnGapPct != 35 {
|
||||
t.Fatalf("expected warning gap to remain 35, got %d", m.config.DockerDefaults.ServiceWarnGapPct)
|
||||
}
|
||||
if m.config.DockerDefaults.ServiceCritGapPct != 35 {
|
||||
t.Fatalf("expected critical gap to be clamped to 35, got %d", m.config.DockerDefaults.ServiceCritGapPct)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDockerServiceAlertUsesClampedCriticalGap(t *testing.T) {
|
||||
m := NewManager()
|
||||
m.ClearActiveAlerts()
|
||||
|
||||
cfg := AlertConfig{
|
||||
Enabled: true,
|
||||
GuestDefaults: ThresholdConfig{},
|
||||
NodeDefaults: ThresholdConfig{},
|
||||
HostDefaults: ThresholdConfig{},
|
||||
StorageDefault: HysteresisThreshold{},
|
||||
DockerDefaults: DockerThresholdConfig{
|
||||
ServiceWarnGapPct: 20,
|
||||
ServiceCritGapPct: 5,
|
||||
},
|
||||
PMGDefaults: PMGThresholdConfig{},
|
||||
SnapshotDefaults: SnapshotAlertConfig{},
|
||||
BackupDefaults: BackupAlertConfig{},
|
||||
Overrides: make(map[string]ThresholdConfig),
|
||||
Schedule: ScheduleConfig{},
|
||||
}
|
||||
|
||||
m.UpdateConfig(cfg)
|
||||
|
||||
host := models.DockerHost{
|
||||
ID: "docker-host-1",
|
||||
DisplayName: "Docker Host",
|
||||
Hostname: "docker-host.local",
|
||||
Services: []models.DockerService{
|
||||
{
|
||||
ID: "svc-123",
|
||||
Name: "api",
|
||||
DesiredTasks: 10,
|
||||
RunningTasks: 7,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
m.CheckDockerHost(host)
|
||||
|
||||
resourceID := dockerServiceResourceID(host.ID, "svc-123", "api")
|
||||
alertID := fmt.Sprintf("docker-service-health-%s", resourceID)
|
||||
|
||||
alert, exists := m.activeAlerts[alertID]
|
||||
if !exists {
|
||||
t.Fatalf("expected docker service alert %s to be raised", alertID)
|
||||
}
|
||||
if alert.Level != AlertLevelCritical {
|
||||
t.Fatalf("expected critical severity when replicas 7/10, got %s", alert.Level)
|
||||
}
|
||||
if pct, ok := alert.Metadata["percentMissing"].(float64); !ok || math.Abs(pct-30.0) > 0.01 {
|
||||
t.Fatalf("expected percentMissing metadata ~30, got %v", alert.Metadata["percentMissing"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeDockerIgnoredPrefixes(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
|
|
|||
|
|
@ -124,3 +124,26 @@ func TestUpdateConfigClearsDockerContainerAlertsWhenDisabled(t *testing.T) {
|
|||
t.Fatalf("expected dockerLastExitCode map to be cleared when DisableAllDockerContainers is enabled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateConfigClearsDockerServiceAlertsWhenDisabled(t *testing.T) {
|
||||
manager := NewManager()
|
||||
|
||||
serviceResourceID := "docker:host-2/service/frontend"
|
||||
serviceAlertID := "docker-service-health-" + serviceResourceID
|
||||
|
||||
manager.mu.Lock()
|
||||
manager.activeAlerts[serviceAlertID] = &Alert{ID: serviceAlertID, ResourceID: serviceResourceID}
|
||||
manager.mu.Unlock()
|
||||
|
||||
config := manager.GetConfig()
|
||||
config.DisableAllDockerServices = true
|
||||
manager.UpdateConfig(config)
|
||||
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
|
||||
manager.mu.RLock()
|
||||
defer manager.mu.RUnlock()
|
||||
if _, exists := manager.activeAlerts[serviceAlertID]; exists {
|
||||
t.Fatalf("expected docker service alert to be cleared when DisableAllDockerServices is enabled")
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -43,6 +43,10 @@ type Config struct {
|
|||
DisableAutoUpdate bool
|
||||
Targets []TargetConfig
|
||||
ContainerStates []string
|
||||
SwarmScope string
|
||||
IncludeServices bool
|
||||
IncludeTasks bool
|
||||
IncludeContainers bool
|
||||
Logger *zerolog.Logger
|
||||
}
|
||||
|
||||
|
|
@ -110,6 +114,18 @@ func New(cfg Config) (*Agent, error) {
|
|||
}
|
||||
cfg.ContainerStates = stateFilters
|
||||
|
||||
scope, err := normalizeSwarmScope(cfg.SwarmScope)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cfg.SwarmScope = scope
|
||||
|
||||
if !cfg.IncludeContainers && !cfg.IncludeServices && !cfg.IncludeTasks {
|
||||
cfg.IncludeContainers = true
|
||||
cfg.IncludeServices = true
|
||||
cfg.IncludeTasks = true
|
||||
}
|
||||
|
||||
dockerClient, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create docker client: %w", err)
|
||||
|
|
@ -316,11 +332,22 @@ func (a *Agent) buildReport(ctx context.Context) (agentsdocker.Report, error) {
|
|||
|
||||
uptime := readSystemUptime()
|
||||
|
||||
containers, err := a.collectContainers(ctx)
|
||||
if err != nil {
|
||||
return agentsdocker.Report{}, err
|
||||
collectContainers := a.cfg.IncludeContainers
|
||||
if !collectContainers && (a.cfg.IncludeServices || a.cfg.IncludeTasks) && !info.Swarm.ControlAvailable {
|
||||
collectContainers = true
|
||||
}
|
||||
|
||||
var containers []agentsdocker.Container
|
||||
if collectContainers {
|
||||
var err error
|
||||
containers, err = a.collectContainers(ctx)
|
||||
if err != nil {
|
||||
return agentsdocker.Report{}, err
|
||||
}
|
||||
}
|
||||
|
||||
services, tasks, swarmInfo := a.collectSwarmData(ctx, info, containers)
|
||||
|
||||
report := agentsdocker.Report{
|
||||
Agent: agentsdocker.AgentInfo{
|
||||
ID: agentID,
|
||||
|
|
@ -339,8 +366,21 @@ func (a *Agent) buildReport(ctx context.Context) (agentsdocker.Report, error) {
|
|||
TotalMemoryBytes: info.MemTotal,
|
||||
UptimeSeconds: uptime,
|
||||
},
|
||||
Containers: containers,
|
||||
Timestamp: time.Now().UTC(),
|
||||
Timestamp: time.Now().UTC(),
|
||||
}
|
||||
|
||||
if swarmInfo != nil {
|
||||
report.Host.Swarm = swarmInfo
|
||||
}
|
||||
|
||||
if a.cfg.IncludeContainers {
|
||||
report.Containers = containers
|
||||
}
|
||||
if a.cfg.IncludeServices && len(services) > 0 {
|
||||
report.Services = services
|
||||
}
|
||||
if a.cfg.IncludeTasks && len(tasks) > 0 {
|
||||
report.Tasks = tasks
|
||||
}
|
||||
|
||||
if report.Agent.IntervalSeconds <= 0 {
|
||||
|
|
|
|||
|
|
@ -54,3 +54,29 @@ func TestNormalizeContainerStatesInvalid(t *testing.T) {
|
|||
t.Fatalf("expected error for invalid container state")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeSwarmScope(t *testing.T) {
|
||||
tests := map[string]string{
|
||||
"": "node",
|
||||
"node": "node",
|
||||
"NODE": "node",
|
||||
"cluster": "cluster",
|
||||
"AUTO": "auto",
|
||||
}
|
||||
|
||||
for input, expected := range tests {
|
||||
scope, err := normalizeSwarmScope(input)
|
||||
if err != nil {
|
||||
t.Fatalf("normalizeSwarmScope(%q) returned error: %v", input, err)
|
||||
}
|
||||
if scope != expected {
|
||||
t.Fatalf("normalizeSwarmScope(%q)=%q, expected %q", input, scope, expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeSwarmScopeInvalid(t *testing.T) {
|
||||
if _, err := normalizeSwarmScope("invalid"); err == nil {
|
||||
t.Fatalf("expected error for invalid swarm scope")
|
||||
}
|
||||
}
|
||||
|
|
|
|||
563
internal/dockeragent/swarm.go
Normal file
563
internal/dockeragent/swarm.go
Normal file
|
|
@ -0,0 +1,563 @@
|
|||
package dockeragent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/docker/docker/api/types"
|
||||
"github.com/docker/docker/api/types/filters"
|
||||
swarmtypes "github.com/docker/docker/api/types/swarm"
|
||||
systemtypes "github.com/docker/docker/api/types/system"
|
||||
agentsdocker "github.com/rcourtman/pulse-go-rewrite/pkg/agents/docker"
|
||||
)
|
||||
|
||||
const (
|
||||
swarmScopeAuto = "auto"
|
||||
swarmScopeNode = "node"
|
||||
swarmScopeCluster = "cluster"
|
||||
)
|
||||
|
||||
func normalizeSwarmScope(value string) (string, error) {
|
||||
scope := strings.ToLower(strings.TrimSpace(value))
|
||||
if scope == "" {
|
||||
return swarmScopeNode, nil
|
||||
}
|
||||
|
||||
switch scope {
|
||||
case swarmScopeNode, swarmScopeCluster, swarmScopeAuto:
|
||||
return scope, nil
|
||||
default:
|
||||
return "", fmt.Errorf("invalid swarm scope %q: must be node, cluster, or auto", value)
|
||||
}
|
||||
}
|
||||
|
||||
func (a *Agent) resolvedSwarmScope(info systemtypes.Info) string {
|
||||
switch a.cfg.SwarmScope {
|
||||
case swarmScopeAuto:
|
||||
if info.Swarm.ControlAvailable {
|
||||
return swarmScopeCluster
|
||||
}
|
||||
return swarmScopeNode
|
||||
case swarmScopeCluster, swarmScopeNode:
|
||||
return a.cfg.SwarmScope
|
||||
default:
|
||||
return swarmScopeNode
|
||||
}
|
||||
}
|
||||
|
||||
func (a *Agent) collectSwarmData(ctx context.Context, info systemtypes.Info, containers []agentsdocker.Container) ([]agentsdocker.Service, []agentsdocker.Task, *agentsdocker.SwarmInfo) {
|
||||
if info.Swarm.NodeID == "" && string(info.Swarm.LocalNodeState) == "" && strings.TrimSpace(info.Swarm.Error) == "" {
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
scope := a.resolvedSwarmScope(info)
|
||||
effectiveScope := scope
|
||||
|
||||
nodeRole := "worker"
|
||||
if info.Swarm.ControlAvailable {
|
||||
nodeRole = "manager"
|
||||
}
|
||||
|
||||
swarmInfo := &agentsdocker.SwarmInfo{
|
||||
NodeID: info.Swarm.NodeID,
|
||||
NodeRole: nodeRole,
|
||||
LocalState: string(info.Swarm.LocalNodeState),
|
||||
ControlAvailable: info.Swarm.ControlAvailable,
|
||||
Error: strings.TrimSpace(info.Swarm.Error),
|
||||
Scope: scope,
|
||||
}
|
||||
|
||||
if info.Swarm.Cluster != nil {
|
||||
swarmInfo.ClusterID = info.Swarm.Cluster.ID
|
||||
swarmInfo.ClusterName = info.Swarm.Cluster.Spec.Annotations.Name
|
||||
}
|
||||
|
||||
includeServices := a.cfg.IncludeServices
|
||||
includeTasks := a.cfg.IncludeTasks
|
||||
|
||||
if info.Swarm.LocalNodeState != swarmtypes.LocalNodeStateActive {
|
||||
if !includeServices {
|
||||
includeServices = false
|
||||
}
|
||||
if !includeTasks {
|
||||
includeTasks = false
|
||||
}
|
||||
return nil, nil, swarmInfo
|
||||
}
|
||||
|
||||
var services []agentsdocker.Service
|
||||
var tasks []agentsdocker.Task
|
||||
|
||||
containerIndex := buildContainerIndex(containers)
|
||||
|
||||
if info.Swarm.ControlAvailable && (includeServices || includeTasks) {
|
||||
managerServices, managerTasks, err := a.collectSwarmDataFromManager(ctx, info, scope, containerIndex, includeServices, includeTasks)
|
||||
if err != nil {
|
||||
a.logger.Warn().Err(err).Msg("Failed to collect swarm data from manager; falling back to local inference")
|
||||
} else {
|
||||
if includeServices {
|
||||
services = managerServices
|
||||
}
|
||||
if includeTasks {
|
||||
tasks = managerTasks
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if includeTasks && len(tasks) == 0 {
|
||||
tasks = deriveSwarmTasksFromContainers(containers, info)
|
||||
if len(tasks) > 0 {
|
||||
effectiveScope = swarmScopeNode
|
||||
}
|
||||
}
|
||||
|
||||
if includeServices && len(services) == 0 {
|
||||
services = deriveSwarmServicesFromData(tasks, containers)
|
||||
if len(services) > 0 {
|
||||
effectiveScope = swarmScopeNode
|
||||
}
|
||||
}
|
||||
|
||||
if includeTasks && len(tasks) > 0 {
|
||||
sort.Slice(tasks, func(i, j int) bool {
|
||||
if tasks[i].ServiceName == tasks[j].ServiceName {
|
||||
if tasks[i].Slot == tasks[j].Slot {
|
||||
return tasks[i].ID < tasks[j].ID
|
||||
}
|
||||
return tasks[i].Slot < tasks[j].Slot
|
||||
}
|
||||
return tasks[i].ServiceName < tasks[j].ServiceName
|
||||
})
|
||||
}
|
||||
|
||||
if includeServices && len(services) > 0 {
|
||||
sort.Slice(services, func(i, j int) bool {
|
||||
if services[i].Name == services[j].Name {
|
||||
return services[i].ID < services[j].ID
|
||||
}
|
||||
return services[i].Name < services[j].Name
|
||||
})
|
||||
}
|
||||
|
||||
swarmInfo.Scope = effectiveScope
|
||||
|
||||
if !includeServices {
|
||||
services = nil
|
||||
}
|
||||
if !includeTasks {
|
||||
tasks = nil
|
||||
}
|
||||
|
||||
return services, tasks, swarmInfo
|
||||
}
|
||||
|
||||
func (a *Agent) collectSwarmDataFromManager(ctx context.Context, info systemtypes.Info, scope string, containers map[string]agentsdocker.Container, includeServices, includeTasks bool) ([]agentsdocker.Service, []agentsdocker.Task, error) {
|
||||
serviceList, err := a.docker.ServiceList(ctx, types.ServiceListOptions{Status: true})
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
servicePointers := make(map[string]*swarmtypes.Service, len(serviceList))
|
||||
for i := range serviceList {
|
||||
servicePointers[serviceList[i].ID] = &serviceList[i]
|
||||
}
|
||||
|
||||
var services []agentsdocker.Service
|
||||
if includeServices {
|
||||
services = make([]agentsdocker.Service, 0, len(serviceList))
|
||||
for i := range serviceList {
|
||||
services = append(services, mapSwarmService(&serviceList[i]))
|
||||
}
|
||||
}
|
||||
|
||||
var tasks []agentsdocker.Task
|
||||
if includeTasks {
|
||||
taskFilters := filters.NewArgs()
|
||||
if scope != swarmScopeCluster && info.Swarm.NodeID != "" {
|
||||
taskFilters.Add("node", info.Swarm.NodeID)
|
||||
}
|
||||
|
||||
taskList, err := a.docker.TaskList(ctx, types.TaskListOptions{Filters: taskFilters})
|
||||
if err != nil {
|
||||
return services, nil, err
|
||||
}
|
||||
|
||||
tasks = make([]agentsdocker.Task, 0, len(taskList))
|
||||
for i := range taskList {
|
||||
var svc *swarmtypes.Service
|
||||
if ptr, ok := servicePointers[taskList[i].ServiceID]; ok {
|
||||
svc = ptr
|
||||
}
|
||||
tasks = append(tasks, mapSwarmTask(&taskList[i], svc, containers))
|
||||
}
|
||||
|
||||
if scope == swarmScopeNode && includeServices && len(services) > 0 {
|
||||
used := make(map[string]struct{}, len(tasks))
|
||||
for _, task := range tasks {
|
||||
if task.ServiceID != "" {
|
||||
used[task.ServiceID] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
filtered := services[:0]
|
||||
for _, svc := range services {
|
||||
if _, ok := used[svc.ID]; ok || len(used) == 0 {
|
||||
filtered = append(filtered, svc)
|
||||
}
|
||||
}
|
||||
services = filtered
|
||||
}
|
||||
}
|
||||
|
||||
return services, tasks, nil
|
||||
}
|
||||
|
||||
func mapSwarmService(svc *swarmtypes.Service) agentsdocker.Service {
|
||||
service := agentsdocker.Service{
|
||||
ID: svc.ID,
|
||||
Name: svc.Spec.Annotations.Name,
|
||||
Mode: serviceMode(svc.Spec.Mode),
|
||||
}
|
||||
|
||||
if svc.Spec.TaskTemplate.ContainerSpec != nil {
|
||||
service.Image = svc.Spec.TaskTemplate.ContainerSpec.Image
|
||||
}
|
||||
|
||||
if svc.Spec.Annotations.Labels != nil {
|
||||
service.Labels = copyStringMap(svc.Spec.Annotations.Labels)
|
||||
if stack, ok := svc.Spec.Annotations.Labels["com.docker.stack.namespace"]; ok {
|
||||
service.Stack = stack
|
||||
}
|
||||
}
|
||||
|
||||
if svc.ServiceStatus != nil {
|
||||
service.DesiredTasks = int(svc.ServiceStatus.DesiredTasks)
|
||||
service.RunningTasks = int(svc.ServiceStatus.RunningTasks)
|
||||
service.CompletedTasks = int(svc.ServiceStatus.CompletedTasks)
|
||||
}
|
||||
|
||||
if svc.UpdateStatus != nil {
|
||||
update := &agentsdocker.ServiceUpdate{
|
||||
State: string(svc.UpdateStatus.State),
|
||||
Message: svc.UpdateStatus.Message,
|
||||
}
|
||||
if svc.UpdateStatus.CompletedAt != nil {
|
||||
completed := *svc.UpdateStatus.CompletedAt
|
||||
if !completed.IsZero() {
|
||||
update.CompletedAt = &completed
|
||||
}
|
||||
}
|
||||
service.UpdateStatus = update
|
||||
}
|
||||
|
||||
if len(svc.Endpoint.Ports) > 0 {
|
||||
service.EndpointPorts = make([]agentsdocker.ServicePort, len(svc.Endpoint.Ports))
|
||||
for i, port := range svc.Endpoint.Ports {
|
||||
service.EndpointPorts[i] = agentsdocker.ServicePort{
|
||||
Name: port.Name,
|
||||
Protocol: string(port.Protocol),
|
||||
TargetPort: port.TargetPort,
|
||||
PublishedPort: port.PublishedPort,
|
||||
PublishMode: string(port.PublishMode),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !svc.Meta.CreatedAt.IsZero() {
|
||||
created := svc.Meta.CreatedAt
|
||||
service.CreatedAt = &created
|
||||
}
|
||||
if !svc.Meta.UpdatedAt.IsZero() {
|
||||
updated := svc.Meta.UpdatedAt
|
||||
service.UpdatedAt = &updated
|
||||
}
|
||||
|
||||
return service
|
||||
}
|
||||
|
||||
func mapSwarmTask(task *swarmtypes.Task, svc *swarmtypes.Service, containers map[string]agentsdocker.Container) agentsdocker.Task {
|
||||
result := agentsdocker.Task{
|
||||
ID: task.ID,
|
||||
ServiceID: task.ServiceID,
|
||||
Slot: task.Slot,
|
||||
NodeID: task.NodeID,
|
||||
DesiredState: string(task.DesiredState),
|
||||
CurrentState: string(task.Status.State),
|
||||
Error: task.Status.Err,
|
||||
Message: task.Status.Message,
|
||||
CreatedAt: task.Meta.CreatedAt,
|
||||
}
|
||||
|
||||
if svc != nil {
|
||||
result.ServiceName = svc.Spec.Annotations.Name
|
||||
}
|
||||
|
||||
if !task.Meta.UpdatedAt.IsZero() {
|
||||
updated := task.Meta.UpdatedAt
|
||||
result.UpdatedAt = &updated
|
||||
}
|
||||
|
||||
if ts := task.Status.Timestamp; !ts.IsZero() {
|
||||
if task.Status.State == swarmtypes.TaskStateRunning {
|
||||
started := ts
|
||||
result.StartedAt = &started
|
||||
} else if isTaskCompletedState(string(task.Status.State)) {
|
||||
completed := ts
|
||||
result.CompletedAt = &completed
|
||||
}
|
||||
}
|
||||
|
||||
if cs := task.Status.ContainerStatus; cs != nil {
|
||||
if cs.ContainerID != "" {
|
||||
result.ContainerID = cs.ContainerID
|
||||
if container, ok := lookupContainer(containers, cs.ContainerID); ok {
|
||||
result.ContainerID = container.ID
|
||||
result.ContainerName = container.Name
|
||||
if container.StartedAt != nil && !container.StartedAt.IsZero() && result.StartedAt == nil {
|
||||
started := *container.StartedAt
|
||||
result.StartedAt = &started
|
||||
}
|
||||
if container.FinishedAt != nil && !container.FinishedAt.IsZero() && result.CompletedAt == nil {
|
||||
finished := *container.FinishedAt
|
||||
result.CompletedAt = &finished
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func serviceMode(mode swarmtypes.ServiceMode) string {
|
||||
switch {
|
||||
case mode.Global != nil:
|
||||
return "global"
|
||||
case mode.ReplicatedJob != nil:
|
||||
return "replicated-job"
|
||||
case mode.GlobalJob != nil:
|
||||
return "global-job"
|
||||
case mode.Replicated != nil:
|
||||
return "replicated"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func buildContainerIndex(containers []agentsdocker.Container) map[string]agentsdocker.Container {
|
||||
if len(containers) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
index := make(map[string]agentsdocker.Container, len(containers)*2)
|
||||
for _, c := range containers {
|
||||
index[c.ID] = c
|
||||
if len(c.ID) >= 12 {
|
||||
index[c.ID[:12]] = c
|
||||
}
|
||||
}
|
||||
return index
|
||||
}
|
||||
|
||||
func lookupContainer(index map[string]agentsdocker.Container, id string) (agentsdocker.Container, bool) {
|
||||
if len(index) == 0 {
|
||||
return agentsdocker.Container{}, false
|
||||
}
|
||||
|
||||
if container, ok := index[id]; ok {
|
||||
return container, true
|
||||
}
|
||||
if len(id) > 12 {
|
||||
if container, ok := index[id[:12]]; ok {
|
||||
return container, true
|
||||
}
|
||||
}
|
||||
return agentsdocker.Container{}, false
|
||||
}
|
||||
|
||||
func deriveSwarmTasksFromContainers(containers []agentsdocker.Container, info systemtypes.Info) []agentsdocker.Task {
|
||||
if len(containers) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
tasks := make([]agentsdocker.Task, 0, len(containers))
|
||||
|
||||
for _, container := range containers {
|
||||
if len(container.Labels) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
serviceID := container.Labels["com.docker.swarm.service.id"]
|
||||
serviceName := container.Labels["com.docker.swarm.service.name"]
|
||||
if serviceID == "" && serviceName == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
taskID := container.Labels["com.docker.swarm.task.id"]
|
||||
if taskID == "" {
|
||||
taskID = container.ID
|
||||
}
|
||||
|
||||
task := agentsdocker.Task{
|
||||
ID: taskID,
|
||||
ServiceID: serviceID,
|
||||
ServiceName: serviceName,
|
||||
ContainerID: container.ID,
|
||||
ContainerName: container.Name,
|
||||
NodeID: container.Labels["com.docker.swarm.node.id"],
|
||||
NodeName: container.Labels["com.docker.swarm.node.name"],
|
||||
DesiredState: container.Labels["com.docker.swarm.task.desired-state"],
|
||||
CurrentState: strings.ToLower(strings.TrimSpace(container.State)),
|
||||
CreatedAt: container.CreatedAt,
|
||||
}
|
||||
|
||||
if task.NodeID == "" {
|
||||
task.NodeID = info.Swarm.NodeID
|
||||
}
|
||||
|
||||
if slotStr := container.Labels["com.docker.swarm.task.slot"]; slotStr != "" {
|
||||
if slot, err := strconv.Atoi(slotStr); err == nil {
|
||||
task.Slot = slot
|
||||
}
|
||||
}
|
||||
|
||||
if msg := container.Labels["com.docker.swarm.task.message"]; msg != "" {
|
||||
task.Message = msg
|
||||
}
|
||||
if errMsg := container.Labels["com.docker.swarm.task.error"]; errMsg != "" {
|
||||
task.Error = errMsg
|
||||
}
|
||||
|
||||
if container.StartedAt != nil && !container.StartedAt.IsZero() {
|
||||
started := *container.StartedAt
|
||||
task.StartedAt = &started
|
||||
}
|
||||
if container.FinishedAt != nil && !container.FinishedAt.IsZero() {
|
||||
finished := *container.FinishedAt
|
||||
task.CompletedAt = &finished
|
||||
}
|
||||
|
||||
tasks = append(tasks, task)
|
||||
}
|
||||
|
||||
return tasks
|
||||
}
|
||||
|
||||
func deriveSwarmServicesFromData(tasks []agentsdocker.Task, containers []agentsdocker.Container) []agentsdocker.Service {
|
||||
if len(tasks) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
type aggregate struct {
|
||||
service agentsdocker.Service
|
||||
total int
|
||||
running int
|
||||
completed int
|
||||
}
|
||||
|
||||
aggregates := make(map[string]*aggregate)
|
||||
for _, task := range tasks {
|
||||
key := task.ServiceID
|
||||
if key == "" {
|
||||
key = task.ServiceName
|
||||
}
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
agg, ok := aggregates[key]
|
||||
if !ok {
|
||||
serviceID := task.ServiceID
|
||||
if serviceID == "" {
|
||||
serviceID = key
|
||||
}
|
||||
agg = &aggregate{
|
||||
service: agentsdocker.Service{
|
||||
ID: serviceID,
|
||||
Name: task.ServiceName,
|
||||
},
|
||||
}
|
||||
aggregates[key] = agg
|
||||
}
|
||||
|
||||
if task.ServiceName != "" {
|
||||
agg.service.Name = task.ServiceName
|
||||
}
|
||||
|
||||
agg.total++
|
||||
if strings.EqualFold(task.CurrentState, "running") {
|
||||
agg.running++
|
||||
}
|
||||
if isTaskCompletedState(task.CurrentState) {
|
||||
agg.completed++
|
||||
}
|
||||
}
|
||||
|
||||
if len(aggregates) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, container := range containers {
|
||||
if len(container.Labels) == 0 {
|
||||
continue
|
||||
}
|
||||
key := container.Labels["com.docker.swarm.service.id"]
|
||||
if key == "" {
|
||||
key = container.Labels["com.docker.swarm.service.name"]
|
||||
}
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
agg, ok := aggregates[key]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if agg.service.Image == "" {
|
||||
agg.service.Image = container.Image
|
||||
}
|
||||
if stack := container.Labels["com.docker.stack.namespace"]; stack != "" {
|
||||
if agg.service.Stack == "" {
|
||||
agg.service.Stack = stack
|
||||
}
|
||||
if agg.service.Labels == nil {
|
||||
agg.service.Labels = map[string]string{}
|
||||
}
|
||||
agg.service.Labels["com.docker.stack.namespace"] = stack
|
||||
}
|
||||
}
|
||||
|
||||
services := make([]agentsdocker.Service, 0, len(aggregates))
|
||||
for _, agg := range aggregates {
|
||||
agg.service.DesiredTasks = agg.total
|
||||
agg.service.RunningTasks = agg.running
|
||||
agg.service.CompletedTasks = agg.completed
|
||||
if len(agg.service.Labels) == 0 {
|
||||
agg.service.Labels = nil
|
||||
}
|
||||
services = append(services, agg.service)
|
||||
}
|
||||
|
||||
return services
|
||||
}
|
||||
|
||||
func copyStringMap(source map[string]string) map[string]string {
|
||||
if len(source) == 0 {
|
||||
return nil
|
||||
}
|
||||
result := make(map[string]string, len(source))
|
||||
for k, v := range source {
|
||||
result[k] = v
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func isTaskCompletedState(state string) bool {
|
||||
switch strings.ToLower(state) {
|
||||
case "completed", "complete", "shutdown", "failed", "rejected":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
|
@ -85,6 +85,15 @@ var dockerAgentVersions = []string{
|
|||
"0.1.0-dev",
|
||||
}
|
||||
|
||||
var dockerImageTags = []string{
|
||||
"1.0.0",
|
||||
"1.1.2",
|
||||
"2025.09.1",
|
||||
"2025.10.4",
|
||||
"latest",
|
||||
"stable",
|
||||
}
|
||||
|
||||
var genericHostProfiles = []struct {
|
||||
Platform string
|
||||
OSName string
|
||||
|
|
@ -1079,6 +1088,28 @@ func generateDockerHosts(config MockConfig) []models.DockerHost {
|
|||
}
|
||||
}
|
||||
|
||||
swarmInfo := &models.DockerSwarmInfo{
|
||||
NodeID: fmt.Sprintf("%s-node", hostID),
|
||||
NodeRole: "worker",
|
||||
LocalState: "active",
|
||||
ControlAvailable: false,
|
||||
Scope: "node",
|
||||
}
|
||||
|
||||
var services []models.DockerService
|
||||
var tasks []models.DockerTask
|
||||
if i%2 == 0 {
|
||||
swarmInfo.NodeRole = "manager"
|
||||
swarmInfo.ControlAvailable = true
|
||||
swarmInfo.Scope = "cluster"
|
||||
services, tasks = generateDockerServicesAndTasks(hostname, containers, now)
|
||||
if len(services) == 0 {
|
||||
swarmInfo.Scope = "node"
|
||||
}
|
||||
} else if i%3 == 0 {
|
||||
services, tasks = generateDockerServicesAndTasks(hostname, containers, now)
|
||||
}
|
||||
|
||||
host := models.DockerHost{
|
||||
ID: hostID,
|
||||
AgentID: fmt.Sprintf("agent-%s", randomHexString(6)),
|
||||
|
|
@ -1097,6 +1128,9 @@ func generateDockerHosts(config MockConfig) []models.DockerHost {
|
|||
IntervalSeconds: interval,
|
||||
AgentVersion: agentVersion,
|
||||
Containers: containers,
|
||||
Services: services,
|
||||
Tasks: tasks,
|
||||
Swarm: swarmInfo,
|
||||
}
|
||||
|
||||
hosts = append(hosts, host)
|
||||
|
|
@ -1258,6 +1292,23 @@ func generateHosts(config MockConfig) []models.Host {
|
|||
}
|
||||
}
|
||||
|
||||
var tokenID, tokenName, tokenHint string
|
||||
var tokenLastUsed *time.Time
|
||||
if rand.Float64() < 0.8 {
|
||||
tokenID = fmt.Sprintf("hst_%s", randomHexString(8))
|
||||
tokenName = fmt.Sprintf("%s agent", displayName)
|
||||
tokenHint = fmt.Sprintf("%s…%s", tokenID[:3], tokenID[len(tokenID)-2:])
|
||||
lastUsed := now.Add(-time.Duration(rand.Intn(720)) * time.Minute)
|
||||
tokenLastUsed = &lastUsed
|
||||
}
|
||||
|
||||
if rand.Float64() < 0.1 {
|
||||
// Simulate a revoked token that hasn't been rotated yet
|
||||
tokenID = ""
|
||||
tokenHint = fmt.Sprintf("%s…%s", randomHexString(3), randomHexString(2))
|
||||
tokenLastUsed = nil
|
||||
}
|
||||
|
||||
host := models.Host{
|
||||
ID: fmt.Sprintf("host-%s-%d", profile.Platform, i+1),
|
||||
Hostname: hostname,
|
||||
|
|
@ -1280,6 +1331,10 @@ func generateHosts(config MockConfig) []models.Host {
|
|||
LastSeen: lastSeen,
|
||||
AgentVersion: hostAgentVersions[rand.Intn(len(hostAgentVersions))],
|
||||
Tags: tags,
|
||||
TokenID: tokenID,
|
||||
TokenName: tokenName,
|
||||
TokenHint: tokenHint,
|
||||
TokenLastUsedAt: tokenLastUsed,
|
||||
}
|
||||
|
||||
hosts = append(hosts, host)
|
||||
|
|
@ -3213,6 +3268,11 @@ func updateDockerHosts(data *models.StateSnapshot, config MockConfig) {
|
|||
data.ConnectionHealth[dockerConnectionPrefix+host.ID] = host.Status != "offline"
|
||||
}
|
||||
|
||||
if len(host.Services) > 0 || len(host.Tasks) > 0 {
|
||||
recalculateDockerServiceHealth(host, now)
|
||||
}
|
||||
ensureDockerSwarmInfo(host)
|
||||
|
||||
if host.Status == "offline" {
|
||||
continue
|
||||
}
|
||||
|
|
@ -3244,6 +3304,151 @@ func updateDockerHosts(data *models.StateSnapshot, config MockConfig) {
|
|||
}
|
||||
}
|
||||
|
||||
func serviceKey(id, name string) string {
|
||||
if id != "" {
|
||||
return id
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
func recalculateDockerServiceHealth(host *models.DockerHost, now time.Time) {
|
||||
if host == nil {
|
||||
return
|
||||
}
|
||||
|
||||
containerByID := make(map[string]models.DockerContainer, len(host.Containers))
|
||||
for _, container := range host.Containers {
|
||||
containerByID[container.ID] = container
|
||||
if len(container.ID) >= 12 {
|
||||
containerByID[container.ID[:12]] = container
|
||||
}
|
||||
}
|
||||
|
||||
tasksByService := make(map[string][]int, len(host.Services))
|
||||
for idx := range host.Tasks {
|
||||
task := &host.Tasks[idx]
|
||||
key := serviceKey(task.ServiceID, task.ServiceName)
|
||||
tasksByService[key] = append(tasksByService[key], idx)
|
||||
|
||||
container, ok := containerByID[task.ContainerID]
|
||||
if !ok {
|
||||
if host.Status == "offline" {
|
||||
task.CurrentState = "shutdown"
|
||||
if task.CompletedAt == nil {
|
||||
completed := now.Add(-time.Minute)
|
||||
task.CompletedAt = &completed
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
state := strings.ToLower(container.State)
|
||||
switch state {
|
||||
case "running":
|
||||
task.CurrentState = "running"
|
||||
if container.StartedAt != nil {
|
||||
started := *container.StartedAt
|
||||
task.StartedAt = &started
|
||||
}
|
||||
task.CompletedAt = nil
|
||||
case "paused":
|
||||
task.CurrentState = "paused"
|
||||
if container.StartedAt != nil {
|
||||
started := *container.StartedAt
|
||||
task.StartedAt = &started
|
||||
}
|
||||
default:
|
||||
task.CurrentState = state
|
||||
if container.FinishedAt != nil {
|
||||
finished := *container.FinishedAt
|
||||
task.CompletedAt = &finished
|
||||
} else if host.Status == "offline" {
|
||||
if task.CompletedAt == nil {
|
||||
finished := now.Add(-2 * time.Minute)
|
||||
task.CompletedAt = &finished
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for idx := range host.Services {
|
||||
service := &host.Services[idx]
|
||||
key := serviceKey(service.ID, service.Name)
|
||||
taskIdxs := tasksByService[key]
|
||||
|
||||
if service.DesiredTasks <= 0 {
|
||||
service.DesiredTasks = len(taskIdxs)
|
||||
}
|
||||
|
||||
running := 0
|
||||
completed := 0
|
||||
for _, taskIndex := range taskIdxs {
|
||||
task := host.Tasks[taskIndex]
|
||||
state := strings.ToLower(task.CurrentState)
|
||||
if state == "running" {
|
||||
running++
|
||||
}
|
||||
if task.CompletedAt != nil || state == "shutdown" || state == "failed" {
|
||||
completed++
|
||||
}
|
||||
}
|
||||
|
||||
service.RunningTasks = running
|
||||
service.CompletedTasks = completed
|
||||
|
||||
if service.DesiredTasks > 0 && running < service.DesiredTasks {
|
||||
if service.UpdateStatus == nil {
|
||||
service.UpdateStatus = &models.DockerServiceUpdate{}
|
||||
}
|
||||
service.UpdateStatus.State = "rollback_started"
|
||||
service.UpdateStatus.Message = "Service replicas below desired threshold"
|
||||
service.UpdateStatus.CompletedAt = nil
|
||||
} else if service.UpdateStatus != nil && running >= service.DesiredTasks {
|
||||
service.UpdateStatus = nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func ensureDockerSwarmInfo(host *models.DockerHost) {
|
||||
if host == nil {
|
||||
return
|
||||
}
|
||||
|
||||
if host.Swarm == nil {
|
||||
host.Swarm = &models.DockerSwarmInfo{
|
||||
NodeID: fmt.Sprintf("%s-node", host.ID),
|
||||
NodeRole: "worker",
|
||||
LocalState: "active",
|
||||
ControlAvailable: false,
|
||||
Scope: "node",
|
||||
}
|
||||
}
|
||||
|
||||
if host.Swarm.NodeID == "" {
|
||||
host.Swarm.NodeID = fmt.Sprintf("%s-node", host.ID)
|
||||
}
|
||||
if host.Swarm.NodeRole == "" {
|
||||
host.Swarm.NodeRole = "worker"
|
||||
}
|
||||
|
||||
if host.Status == "offline" {
|
||||
host.Swarm.LocalState = "inactive"
|
||||
} else {
|
||||
host.Swarm.LocalState = "active"
|
||||
}
|
||||
|
||||
if host.Swarm.NodeRole == "manager" {
|
||||
if host.Swarm.ControlAvailable && len(host.Services) > 0 {
|
||||
host.Swarm.Scope = "cluster"
|
||||
} else {
|
||||
host.Swarm.Scope = "node"
|
||||
}
|
||||
} else {
|
||||
host.Swarm.Scope = "node"
|
||||
host.Swarm.ControlAvailable = false
|
||||
}
|
||||
}
|
||||
|
||||
func updateHosts(data *models.StateSnapshot, config MockConfig) {
|
||||
if len(data.Hosts) == 0 {
|
||||
return
|
||||
|
|
@ -3412,3 +3617,147 @@ func generateDisksForNode(node models.Node) []models.PhysicalDisk {
|
|||
|
||||
return disks
|
||||
}
|
||||
|
||||
func generateDockerServicesAndTasks(hostname string, containers []models.DockerContainer, now time.Time) ([]models.DockerService, []models.DockerTask) {
|
||||
if len(containers) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
type svcAgg struct {
|
||||
service models.DockerService
|
||||
tasks []models.DockerTask
|
||||
}
|
||||
|
||||
aggregates := make(map[string]*svcAgg)
|
||||
stackNames := []string{"frontend", "backend", "ops", "infra"}
|
||||
|
||||
for idx, container := range containers {
|
||||
baseName := strings.Split(container.Name, "-")[0]
|
||||
stack := stackNames[idx%len(stackNames)]
|
||||
serviceName := fmt.Sprintf("%s-%s", stack, baseName)
|
||||
serviceID := fmt.Sprintf("svc-%s-%d", stack, idx)
|
||||
|
||||
agg, exists := aggregates[serviceID]
|
||||
if !exists {
|
||||
agg = &svcAgg{
|
||||
service: models.DockerService{
|
||||
ID: serviceID,
|
||||
Name: serviceName,
|
||||
Mode: []string{"replicated", "global"}[rand.Intn(2)],
|
||||
Labels: map[string]string{
|
||||
"com.docker.stack.namespace": stack,
|
||||
},
|
||||
EndpointPorts: []models.DockerServicePort{
|
||||
{
|
||||
Protocol: "tcp",
|
||||
TargetPort: uint32(8000 + idx%10),
|
||||
PublishedPort: uint32(18000 + idx%10),
|
||||
PublishMode: "ingress",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
if rand.Float64() < 0.5 {
|
||||
agg.service.Image = fmt.Sprintf("registry.example.com/%s:%s", serviceName, dockerImageTags[rand.Intn(len(dockerImageTags))])
|
||||
}
|
||||
aggregates[serviceID] = agg
|
||||
}
|
||||
|
||||
desired := 1 + rand.Intn(4)
|
||||
agg.service.DesiredTasks += desired
|
||||
|
||||
slots := desired
|
||||
if agg.service.Mode == "global" {
|
||||
slots = 1
|
||||
}
|
||||
|
||||
for slot := 0; slot < slots; slot++ {
|
||||
currentState := "running"
|
||||
if rand.Float64() < 0.15 {
|
||||
currentState = []string{"failed", "shutdown", "pending", "starting"}[rand.Intn(4)]
|
||||
}
|
||||
|
||||
taskID := fmt.Sprintf("%s-task-%d", serviceID, slot)
|
||||
task := models.DockerTask{
|
||||
ID: taskID,
|
||||
ServiceID: serviceID,
|
||||
ServiceName: serviceName,
|
||||
Slot: slot + 1,
|
||||
NodeID: fmt.Sprintf("node-%s", hostname),
|
||||
NodeName: hostname,
|
||||
DesiredState: "running",
|
||||
CurrentState: currentState,
|
||||
ContainerID: container.ID,
|
||||
ContainerName: container.Name,
|
||||
CreatedAt: now.Add(-time.Duration(rand.Intn(48)) * time.Hour),
|
||||
}
|
||||
|
||||
if container.StartedAt != nil {
|
||||
started := *container.StartedAt
|
||||
task.StartedAt = &started
|
||||
}
|
||||
if container.FinishedAt != nil {
|
||||
finished := *container.FinishedAt
|
||||
task.CompletedAt = &finished
|
||||
}
|
||||
if currentState == "running" {
|
||||
task.StartedAt = ptrTime(now.Add(-time.Duration(30+rand.Intn(3600)) * time.Second))
|
||||
}
|
||||
|
||||
if currentState == "failed" || currentState == "shutdown" {
|
||||
task.Error = "container exit"
|
||||
task.Message = "Replica exited unexpectedly"
|
||||
}
|
||||
|
||||
agg.tasks = append(agg.tasks, task)
|
||||
}
|
||||
}
|
||||
|
||||
services := make([]models.DockerService, 0, len(aggregates))
|
||||
tasks := make([]models.DockerTask, 0, len(containers))
|
||||
|
||||
for _, agg := range aggregates {
|
||||
running := 0
|
||||
completed := 0
|
||||
for _, task := range agg.tasks {
|
||||
if strings.EqualFold(task.CurrentState, "running") {
|
||||
running++
|
||||
}
|
||||
if task.CompletedAt != nil && strings.EqualFold(task.CurrentState, "shutdown") {
|
||||
completed++
|
||||
}
|
||||
}
|
||||
agg.service.RunningTasks = running
|
||||
agg.service.CompletedTasks = completed
|
||||
|
||||
if running < agg.service.DesiredTasks {
|
||||
agg.service.UpdateStatus = &models.DockerServiceUpdate{
|
||||
State: "rollback_started",
|
||||
Message: "Service replicas below desired",
|
||||
CompletedAt: nil,
|
||||
}
|
||||
}
|
||||
|
||||
services = append(services, agg.service)
|
||||
tasks = append(tasks, agg.tasks...)
|
||||
}
|
||||
|
||||
sort.Slice(services, func(i, j int) bool {
|
||||
if services[i].Name == services[j].Name {
|
||||
return services[i].ID < services[j].ID
|
||||
}
|
||||
return services[i].Name < services[j].Name
|
||||
})
|
||||
|
||||
sort.Slice(tasks, func(i, j int) bool {
|
||||
if tasks[i].ServiceName == tasks[j].ServiceName {
|
||||
if tasks[i].Slot == tasks[j].Slot {
|
||||
return tasks[i].ID < tasks[j].ID
|
||||
}
|
||||
return tasks[i].Slot < tasks[j].Slot
|
||||
}
|
||||
return tasks[i].ServiceName < tasks[j].ServiceName
|
||||
})
|
||||
|
||||
return services, tasks
|
||||
}
|
||||
|
|
|
|||
|
|
@ -27,6 +27,105 @@ func TestGenerateMockDataIncludesDockerHosts(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestGenerateMockDataIncludesSwarmServices(t *testing.T) {
|
||||
cfg := DefaultConfig
|
||||
cfg.DockerHostCount = 4
|
||||
cfg.DockerContainersPerHost = 6
|
||||
cfg.RandomMetrics = false
|
||||
|
||||
data := GenerateMockData(cfg)
|
||||
|
||||
found := false
|
||||
for _, host := range data.DockerHosts {
|
||||
if len(host.Services) == 0 {
|
||||
continue
|
||||
}
|
||||
if host.Swarm == nil {
|
||||
t.Fatalf("expected swarm metadata for host %s", host.ID)
|
||||
}
|
||||
if len(host.Tasks) == 0 {
|
||||
t.Fatalf("expected tasks for service host %s", host.ID)
|
||||
}
|
||||
found = true
|
||||
break
|
||||
}
|
||||
|
||||
if !found {
|
||||
t.Fatalf("expected at least one docker host with swarm services")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateMockDataIncludesHostAgents(t *testing.T) {
|
||||
cfg := DefaultConfig
|
||||
cfg.GenericHostCount = 5
|
||||
cfg.RandomMetrics = false
|
||||
|
||||
data := GenerateMockData(cfg)
|
||||
|
||||
if len(data.Hosts) != cfg.GenericHostCount {
|
||||
t.Fatalf("expected %d host agents, got %d", cfg.GenericHostCount, len(data.Hosts))
|
||||
}
|
||||
|
||||
for _, host := range data.Hosts {
|
||||
if host.ID == "" {
|
||||
t.Fatalf("host agent missing id: %+v", host)
|
||||
}
|
||||
if host.Hostname == "" {
|
||||
t.Fatalf("host agent missing hostname: %+v", host)
|
||||
}
|
||||
if host.Status == "" {
|
||||
t.Fatalf("host agent missing status: %+v", host)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMockStateIncludesHostAgents(t *testing.T) {
|
||||
SetEnabled(true)
|
||||
t.Cleanup(func() {
|
||||
SetEnabled(false)
|
||||
})
|
||||
|
||||
state := GetMockState()
|
||||
if len(state.Hosts) == 0 {
|
||||
t.Fatalf("expected hosts in mock state, got %d", len(state.Hosts))
|
||||
}
|
||||
|
||||
frontend := state.ToFrontend()
|
||||
if len(frontend.Hosts) == 0 {
|
||||
t.Fatalf("expected hosts in frontend state, got %d", len(frontend.Hosts))
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateMetricsMaintainsServiceHealth(t *testing.T) {
|
||||
cfg := DefaultConfig
|
||||
cfg.DockerHostCount = 3
|
||||
cfg.DockerContainersPerHost = 6
|
||||
|
||||
data := GenerateMockData(cfg)
|
||||
UpdateMetrics(&data, cfg)
|
||||
|
||||
for _, host := range data.DockerHosts {
|
||||
if len(host.Services) == 0 {
|
||||
continue
|
||||
}
|
||||
if host.Swarm == nil {
|
||||
t.Fatalf("expected swarm metadata for host %s after update", host.ID)
|
||||
}
|
||||
|
||||
for _, svc := range host.Services {
|
||||
if svc.DesiredTasks < 0 {
|
||||
t.Fatalf("service %s has negative desired tasks", svc.Name)
|
||||
}
|
||||
if svc.RunningTasks < 0 {
|
||||
t.Fatalf("service %s has negative running tasks", svc.Name)
|
||||
}
|
||||
if svc.RunningTasks > svc.DesiredTasks && svc.DesiredTasks > 0 {
|
||||
t.Fatalf("service %s has running (%d) > desired (%d)", svc.Name, svc.RunningTasks, svc.DesiredTasks)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateMockDataIncludesPMGInstances(t *testing.T) {
|
||||
cfg := DefaultConfig
|
||||
|
||||
|
|
|
|||
|
|
@ -93,6 +93,7 @@ func enableMockMode(fromInit bool) {
|
|||
Int("nodes", config.NodeCount).
|
||||
Int("vms_per_node", config.VMsPerNode).
|
||||
Int("lxcs_per_node", config.LXCsPerNode).
|
||||
Int("host_agents", config.GenericHostCount).
|
||||
Int("docker_hosts", config.DockerHostCount).
|
||||
Int("docker_containers_per_host", config.DockerContainersPerHost).
|
||||
Bool("random_metrics", config.RandomMetrics).
|
||||
|
|
@ -207,6 +208,12 @@ func LoadMockConfig() MockConfig {
|
|||
}
|
||||
}
|
||||
|
||||
if val := os.Getenv("PULSE_MOCK_GENERIC_HOSTS"); val != "" {
|
||||
if n, err := strconv.Atoi(val); err == nil && n >= 0 {
|
||||
config.GenericHostCount = n
|
||||
}
|
||||
}
|
||||
|
||||
if val := os.Getenv("PULSE_MOCK_RANDOM_METRICS"); val != "" {
|
||||
config.RandomMetrics = val == "true"
|
||||
}
|
||||
|
|
@ -304,6 +311,7 @@ func cloneState(state models.StateSnapshot) models.StateSnapshot {
|
|||
VMs: append([]models.VM(nil), state.VMs...),
|
||||
Containers: append([]models.Container(nil), state.Containers...),
|
||||
DockerHosts: append([]models.DockerHost(nil), state.DockerHosts...),
|
||||
Hosts: append([]models.Host(nil), state.Hosts...),
|
||||
PMGInstances: append([]models.PMGInstance(nil), state.PMGInstances...),
|
||||
Storage: append([]models.Storage(nil), state.Storage...),
|
||||
CephClusters: append([]models.CephCluster(nil), state.CephClusters...),
|
||||
|
|
|
|||
|
|
@ -235,6 +235,25 @@ func (d DockerHost) ToFrontend() DockerHostFrontend {
|
|||
h.Containers[i] = ct.ToFrontend()
|
||||
}
|
||||
|
||||
if len(d.Services) > 0 {
|
||||
h.Services = make([]DockerServiceFrontend, len(d.Services))
|
||||
for i, svc := range d.Services {
|
||||
h.Services[i] = svc.ToFrontend()
|
||||
}
|
||||
}
|
||||
|
||||
if len(d.Tasks) > 0 {
|
||||
h.Tasks = make([]DockerTaskFrontend, len(d.Tasks))
|
||||
for i, task := range d.Tasks {
|
||||
h.Tasks[i] = task.ToFrontend()
|
||||
}
|
||||
}
|
||||
|
||||
if d.Swarm != nil {
|
||||
sw := d.Swarm.ToFrontend()
|
||||
h.Swarm = &sw
|
||||
}
|
||||
|
||||
if d.Command != nil {
|
||||
h.Command = toDockerHostCommandFrontend(*d.Command)
|
||||
}
|
||||
|
|
@ -362,6 +381,126 @@ func (c DockerContainer) ToFrontend() DockerContainerFrontend {
|
|||
return container
|
||||
}
|
||||
|
||||
// ToFrontend converts a DockerService to DockerServiceFrontend.
|
||||
func (s DockerService) ToFrontend() DockerServiceFrontend {
|
||||
service := DockerServiceFrontend{
|
||||
ID: s.ID,
|
||||
Name: s.Name,
|
||||
Stack: s.Stack,
|
||||
Image: s.Image,
|
||||
Mode: s.Mode,
|
||||
DesiredTasks: s.DesiredTasks,
|
||||
RunningTasks: s.RunningTasks,
|
||||
CompletedTasks: s.CompletedTasks,
|
||||
Labels: nil,
|
||||
}
|
||||
|
||||
if len(s.Labels) > 0 {
|
||||
service.Labels = make(map[string]string, len(s.Labels))
|
||||
for k, v := range s.Labels {
|
||||
service.Labels[k] = v
|
||||
}
|
||||
}
|
||||
|
||||
if len(s.EndpointPorts) > 0 {
|
||||
service.EndpointPorts = make([]DockerServicePortFrontend, len(s.EndpointPorts))
|
||||
for i, port := range s.EndpointPorts {
|
||||
service.EndpointPorts[i] = port.ToFrontend()
|
||||
}
|
||||
}
|
||||
|
||||
if s.UpdateStatus != nil {
|
||||
update := s.UpdateStatus.ToFrontend()
|
||||
service.UpdateStatus = &update
|
||||
}
|
||||
|
||||
if s.CreatedAt != nil && !s.CreatedAt.IsZero() {
|
||||
ts := s.CreatedAt.Unix() * 1000
|
||||
service.CreatedAt = &ts
|
||||
}
|
||||
if s.UpdatedAt != nil && !s.UpdatedAt.IsZero() {
|
||||
ts := s.UpdatedAt.Unix() * 1000
|
||||
service.UpdatedAt = &ts
|
||||
}
|
||||
|
||||
return service
|
||||
}
|
||||
|
||||
// ToFrontend converts a DockerServicePort to DockerServicePortFrontend.
|
||||
func (p DockerServicePort) ToFrontend() DockerServicePortFrontend {
|
||||
return DockerServicePortFrontend{
|
||||
Name: p.Name,
|
||||
Protocol: p.Protocol,
|
||||
TargetPort: p.TargetPort,
|
||||
PublishedPort: p.PublishedPort,
|
||||
PublishMode: p.PublishMode,
|
||||
}
|
||||
}
|
||||
|
||||
// ToFrontend converts a DockerServiceUpdate to DockerServiceUpdateFrontend.
|
||||
func (u DockerServiceUpdate) ToFrontend() DockerServiceUpdateFrontend {
|
||||
update := DockerServiceUpdateFrontend{
|
||||
State: u.State,
|
||||
Message: u.Message,
|
||||
}
|
||||
if u.CompletedAt != nil && !u.CompletedAt.IsZero() {
|
||||
ts := u.CompletedAt.Unix() * 1000
|
||||
update.CompletedAt = &ts
|
||||
}
|
||||
return update
|
||||
}
|
||||
|
||||
// ToFrontend converts a DockerTask to DockerTaskFrontend.
|
||||
func (t DockerTask) ToFrontend() DockerTaskFrontend {
|
||||
task := DockerTaskFrontend{
|
||||
ID: t.ID,
|
||||
ServiceID: t.ServiceID,
|
||||
ServiceName: t.ServiceName,
|
||||
Slot: t.Slot,
|
||||
NodeID: t.NodeID,
|
||||
NodeName: t.NodeName,
|
||||
DesiredState: t.DesiredState,
|
||||
CurrentState: t.CurrentState,
|
||||
Error: t.Error,
|
||||
Message: t.Message,
|
||||
ContainerID: t.ContainerID,
|
||||
ContainerName: t.ContainerName,
|
||||
}
|
||||
|
||||
if !t.CreatedAt.IsZero() {
|
||||
ts := t.CreatedAt.Unix() * 1000
|
||||
task.CreatedAt = &ts
|
||||
}
|
||||
if t.UpdatedAt != nil && !t.UpdatedAt.IsZero() {
|
||||
ts := t.UpdatedAt.Unix() * 1000
|
||||
task.UpdatedAt = &ts
|
||||
}
|
||||
if t.StartedAt != nil && !t.StartedAt.IsZero() {
|
||||
ts := t.StartedAt.Unix() * 1000
|
||||
task.StartedAt = &ts
|
||||
}
|
||||
if t.CompletedAt != nil && !t.CompletedAt.IsZero() {
|
||||
ts := t.CompletedAt.Unix() * 1000
|
||||
task.CompletedAt = &ts
|
||||
}
|
||||
|
||||
return task
|
||||
}
|
||||
|
||||
// ToFrontend converts DockerSwarmInfo to DockerSwarmFrontend.
|
||||
func (s DockerSwarmInfo) ToFrontend() DockerSwarmFrontend {
|
||||
return DockerSwarmFrontend{
|
||||
NodeID: s.NodeID,
|
||||
NodeRole: s.NodeRole,
|
||||
LocalState: s.LocalState,
|
||||
ControlAvailable: s.ControlAvailable,
|
||||
ClusterID: s.ClusterID,
|
||||
ClusterName: s.ClusterName,
|
||||
Scope: s.Scope,
|
||||
Error: s.Error,
|
||||
}
|
||||
}
|
||||
|
||||
func hostSensorSummaryToFrontend(src HostSensorSummary) *HostSensorSummaryFrontend {
|
||||
if len(src.TemperatureCelsius) == 0 && len(src.FanRPM) == 0 && len(src.Additional) == 0 {
|
||||
return nil
|
||||
|
|
|
|||
|
|
@ -207,6 +207,9 @@ type DockerHost struct {
|
|||
IntervalSeconds int `json:"intervalSeconds"`
|
||||
AgentVersion string `json:"agentVersion,omitempty"`
|
||||
Containers []DockerContainer `json:"containers"`
|
||||
Services []DockerService `json:"services,omitempty"`
|
||||
Tasks []DockerTask `json:"tasks,omitempty"`
|
||||
Swarm *DockerSwarmInfo `json:"swarm,omitempty"`
|
||||
TokenID string `json:"tokenId,omitempty"`
|
||||
TokenName string `json:"tokenName,omitempty"`
|
||||
TokenHint string `json:"tokenHint,omitempty"`
|
||||
|
|
@ -254,6 +257,71 @@ type DockerContainerNetworkLink struct {
|
|||
IPv6 string `json:"ipv6,omitempty"`
|
||||
}
|
||||
|
||||
// DockerService summarises a Docker Swarm service.
|
||||
type DockerService struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Stack string `json:"stack,omitempty"`
|
||||
Image string `json:"image,omitempty"`
|
||||
Mode string `json:"mode,omitempty"`
|
||||
DesiredTasks int `json:"desiredTasks,omitempty"`
|
||||
RunningTasks int `json:"runningTasks,omitempty"`
|
||||
CompletedTasks int `json:"completedTasks,omitempty"`
|
||||
UpdateStatus *DockerServiceUpdate `json:"updateStatus,omitempty"`
|
||||
Labels map[string]string `json:"labels,omitempty"`
|
||||
EndpointPorts []DockerServicePort `json:"endpointPorts,omitempty"`
|
||||
CreatedAt *time.Time `json:"createdAt,omitempty"`
|
||||
UpdatedAt *time.Time `json:"updatedAt,omitempty"`
|
||||
}
|
||||
|
||||
// DockerServicePort describes a published service port.
|
||||
type DockerServicePort struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
Protocol string `json:"protocol,omitempty"`
|
||||
TargetPort uint32 `json:"targetPort,omitempty"`
|
||||
PublishedPort uint32 `json:"publishedPort,omitempty"`
|
||||
PublishMode string `json:"publishMode,omitempty"`
|
||||
}
|
||||
|
||||
// DockerServiceUpdate captures service update progress.
|
||||
type DockerServiceUpdate struct {
|
||||
State string `json:"state,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
CompletedAt *time.Time `json:"completedAt,omitempty"`
|
||||
}
|
||||
|
||||
// DockerTask summarises a Swarm task.
|
||||
type DockerTask struct {
|
||||
ID string `json:"id"`
|
||||
ServiceID string `json:"serviceId,omitempty"`
|
||||
ServiceName string `json:"serviceName,omitempty"`
|
||||
Slot int `json:"slot,omitempty"`
|
||||
NodeID string `json:"nodeId,omitempty"`
|
||||
NodeName string `json:"nodeName,omitempty"`
|
||||
DesiredState string `json:"desiredState,omitempty"`
|
||||
CurrentState string `json:"currentState,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
ContainerID string `json:"containerId,omitempty"`
|
||||
ContainerName string `json:"containerName,omitempty"`
|
||||
CreatedAt time.Time `json:"createdAt,omitempty"`
|
||||
UpdatedAt *time.Time `json:"updatedAt,omitempty"`
|
||||
StartedAt *time.Time `json:"startedAt,omitempty"`
|
||||
CompletedAt *time.Time `json:"completedAt,omitempty"`
|
||||
}
|
||||
|
||||
// DockerSwarmInfo captures node-level swarm metadata.
|
||||
type DockerSwarmInfo struct {
|
||||
NodeID string `json:"nodeId,omitempty"`
|
||||
NodeRole string `json:"nodeRole,omitempty"`
|
||||
LocalState string `json:"localState,omitempty"`
|
||||
ControlAvailable bool `json:"controlAvailable,omitempty"`
|
||||
ClusterID string `json:"clusterId,omitempty"`
|
||||
ClusterName string `json:"clusterName,omitempty"`
|
||||
Scope string `json:"scope,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// DockerHostCommandStatus tracks the lifecycle of a control command issued to a Docker host.
|
||||
type DockerHostCommandStatus struct {
|
||||
ID string `json:"id"`
|
||||
|
|
|
|||
|
|
@ -114,6 +114,9 @@ type DockerHostFrontend struct {
|
|||
IntervalSeconds int `json:"intervalSeconds"`
|
||||
AgentVersion string `json:"agentVersion,omitempty"`
|
||||
Containers []DockerContainerFrontend `json:"containers"`
|
||||
Services []DockerServiceFrontend `json:"services,omitempty"`
|
||||
Tasks []DockerTaskFrontend `json:"tasks,omitempty"`
|
||||
Swarm *DockerSwarmFrontend `json:"swarm,omitempty"`
|
||||
TokenID string `json:"tokenId,omitempty"`
|
||||
TokenName string `json:"tokenName,omitempty"`
|
||||
TokenHint string `json:"tokenHint,omitempty"`
|
||||
|
|
@ -160,6 +163,71 @@ type DockerContainerNetworkFrontend struct {
|
|||
IPv6 string `json:"ipv6,omitempty"`
|
||||
}
|
||||
|
||||
// DockerServiceFrontend represents a Swarm service for the frontend.
|
||||
type DockerServiceFrontend struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Stack string `json:"stack,omitempty"`
|
||||
Image string `json:"image,omitempty"`
|
||||
Mode string `json:"mode,omitempty"`
|
||||
DesiredTasks int `json:"desiredTasks,omitempty"`
|
||||
RunningTasks int `json:"runningTasks,omitempty"`
|
||||
CompletedTasks int `json:"completedTasks,omitempty"`
|
||||
Labels map[string]string `json:"labels,omitempty"`
|
||||
EndpointPorts []DockerServicePortFrontend `json:"endpointPorts,omitempty"`
|
||||
UpdateStatus *DockerServiceUpdateFrontend `json:"updateStatus,omitempty"`
|
||||
CreatedAt *int64 `json:"createdAt,omitempty"`
|
||||
UpdatedAt *int64 `json:"updatedAt,omitempty"`
|
||||
}
|
||||
|
||||
// DockerServicePortFrontend represents a published service port.
|
||||
type DockerServicePortFrontend struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
Protocol string `json:"protocol,omitempty"`
|
||||
TargetPort uint32 `json:"targetPort,omitempty"`
|
||||
PublishedPort uint32 `json:"publishedPort,omitempty"`
|
||||
PublishMode string `json:"publishMode,omitempty"`
|
||||
}
|
||||
|
||||
// DockerServiceUpdateFrontend exposes service update status to the UI.
|
||||
type DockerServiceUpdateFrontend struct {
|
||||
State string `json:"state,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
CompletedAt *int64 `json:"completedAt,omitempty"`
|
||||
}
|
||||
|
||||
// DockerTaskFrontend represents a Swarm task replica.
|
||||
type DockerTaskFrontend struct {
|
||||
ID string `json:"id"`
|
||||
ServiceID string `json:"serviceId,omitempty"`
|
||||
ServiceName string `json:"serviceName,omitempty"`
|
||||
Slot int `json:"slot,omitempty"`
|
||||
NodeID string `json:"nodeId,omitempty"`
|
||||
NodeName string `json:"nodeName,omitempty"`
|
||||
DesiredState string `json:"desiredState,omitempty"`
|
||||
CurrentState string `json:"currentState,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
ContainerID string `json:"containerId,omitempty"`
|
||||
ContainerName string `json:"containerName,omitempty"`
|
||||
CreatedAt *int64 `json:"createdAt,omitempty"`
|
||||
UpdatedAt *int64 `json:"updatedAt,omitempty"`
|
||||
StartedAt *int64 `json:"startedAt,omitempty"`
|
||||
CompletedAt *int64 `json:"completedAt,omitempty"`
|
||||
}
|
||||
|
||||
// DockerSwarmFrontend summarises node-level swarm details.
|
||||
type DockerSwarmFrontend struct {
|
||||
NodeID string `json:"nodeId,omitempty"`
|
||||
NodeRole string `json:"nodeRole,omitempty"`
|
||||
LocalState string `json:"localState,omitempty"`
|
||||
ControlAvailable bool `json:"controlAvailable,omitempty"`
|
||||
ClusterID string `json:"clusterId,omitempty"`
|
||||
ClusterName string `json:"clusterName,omitempty"`
|
||||
Scope string `json:"scope,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// DockerHostCommandFrontend exposes docker host command state to the UI.
|
||||
type DockerHostCommandFrontend struct {
|
||||
ID string `json:"id"`
|
||||
|
|
|
|||
|
|
@ -530,6 +530,139 @@ func cloneStringFloatMap(src map[string]float64) map[string]float64 {
|
|||
return out
|
||||
}
|
||||
|
||||
func cloneStringMap(src map[string]string) map[string]string {
|
||||
if len(src) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]string, len(src))
|
||||
for k, v := range src {
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func convertDockerServices(services []agentsdocker.Service) []models.DockerService {
|
||||
if len(services) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
result := make([]models.DockerService, 0, len(services))
|
||||
for _, svc := range services {
|
||||
service := models.DockerService{
|
||||
ID: svc.ID,
|
||||
Name: svc.Name,
|
||||
Stack: svc.Stack,
|
||||
Image: svc.Image,
|
||||
Mode: svc.Mode,
|
||||
DesiredTasks: svc.DesiredTasks,
|
||||
RunningTasks: svc.RunningTasks,
|
||||
CompletedTasks: svc.CompletedTasks,
|
||||
}
|
||||
|
||||
if len(svc.Labels) > 0 {
|
||||
service.Labels = cloneStringMap(svc.Labels)
|
||||
}
|
||||
|
||||
if len(svc.EndpointPorts) > 0 {
|
||||
ports := make([]models.DockerServicePort, len(svc.EndpointPorts))
|
||||
for i, port := range svc.EndpointPorts {
|
||||
ports[i] = models.DockerServicePort{
|
||||
Name: port.Name,
|
||||
Protocol: port.Protocol,
|
||||
TargetPort: port.TargetPort,
|
||||
PublishedPort: port.PublishedPort,
|
||||
PublishMode: port.PublishMode,
|
||||
}
|
||||
}
|
||||
service.EndpointPorts = ports
|
||||
}
|
||||
|
||||
if svc.UpdateStatus != nil {
|
||||
update := &models.DockerServiceUpdate{
|
||||
State: svc.UpdateStatus.State,
|
||||
Message: svc.UpdateStatus.Message,
|
||||
}
|
||||
if svc.UpdateStatus.CompletedAt != nil && !svc.UpdateStatus.CompletedAt.IsZero() {
|
||||
completed := *svc.UpdateStatus.CompletedAt
|
||||
update.CompletedAt = &completed
|
||||
}
|
||||
service.UpdateStatus = update
|
||||
}
|
||||
|
||||
if svc.CreatedAt != nil && !svc.CreatedAt.IsZero() {
|
||||
created := *svc.CreatedAt
|
||||
service.CreatedAt = &created
|
||||
}
|
||||
if svc.UpdatedAt != nil && !svc.UpdatedAt.IsZero() {
|
||||
updated := *svc.UpdatedAt
|
||||
service.UpdatedAt = &updated
|
||||
}
|
||||
|
||||
result = append(result, service)
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func convertDockerTasks(tasks []agentsdocker.Task) []models.DockerTask {
|
||||
if len(tasks) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
result := make([]models.DockerTask, 0, len(tasks))
|
||||
for _, task := range tasks {
|
||||
modelTask := models.DockerTask{
|
||||
ID: task.ID,
|
||||
ServiceID: task.ServiceID,
|
||||
ServiceName: task.ServiceName,
|
||||
Slot: task.Slot,
|
||||
NodeID: task.NodeID,
|
||||
NodeName: task.NodeName,
|
||||
DesiredState: task.DesiredState,
|
||||
CurrentState: task.CurrentState,
|
||||
Error: task.Error,
|
||||
Message: task.Message,
|
||||
ContainerID: task.ContainerID,
|
||||
ContainerName: task.ContainerName,
|
||||
CreatedAt: task.CreatedAt,
|
||||
}
|
||||
|
||||
if task.UpdatedAt != nil && !task.UpdatedAt.IsZero() {
|
||||
updated := *task.UpdatedAt
|
||||
modelTask.UpdatedAt = &updated
|
||||
}
|
||||
if task.StartedAt != nil && !task.StartedAt.IsZero() {
|
||||
started := *task.StartedAt
|
||||
modelTask.StartedAt = &started
|
||||
}
|
||||
if task.CompletedAt != nil && !task.CompletedAt.IsZero() {
|
||||
completed := *task.CompletedAt
|
||||
modelTask.CompletedAt = &completed
|
||||
}
|
||||
|
||||
result = append(result, modelTask)
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func convertDockerSwarmInfo(info *agentsdocker.SwarmInfo) *models.DockerSwarmInfo {
|
||||
if info == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
return &models.DockerSwarmInfo{
|
||||
NodeID: info.NodeID,
|
||||
NodeRole: info.NodeRole,
|
||||
LocalState: info.LocalState,
|
||||
ControlAvailable: info.ControlAvailable,
|
||||
ClusterID: info.ClusterID,
|
||||
ClusterName: info.ClusterName,
|
||||
Scope: info.Scope,
|
||||
Error: info.Error,
|
||||
}
|
||||
}
|
||||
|
||||
// shouldRunBackupPoll determines whether a backup polling cycle should execute.
|
||||
// Returns whether polling should run, a human-readable skip reason, and the timestamp to record.
|
||||
func (m *Monitor) shouldRunBackupPoll(last time.Time, now time.Time) (bool, string, time.Time) {
|
||||
|
|
@ -1253,6 +1386,10 @@ func (m *Monitor) ApplyDockerReport(report agentsdocker.Report, tokenRecord *con
|
|||
containers = append(containers, container)
|
||||
}
|
||||
|
||||
services := convertDockerServices(report.Services)
|
||||
tasks := convertDockerTasks(report.Tasks)
|
||||
swarmInfo := convertDockerSwarmInfo(report.Host.Swarm)
|
||||
|
||||
host := models.DockerHost{
|
||||
ID: identifier,
|
||||
AgentID: agentID,
|
||||
|
|
@ -1271,6 +1408,9 @@ func (m *Monitor) ApplyDockerReport(report agentsdocker.Report, tokenRecord *con
|
|||
IntervalSeconds: report.Agent.IntervalSeconds,
|
||||
AgentVersion: report.Agent.Version,
|
||||
Containers: containers,
|
||||
Services: services,
|
||||
Tasks: tasks,
|
||||
Swarm: swarmInfo,
|
||||
}
|
||||
|
||||
if tokenRecord != nil {
|
||||
|
|
|
|||
|
|
@ -161,7 +161,7 @@ func GetCurrentVersion() (*VersionInfo, error) {
|
|||
}
|
||||
|
||||
// Final fallback
|
||||
return buildInfo("4.25.0", "release", false), nil
|
||||
return buildInfo("4.26.0", "release", false), nil
|
||||
}
|
||||
|
||||
// normalizeVersionString ensures any version string can be parsed as semantic version.
|
||||
|
|
|
|||
5
mock.env
5
mock.env
|
|
@ -12,7 +12,7 @@
|
|||
# Documentation: docs/development/MOCK_MODE.md
|
||||
|
||||
# Enable/disable mock mode (false = use real Proxmox infrastructure)
|
||||
PULSE_MOCK_MODE=false
|
||||
PULSE_MOCK_MODE=true
|
||||
|
||||
# Number of mock nodes to generate (mix of clustered and standalone)
|
||||
# First 5 nodes form a cluster, remaining nodes are standalone
|
||||
|
|
@ -26,6 +26,9 @@ PULSE_MOCK_VMS_PER_NODE=5
|
|||
# Containers have lighter resource usage than VMs
|
||||
PULSE_MOCK_LXCS_PER_NODE=8
|
||||
|
||||
# Number of standalone hosts (Pulse host agents) to simulate
|
||||
PULSE_MOCK_GENERIC_HOSTS=6
|
||||
|
||||
# Number of Docker hosts to simulate
|
||||
PULSE_MOCK_DOCKER_HOSTS=3
|
||||
|
||||
|
|
|
|||
|
|
@ -7,6 +7,8 @@ type Report struct {
|
|||
Agent AgentInfo `json:"agent"`
|
||||
Host HostInfo `json:"host"`
|
||||
Containers []Container `json:"containers"`
|
||||
Services []Service `json:"services,omitempty"`
|
||||
Tasks []Task `json:"tasks,omitempty"`
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
}
|
||||
|
||||
|
|
@ -19,16 +21,17 @@ type AgentInfo struct {
|
|||
|
||||
// HostInfo contains metadata about the Docker host where the agent runs.
|
||||
type HostInfo struct {
|
||||
Hostname string `json:"hostname"`
|
||||
Name string `json:"name,omitempty"`
|
||||
MachineID string `json:"machineId,omitempty"`
|
||||
OS string `json:"os,omitempty"`
|
||||
KernelVersion string `json:"kernelVersion,omitempty"`
|
||||
Architecture string `json:"architecture,omitempty"`
|
||||
DockerVersion string `json:"dockerVersion,omitempty"`
|
||||
TotalCPU int `json:"totalCpu,omitempty"`
|
||||
TotalMemoryBytes int64 `json:"totalMemoryBytes,omitempty"`
|
||||
UptimeSeconds int64 `json:"uptimeSeconds,omitempty"`
|
||||
Hostname string `json:"hostname"`
|
||||
Name string `json:"name,omitempty"`
|
||||
MachineID string `json:"machineId,omitempty"`
|
||||
OS string `json:"os,omitempty"`
|
||||
KernelVersion string `json:"kernelVersion,omitempty"`
|
||||
Architecture string `json:"architecture,omitempty"`
|
||||
DockerVersion string `json:"dockerVersion,omitempty"`
|
||||
TotalCPU int `json:"totalCpu,omitempty"`
|
||||
TotalMemoryBytes int64 `json:"totalMemoryBytes,omitempty"`
|
||||
UptimeSeconds int64 `json:"uptimeSeconds,omitempty"`
|
||||
Swarm *SwarmInfo `json:"swarm,omitempty"`
|
||||
}
|
||||
|
||||
// Container captures the runtime state for a Docker container at report time.
|
||||
|
|
@ -79,3 +82,68 @@ func (r Report) AgentKey() string {
|
|||
}
|
||||
return r.Host.Hostname
|
||||
}
|
||||
|
||||
// SwarmInfo captures metadata about the Docker Swarm state for the reporting node.
|
||||
type SwarmInfo struct {
|
||||
NodeID string `json:"nodeId,omitempty"`
|
||||
NodeRole string `json:"nodeRole,omitempty"`
|
||||
LocalState string `json:"localState,omitempty"`
|
||||
ControlAvailable bool `json:"controlAvailable,omitempty"`
|
||||
ClusterID string `json:"clusterId,omitempty"`
|
||||
ClusterName string `json:"clusterName,omitempty"`
|
||||
Scope string `json:"scope,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// Service summarises a Docker Swarm service and its aggregate status.
|
||||
type Service struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Stack string `json:"stack,omitempty"`
|
||||
Image string `json:"image,omitempty"`
|
||||
Mode string `json:"mode,omitempty"`
|
||||
DesiredTasks int `json:"desiredTasks,omitempty"`
|
||||
RunningTasks int `json:"runningTasks,omitempty"`
|
||||
CompletedTasks int `json:"completedTasks,omitempty"`
|
||||
Labels map[string]string `json:"labels,omitempty"`
|
||||
EndpointPorts []ServicePort `json:"endpointPorts,omitempty"`
|
||||
UpdateStatus *ServiceUpdate `json:"updateStatus,omitempty"`
|
||||
CreatedAt *time.Time `json:"createdAt,omitempty"`
|
||||
UpdatedAt *time.Time `json:"updatedAt,omitempty"`
|
||||
}
|
||||
|
||||
// ServicePort describes an exposed service endpoint.
|
||||
type ServicePort struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
Protocol string `json:"protocol,omitempty"`
|
||||
TargetPort uint32 `json:"targetPort,omitempty"`
|
||||
PublishedPort uint32 `json:"publishedPort,omitempty"`
|
||||
PublishMode string `json:"publishMode,omitempty"`
|
||||
}
|
||||
|
||||
// ServiceUpdate captures the current rolling update status for a service.
|
||||
type ServiceUpdate struct {
|
||||
State string `json:"state,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
CompletedAt *time.Time `json:"completedAt,omitempty"`
|
||||
}
|
||||
|
||||
// Task summarises an individual Docker Swarm task (replica).
|
||||
type Task struct {
|
||||
ID string `json:"id"`
|
||||
ServiceID string `json:"serviceId,omitempty"`
|
||||
ServiceName string `json:"serviceName,omitempty"`
|
||||
Slot int `json:"slot,omitempty"`
|
||||
NodeID string `json:"nodeId,omitempty"`
|
||||
NodeName string `json:"nodeName,omitempty"`
|
||||
DesiredState string `json:"desiredState,omitempty"`
|
||||
CurrentState string `json:"currentState,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
ContainerID string `json:"containerId,omitempty"`
|
||||
ContainerName string `json:"containerName,omitempty"`
|
||||
CreatedAt time.Time `json:"createdAt,omitempty"`
|
||||
UpdatedAt *time.Time `json:"updatedAt,omitempty"`
|
||||
StartedAt *time.Time `json:"startedAt,omitempty"`
|
||||
CompletedAt *time.Time `json:"completedAt,omitempty"`
|
||||
}
|
||||
|
|
|
|||
|
|
@ -108,9 +108,10 @@ show_status() {
|
|||
source "$MOCK_ENV_FILE"
|
||||
if [ "$PULSE_MOCK_MODE" = "true" ]; then
|
||||
echo -e "${GREEN}Mock Mode: ENABLED${NC}"
|
||||
echo " Nodes: $PULSE_MOCK_NODES"
|
||||
echo " VMs per node: $PULSE_MOCK_VMS_PER_NODE"
|
||||
echo " LXCs per node: $PULSE_MOCK_LXCS_PER_NODE"
|
||||
echo " Nodes: ${PULSE_MOCK_NODES:-0}"
|
||||
echo " VMs per node: ${PULSE_MOCK_VMS_PER_NODE:-0}"
|
||||
echo " LXCs per node: ${PULSE_MOCK_LXCS_PER_NODE:-0}"
|
||||
echo " Host agents: ${PULSE_MOCK_GENERIC_HOSTS:-0}"
|
||||
echo " Docker hosts: ${PULSE_MOCK_DOCKER_HOSTS:-0}"
|
||||
echo " Docker containers/host: ${PULSE_MOCK_DOCKER_CONTAINERS:-0}"
|
||||
echo " Data dir: ${PULSE_DATA_DIR:-/opt/pulse/tmp/mock-data}"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue