mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-04-26 10:31:17 +00:00
security: complete Phase 1 sensor proxy hardening
Implements comprehensive security hardening for pulse-sensor-proxy: - Privilege drop from root to unprivileged user (UID 995) - Hash-chained tamper-evident audit logging with remote forwarding - Per-UID rate limiting (0.2 QPS, burst 2) with concurrency caps - Enhanced command validation with 10+ attack pattern tests - Fuzz testing (7M+ executions, 0 crashes) - SSH hardening, AppArmor/seccomp profiles, operational runbooks All 27 Phase 1 tasks complete. Ready for production deployment.
This commit is contained in:
parent
20d94f4c90
commit
524f42cc28
57 changed files with 4104 additions and 1519 deletions
7
.gitignore
vendored
7
.gitignore
vendored
|
|
@ -65,6 +65,7 @@ AI_DEVELOPMENT.md
|
|||
scripts/pulse-watchdog.sh
|
||||
pulse-watchdog.log
|
||||
.mcp-servers/
|
||||
.codex/
|
||||
|
||||
# Release process files
|
||||
CHANGELOG.md
|
||||
|
|
@ -132,6 +133,12 @@ MOCK_MODE_GUIDE.md
|
|||
secrets.env
|
||||
*secret*.env
|
||||
|
||||
# Browser/session artifacts
|
||||
**/cookies.txt
|
||||
**/cookies-*.txt
|
||||
**/*.har
|
||||
**/*.browser
|
||||
|
||||
# Development documentation (local only)
|
||||
CLAUDE_DEV_SETUP.md
|
||||
AGENT_METRICS_*.md
|
||||
|
|
|
|||
80
Dockerfile
80
Dockerfile
|
|
@ -1,3 +1,6 @@
|
|||
# syntax=docker/dockerfile:1.7-labs
|
||||
ARG BUILD_AGENT=1
|
||||
|
||||
# Build stage for frontend (must be built first for embedding)
|
||||
FROM node:20-alpine AS frontend-builder
|
||||
|
||||
|
|
@ -5,17 +8,20 @@ WORKDIR /app/frontend-modern
|
|||
|
||||
# Copy package files
|
||||
COPY frontend-modern/package*.json ./
|
||||
RUN npm ci
|
||||
RUN --mount=type=cache,id=pulse-npm-cache,target=/root/.npm \
|
||||
npm ci
|
||||
|
||||
# Copy frontend source
|
||||
COPY frontend-modern/ ./
|
||||
|
||||
# Build frontend
|
||||
RUN npm run build
|
||||
RUN --mount=type=cache,id=pulse-npm-cache,target=/root/.npm \
|
||||
npm run build
|
||||
|
||||
# Build stage for Go backend
|
||||
FROM golang:1.24-alpine AS backend-builder
|
||||
|
||||
ARG BUILD_AGENT
|
||||
WORKDIR /app
|
||||
|
||||
# Install build dependencies
|
||||
|
|
@ -23,7 +29,9 @@ RUN apk add --no-cache git
|
|||
|
||||
# Copy go mod files for better layer caching
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
RUN --mount=type=cache,id=pulse-go-mod,target=/go/pkg/mod \
|
||||
--mount=type=cache,id=pulse-go-build,target=/root/.cache/go-build \
|
||||
go mod download
|
||||
|
||||
# Copy only necessary source code
|
||||
COPY cmd/ ./cmd/
|
||||
|
|
@ -36,27 +44,46 @@ COPY VERSION ./
|
|||
COPY --from=frontend-builder /app/frontend-modern/dist ./internal/api/frontend-modern/dist
|
||||
|
||||
# Build the binaries with embedded frontend
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build \
|
||||
-ldflags="-s -w" \
|
||||
-trimpath \
|
||||
-o pulse ./cmd/pulse
|
||||
RUN --mount=type=cache,id=pulse-go-mod,target=/go/pkg/mod \
|
||||
--mount=type=cache,id=pulse-go-build,target=/root/.cache/go-build \
|
||||
CGO_ENABLED=0 GOOS=linux go build \
|
||||
-ldflags="-s -w" \
|
||||
-trimpath \
|
||||
-o pulse ./cmd/pulse
|
||||
|
||||
# Build docker-agent for multiple architectures so users can download any arch
|
||||
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \
|
||||
-ldflags="-s -w" \
|
||||
-trimpath \
|
||||
-o pulse-docker-agent-linux-amd64 ./cmd/pulse-docker-agent && \
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build \
|
||||
-ldflags="-s -w" \
|
||||
-trimpath \
|
||||
-o pulse-docker-agent-linux-arm64 ./cmd/pulse-docker-agent && \
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=7 go build \
|
||||
-ldflags="-s -w" \
|
||||
-trimpath \
|
||||
-o pulse-docker-agent-linux-armv7 ./cmd/pulse-docker-agent
|
||||
# Build docker-agent binaries (optional cross-arch builds controlled by BUILD_AGENT)
|
||||
RUN --mount=type=cache,id=pulse-go-mod,target=/go/pkg/mod \
|
||||
--mount=type=cache,id=pulse-go-build,target=/root/.cache/go-build \
|
||||
if [ "${BUILD_AGENT:-1}" = "1" ]; then \
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \
|
||||
-ldflags="-s -w" \
|
||||
-trimpath \
|
||||
-o pulse-docker-agent-linux-amd64 ./cmd/pulse-docker-agent && \
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build \
|
||||
-ldflags="-s -w" \
|
||||
-trimpath \
|
||||
-o pulse-docker-agent-linux-arm64 ./cmd/pulse-docker-agent && \
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=7 go build \
|
||||
-ldflags="-s -w" \
|
||||
-trimpath \
|
||||
-o pulse-docker-agent-linux-armv7 ./cmd/pulse-docker-agent; \
|
||||
else \
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \
|
||||
-ldflags="-s -w" \
|
||||
-trimpath \
|
||||
-o pulse-docker-agent-linux-amd64 ./cmd/pulse-docker-agent && \
|
||||
cp pulse-docker-agent-linux-amd64 pulse-docker-agent-linux-arm64 && \
|
||||
cp pulse-docker-agent-linux-amd64 pulse-docker-agent-linux-armv7; \
|
||||
fi && \
|
||||
cp pulse-docker-agent-linux-amd64 pulse-docker-agent
|
||||
|
||||
# Keep a host-arch symlink for backward compatibility
|
||||
RUN cp pulse-docker-agent-linux-amd64 pulse-docker-agent
|
||||
# Build pulse-sensor-proxy
|
||||
RUN --mount=type=cache,id=pulse-go-mod,target=/go/pkg/mod \
|
||||
--mount=type=cache,id=pulse-go-build,target=/root/.cache/go-build \
|
||||
CGO_ENABLED=0 GOOS=linux go build \
|
||||
-ldflags="-s -w" \
|
||||
-trimpath \
|
||||
-o pulse-sensor-proxy ./cmd/pulse-sensor-proxy
|
||||
|
||||
# Runtime image for the Docker agent (offered via --target agent_runtime)
|
||||
FROM alpine:latest AS agent_runtime
|
||||
|
|
@ -106,10 +133,12 @@ COPY --from=backend-builder /app/VERSION .
|
|||
COPY docker-entrypoint.sh /docker-entrypoint.sh
|
||||
RUN chmod +x /docker-entrypoint.sh
|
||||
|
||||
# Provide docker-agent installer script for HTTP download endpoint
|
||||
# Provide installer scripts for HTTP download endpoints
|
||||
RUN mkdir -p /opt/pulse/scripts
|
||||
COPY scripts/install-docker-agent.sh /opt/pulse/scripts/install-docker-agent.sh
|
||||
RUN chmod 755 /opt/pulse/scripts/install-docker-agent.sh
|
||||
COPY scripts/install-sensor-proxy.sh /opt/pulse/scripts/install-sensor-proxy.sh
|
||||
COPY scripts/install-docker.sh /opt/pulse/scripts/install-docker.sh
|
||||
RUN chmod 755 /opt/pulse/scripts/install-docker-agent.sh /opt/pulse/scripts/install-sensor-proxy.sh /opt/pulse/scripts/install-docker.sh
|
||||
|
||||
# Copy multi-arch docker-agent binaries for download endpoint
|
||||
RUN mkdir -p /opt/pulse/bin
|
||||
|
|
@ -118,6 +147,9 @@ COPY --from=backend-builder /app/pulse-docker-agent-linux-arm64 /opt/pulse/bin/
|
|||
COPY --from=backend-builder /app/pulse-docker-agent-linux-armv7 /opt/pulse/bin/
|
||||
COPY --from=backend-builder /app/pulse-docker-agent /opt/pulse/bin/pulse-docker-agent
|
||||
|
||||
# Copy pulse-sensor-proxy binary for download endpoint
|
||||
COPY --from=backend-builder /app/pulse-sensor-proxy /opt/pulse/bin/pulse-sensor-proxy
|
||||
|
||||
# Create config directory
|
||||
RUN mkdir -p /etc/pulse /data
|
||||
|
||||
|
|
|
|||
187
SECURITY.md
Normal file
187
SECURITY.md
Normal file
|
|
@ -0,0 +1,187 @@
|
|||
# Pulse Security Documentation
|
||||
|
||||
## Critical Security Notice for Production Deployments
|
||||
|
||||
### Container SSH Key Policy (BREAKING CHANGE)
|
||||
|
||||
**Effective immediately, SSH-based temperature monitoring is BLOCKED in containerized Pulse deployments.**
|
||||
|
||||
#### Why This Change?
|
||||
|
||||
Storing SSH private keys inside Docker containers creates an unacceptable security risk in production environments:
|
||||
|
||||
- **Container compromise = Infrastructure compromise**: If an attacker gains access to your Pulse container, they immediately obtain SSH private keys with root access to your Proxmox infrastructure.
|
||||
- **Keys persist in images**: SSH keys can be extracted from container layers and images if pushed to registries.
|
||||
- **No key rotation**: Long-lived keys in containers are difficult to rotate.
|
||||
- **Violates principle of least privilege**: Containers should not hold credentials for the infrastructure they monitor.
|
||||
|
||||
#### Affected Deployments
|
||||
|
||||
✅ **Not Affected** (SSH temperature monitoring still allowed):
|
||||
- Pulse installed directly on a VM or bare metal (non-containerized)
|
||||
- Home lab deployments where you understand and accept the risk
|
||||
|
||||
❌ **BLOCKED** (SSH temperature monitoring disabled):
|
||||
- Pulse running in Docker containers
|
||||
- Pulse running in LXC containers
|
||||
- Any deployment where `PULSE_DOCKER=true` or `/.dockerenv` exists
|
||||
|
||||
#### Migration Path
|
||||
|
||||
**For Production Container Deployments:**
|
||||
|
||||
1. **Deploy pulse-sensor-proxy on each Proxmox host:**
|
||||
```bash
|
||||
# On each Proxmox host
|
||||
curl -o /usr/local/bin/pulse-sensor-proxy \
|
||||
https://github.com/rcourtman/pulse/releases/latest/download/pulse-sensor-proxy
|
||||
|
||||
chmod +x /usr/local/bin/pulse-sensor-proxy
|
||||
```
|
||||
|
||||
2. **Create systemd service** (`/etc/systemd/system/pulse-sensor-proxy.service`):
|
||||
```ini
|
||||
[Unit]
|
||||
Description=Pulse Temperature Sensor Proxy
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=root
|
||||
ExecStart=/usr/local/bin/pulse-sensor-proxy
|
||||
Restart=on-failure
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
```
|
||||
|
||||
3. **Enable and start:**
|
||||
```bash
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now pulse-sensor-proxy
|
||||
```
|
||||
|
||||
4. **Restart Pulse container** - it will automatically detect and use the proxy
|
||||
|
||||
**Removing Existing SSH Keys:**
|
||||
|
||||
If you previously used SSH-based temperature monitoring in containers:
|
||||
|
||||
```bash
|
||||
# On each Proxmox host, remove Pulse SSH keys
|
||||
sed -i '/# pulse-/d' /root/.ssh/authorized_keys
|
||||
|
||||
# Inside the Pulse container (or destroy and recreate)
|
||||
docker exec pulse rm -rf /home/pulse/.ssh/id_ed25519*
|
||||
```
|
||||
|
||||
#### Technical Details
|
||||
|
||||
**How pulse-sensor-proxy Works:**
|
||||
|
||||
- Runs as a lightweight daemon on the Proxmox host
|
||||
- Exposes a Unix socket at `/run/pulse-sensor-proxy.sock`
|
||||
- Pulse container connects via bind-mounted socket
|
||||
- Only exposes `sensors -j` output - no SSH access
|
||||
- Keys never leave the Proxmox host
|
||||
|
||||
**Security Boundaries:**
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────┐
|
||||
│ Proxmox Host │
|
||||
│ ┌───────────────────────────────┐ │
|
||||
│ │ pulse-sensor-proxy (root) │ │
|
||||
│ │ - Runs sensors -j │ │
|
||||
│ │ - Unix socket only │ │
|
||||
│ └───────────────────────────────┘ │
|
||||
│ │ │
|
||||
│ │ /run/pulse-sensor-proxy.sock
|
||||
│ │ │
|
||||
│ ┌─────────▼─────────────────────┐ │
|
||||
│ │ Container (bind mount) │ │
|
||||
│ │ - No SSH keys │ │
|
||||
│ │ - No root access to host │ │
|
||||
│ └───────────────────────────────┘ │
|
||||
└─────────────────────────────────────┘
|
||||
```
|
||||
|
||||
#### For Home Lab Users
|
||||
|
||||
If you understand and accept the risk, you can still use non-containerized Pulse with SSH keys:
|
||||
|
||||
1. Install Pulse directly on a VM (not in Docker)
|
||||
2. Setup script will offer SSH temperature monitoring
|
||||
3. Follow standard security practices:
|
||||
- Use dedicated monitoring user (not root)
|
||||
- Restrict key with `command="sensors -j"`
|
||||
- Add `from="<pulse-ip>"` restrictions
|
||||
- Rotate keys periodically
|
||||
|
||||
#### Audit Your Deployment
|
||||
|
||||
**Check if you're affected:**
|
||||
```bash
|
||||
# Inside Pulse container
|
||||
ls /home/pulse/.ssh/id_ed25519* 2>/dev/null && echo "⚠️ VULNERABLE"
|
||||
|
||||
# On Proxmox host
|
||||
grep "# pulse-" /root/.ssh/authorized_keys && echo "⚠️ SSH keys present"
|
||||
```
|
||||
|
||||
**Check if proxy is working:**
|
||||
```bash
|
||||
# On Proxmox host
|
||||
systemctl status pulse-sensor-proxy
|
||||
|
||||
# Inside Pulse container
|
||||
docker logs pulse | grep -i "temperature proxy detected"
|
||||
```
|
||||
|
||||
#### Timeline
|
||||
|
||||
- **Now**: SSH key generation blocked in containers (code-level enforcement)
|
||||
- **Next Release**: Setup script updated with clear warnings
|
||||
- **Future**: pulse-sensor-proxy bundled in official releases
|
||||
|
||||
#### Questions?
|
||||
|
||||
- Documentation: https://docs.pulseapp.io/security/containerized-deployments
|
||||
- GitHub Issues: https://github.com/rcourtman/pulse/issues
|
||||
- Security Issues: security@pulseapp.io (private disclosure)
|
||||
|
||||
---
|
||||
|
||||
## General Security Best Practices
|
||||
|
||||
### Authentication
|
||||
|
||||
- Use API tokens with minimal required permissions
|
||||
- Rotate tokens regularly
|
||||
- Never commit tokens to version control
|
||||
- Use read-only tokens where possible
|
||||
|
||||
### Network Security
|
||||
|
||||
- Run Pulse in a dedicated monitoring VLAN
|
||||
- Restrict Pulse's network access to only monitored systems
|
||||
- Use firewall rules to limit inbound connections
|
||||
- Enable TLS for all Proxmox API connections
|
||||
|
||||
### Monitoring
|
||||
|
||||
- Enable audit logging on Proxmox hosts
|
||||
- Monitor Pulse container logs for suspicious activity
|
||||
- Set up alerts for failed authentication attempts
|
||||
- Review access logs regularly
|
||||
|
||||
### Updates
|
||||
|
||||
- Keep Pulse updated to latest stable version
|
||||
- Subscribe to security announcements
|
||||
- Test updates in staging before production
|
||||
- Have rollback plan ready
|
||||
|
||||
---
|
||||
|
||||
Last updated: 2025-10-19
|
||||
290
cmd/pulse-sensor-proxy/audit.go
Normal file
290
cmd/pulse-sensor-proxy/audit.go
Normal file
|
|
@ -0,0 +1,290 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
// auditLogger emits append-only, hash-chained audit events.
|
||||
type auditLogger struct {
|
||||
mu sync.Mutex
|
||||
file *os.File
|
||||
logger zerolog.Logger
|
||||
prevHash []byte
|
||||
sequence uint64
|
||||
}
|
||||
|
||||
// AuditEvent captures a single security-relevant action.
|
||||
type AuditEvent struct {
|
||||
Sequence uint64 `json:"seq"`
|
||||
Timestamp time.Time `json:"ts"`
|
||||
EventType string `json:"event_type"`
|
||||
CorrelationID string `json:"correlation_id,omitempty"`
|
||||
PeerUID *uint32 `json:"peer_uid,omitempty"`
|
||||
PeerGID *uint32 `json:"peer_gid,omitempty"`
|
||||
PeerPID *uint32 `json:"peer_pid,omitempty"`
|
||||
RemoteAddr string `json:"remote_addr,omitempty"`
|
||||
Command string `json:"command,omitempty"`
|
||||
Args []string `json:"args,omitempty"`
|
||||
Target string `json:"target,omitempty"`
|
||||
Decision string `json:"decision,omitempty"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
Limiter string `json:"limiter,omitempty"`
|
||||
ExitCode *int `json:"exit_code,omitempty"`
|
||||
DurationMs *int64 `json:"duration_ms,omitempty"`
|
||||
StdoutHash string `json:"stdout_sha256,omitempty"`
|
||||
StderrHash string `json:"stderr_sha256,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
PrevHash string `json:"prev_hash"`
|
||||
EventHash string `json:"event_hash"`
|
||||
}
|
||||
|
||||
// newAuditLogger opens the audit log file and prepares hash chaining.
|
||||
func newAuditLogger(path string) (*auditLogger, error) {
|
||||
file, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o640)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
writer := zerolog.New(file).With().Timestamp().Logger()
|
||||
|
||||
return &auditLogger{
|
||||
file: file,
|
||||
logger: writer,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Close flushes and closes the audit log file.
|
||||
func (a *auditLogger) Close() error {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
if a.file == nil {
|
||||
return nil
|
||||
}
|
||||
err := a.file.Close()
|
||||
a.file = nil
|
||||
return err
|
||||
}
|
||||
|
||||
// LogConnectionAccepted records an authorized connection.
|
||||
func (a *auditLogger) LogConnectionAccepted(correlationID string, cred *peerCredentials, remote string) {
|
||||
event := AuditEvent{
|
||||
EventType: "connection.accepted",
|
||||
CorrelationID: correlationID,
|
||||
RemoteAddr: remote,
|
||||
Decision: "allowed",
|
||||
}
|
||||
event.applyPeer(cred)
|
||||
a.log(&event)
|
||||
}
|
||||
|
||||
// LogConnectionDenied records a rejected connection attempt.
|
||||
func (a *auditLogger) LogConnectionDenied(correlationID string, cred *peerCredentials, remote, reason string) {
|
||||
event := AuditEvent{
|
||||
EventType: "connection.denied",
|
||||
CorrelationID: correlationID,
|
||||
RemoteAddr: remote,
|
||||
Decision: "denied",
|
||||
Reason: reason,
|
||||
}
|
||||
event.applyPeer(cred)
|
||||
a.log(&event)
|
||||
}
|
||||
|
||||
// LogRateLimitHit records limiter rejections.
|
||||
func (a *auditLogger) LogRateLimitHit(correlationID string, cred *peerCredentials, remote, limiter string) {
|
||||
event := AuditEvent{
|
||||
EventType: "limiter.rejection",
|
||||
CorrelationID: correlationID,
|
||||
RemoteAddr: remote,
|
||||
Decision: "denied",
|
||||
Limiter: limiter,
|
||||
}
|
||||
event.applyPeer(cred)
|
||||
a.log(&event)
|
||||
}
|
||||
|
||||
// LogCommandStart records command execution approval.
|
||||
func (a *auditLogger) LogCommandStart(correlationID string, cred *peerCredentials, remote, target, command string, args []string) {
|
||||
event := AuditEvent{
|
||||
EventType: "command.start",
|
||||
CorrelationID: correlationID,
|
||||
RemoteAddr: remote,
|
||||
Decision: "allowed",
|
||||
Command: command,
|
||||
Args: args,
|
||||
Target: target,
|
||||
}
|
||||
event.applyPeer(cred)
|
||||
a.log(&event)
|
||||
}
|
||||
|
||||
// LogCommandResult records command completion.
|
||||
func (a *auditLogger) LogCommandResult(correlationID string, cred *peerCredentials, remote, target, command string, args []string, exitCode int, duration time.Duration, stdoutHash, stderrHash string, execErr error) {
|
||||
event := AuditEvent{
|
||||
EventType: "command.finish",
|
||||
CorrelationID: correlationID,
|
||||
RemoteAddr: remote,
|
||||
Command: command,
|
||||
Args: args,
|
||||
Target: target,
|
||||
ExitCode: intPtr(exitCode),
|
||||
StdoutHash: stdoutHash,
|
||||
StderrHash: stderrHash,
|
||||
}
|
||||
event.applyPeer(cred)
|
||||
if duration > 0 {
|
||||
ms := duration.Milliseconds()
|
||||
event.DurationMs = int64Ptr(ms)
|
||||
}
|
||||
if execErr != nil {
|
||||
event.Error = execErr.Error()
|
||||
event.Decision = "failed"
|
||||
} else {
|
||||
event.Decision = "completed"
|
||||
}
|
||||
a.log(&event)
|
||||
}
|
||||
|
||||
// LogValidationFailure records validator rejections.
|
||||
func (a *auditLogger) LogValidationFailure(correlationID string, cred *peerCredentials, remote, command string, args []string, reason string) {
|
||||
event := AuditEvent{
|
||||
EventType: "command.validation_failed",
|
||||
CorrelationID: correlationID,
|
||||
RemoteAddr: remote,
|
||||
Command: command,
|
||||
Args: args,
|
||||
Decision: "denied",
|
||||
Reason: reason,
|
||||
}
|
||||
event.applyPeer(cred)
|
||||
a.log(&event)
|
||||
}
|
||||
|
||||
func (e *AuditEvent) applyPeer(cred *peerCredentials) {
|
||||
if cred == nil {
|
||||
return
|
||||
}
|
||||
e.PeerUID = uint32Ptr(cred.uid)
|
||||
e.PeerGID = uint32Ptr(cred.gid)
|
||||
e.PeerPID = uint32Ptr(cred.pid)
|
||||
}
|
||||
|
||||
// log persists the event with hash chaining.
|
||||
func (a *auditLogger) log(event *AuditEvent) {
|
||||
if event == nil {
|
||||
log.Error().Msg("audit log called with nil event")
|
||||
return
|
||||
}
|
||||
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
|
||||
a.sequence++
|
||||
event.Sequence = a.sequence
|
||||
|
||||
if event.Timestamp.IsZero() {
|
||||
event.Timestamp = time.Now().UTC()
|
||||
} else {
|
||||
event.Timestamp = event.Timestamp.UTC()
|
||||
}
|
||||
|
||||
event.PrevHash = hex.EncodeToString(a.prevHash)
|
||||
|
||||
payload, err := eventMarshalForHash(event)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("failed to marshal audit event")
|
||||
return
|
||||
}
|
||||
|
||||
sum := sha256.Sum256(append(a.prevHash, payload...))
|
||||
a.prevHash = sum[:]
|
||||
event.EventHash = hex.EncodeToString(sum[:])
|
||||
|
||||
a.logger.Info().Fields(eventToMap(event)).Send()
|
||||
}
|
||||
|
||||
func eventMarshalForHash(event *AuditEvent) ([]byte, error) {
|
||||
clone := *event
|
||||
clone.EventHash = ""
|
||||
return json.Marshal(clone)
|
||||
}
|
||||
|
||||
func eventToMap(event *AuditEvent) map[string]interface{} {
|
||||
m := map[string]interface{}{
|
||||
"ts": event.Timestamp.Format(time.RFC3339Nano),
|
||||
"event_type": event.EventType,
|
||||
"seq": event.Sequence,
|
||||
"prev_hash": event.PrevHash,
|
||||
"event_hash": event.EventHash,
|
||||
"decision": event.Decision,
|
||||
"correlation_id": event.CorrelationID,
|
||||
}
|
||||
|
||||
if event.PeerUID != nil {
|
||||
m["peer_uid"] = *event.PeerUID
|
||||
}
|
||||
if event.PeerGID != nil {
|
||||
m["peer_gid"] = *event.PeerGID
|
||||
}
|
||||
if event.PeerPID != nil {
|
||||
m["peer_pid"] = *event.PeerPID
|
||||
}
|
||||
if event.RemoteAddr != "" {
|
||||
m["remote_addr"] = event.RemoteAddr
|
||||
}
|
||||
if event.Command != "" {
|
||||
m["command"] = event.Command
|
||||
}
|
||||
if len(event.Args) > 0 {
|
||||
m["args"] = event.Args
|
||||
}
|
||||
if event.Target != "" {
|
||||
m["target"] = event.Target
|
||||
}
|
||||
if event.Reason != "" {
|
||||
m["reason"] = event.Reason
|
||||
}
|
||||
if event.Limiter != "" {
|
||||
m["limiter"] = event.Limiter
|
||||
}
|
||||
if event.ExitCode != nil {
|
||||
m["exit_code"] = *event.ExitCode
|
||||
}
|
||||
if event.DurationMs != nil {
|
||||
m["duration_ms"] = *event.DurationMs
|
||||
}
|
||||
if event.StdoutHash != "" {
|
||||
m["stdout_sha256"] = event.StdoutHash
|
||||
}
|
||||
if event.StderrHash != "" {
|
||||
m["stderr_sha256"] = event.StderrHash
|
||||
}
|
||||
if event.Error != "" {
|
||||
m["error"] = event.Error
|
||||
}
|
||||
|
||||
return m
|
||||
}
|
||||
|
||||
func uint32Ptr(v uint32) *uint32 {
|
||||
value := v
|
||||
return &value
|
||||
}
|
||||
|
||||
func intPtr(v int) *int {
|
||||
value := v
|
||||
return &value
|
||||
}
|
||||
|
||||
func int64Ptr(v int64) *int64 {
|
||||
value := v
|
||||
return &value
|
||||
}
|
||||
64
cmd/pulse-sensor-proxy/audit_test.go
Normal file
64
cmd/pulse-sensor-proxy/audit_test.go
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type auditRecord map[string]interface{}
|
||||
|
||||
func TestAuditLogValidationFailure(t *testing.T) {
|
||||
tmp, err := os.CreateTemp("", "audit-test-*.log")
|
||||
if err != nil {
|
||||
t.Fatalf("temp file: %v", err)
|
||||
}
|
||||
path := tmp.Name()
|
||||
tmp.Close()
|
||||
defer os.Remove(path)
|
||||
|
||||
logger, err := newAuditLogger(path)
|
||||
if err != nil {
|
||||
t.Fatalf("newAuditLogger: %v", err)
|
||||
}
|
||||
|
||||
cred := &peerCredentials{uid: 1000, gid: 1000, pid: 4242}
|
||||
logger.LogValidationFailure("corr-123", cred, "remote", "get_temperature", []string{"node"}, "invalid_node")
|
||||
logger.Close()
|
||||
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("open log: %v", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
scanner := bufio.NewScanner(file)
|
||||
if !scanner.Scan() {
|
||||
t.Fatalf("expected at least one audit entry")
|
||||
}
|
||||
|
||||
var record auditRecord
|
||||
if err := json.Unmarshal(scanner.Bytes(), &record); err != nil {
|
||||
t.Fatalf("unmarshal: %v", err)
|
||||
}
|
||||
|
||||
if record["event_type"] != "command.validation_failed" {
|
||||
t.Fatalf("unexpected event_type: %v", record["event_type"])
|
||||
}
|
||||
if record["correlation_id"] != "corr-123" {
|
||||
t.Fatalf("unexpected correlation id: %v", record["correlation_id"])
|
||||
}
|
||||
if record["command"] != "get_temperature" {
|
||||
t.Fatalf("unexpected command: %v", record["command"])
|
||||
}
|
||||
if record["reason"] != "invalid_node" {
|
||||
t.Fatalf("unexpected reason: %v", record["reason"])
|
||||
}
|
||||
if record["decision"] != "denied" {
|
||||
t.Fatalf("unexpected decision: %v", record["decision"])
|
||||
}
|
||||
if record["event_hash"] == "" {
|
||||
t.Fatalf("expected event_hash to be set")
|
||||
}
|
||||
}
|
||||
|
|
@ -11,14 +11,18 @@ import (
|
|||
"net"
|
||||
"os"
|
||||
"os/signal"
|
||||
"os/user"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/ssh/knownhosts"
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/rs/zerolog/log"
|
||||
"github.com/spf13/cobra"
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// Version information (set at build time with -ldflags)
|
||||
|
|
@ -29,10 +33,12 @@ var (
|
|||
)
|
||||
|
||||
const (
|
||||
defaultSocketPath = "/run/pulse-sensor-proxy/pulse-sensor-proxy.sock"
|
||||
defaultSSHKeyPath = "/var/lib/pulse-sensor-proxy/ssh"
|
||||
defaultConfigPath = "/etc/pulse-sensor-proxy/config.yaml"
|
||||
maxRequestBytes = 16 * 1024 // 16 KiB max request size
|
||||
defaultSocketPath = "/run/pulse-sensor-proxy/pulse-sensor-proxy.sock"
|
||||
defaultSSHKeyPath = "/var/lib/pulse-sensor-proxy/ssh"
|
||||
defaultConfigPath = "/etc/pulse-sensor-proxy/config.yaml"
|
||||
defaultAuditLogPath = "/var/log/pulse/sensor-proxy/audit.log"
|
||||
maxRequestBytes = 16 * 1024 // 16 KiB max request size
|
||||
defaultRunAsUser = "pulse-sensor"
|
||||
)
|
||||
|
||||
func defaultWorkDir() string {
|
||||
|
|
@ -79,17 +85,155 @@ func main() {
|
|||
}
|
||||
}
|
||||
|
||||
type userSpec struct {
|
||||
name string
|
||||
uid int
|
||||
gid int
|
||||
groups []int
|
||||
home string
|
||||
}
|
||||
|
||||
func dropPrivileges(username string) (*userSpec, error) {
|
||||
if username == "" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if os.Geteuid() != 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
spec, err := resolveUserSpec(username)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(spec.groups) == 0 {
|
||||
spec.groups = []int{spec.gid}
|
||||
}
|
||||
|
||||
if err := unix.Setgroups(spec.groups); err != nil {
|
||||
return nil, fmt.Errorf("setgroups: %w", err)
|
||||
}
|
||||
if err := unix.Setgid(spec.gid); err != nil {
|
||||
return nil, fmt.Errorf("setgid: %w", err)
|
||||
}
|
||||
if err := unix.Setuid(spec.uid); err != nil {
|
||||
return nil, fmt.Errorf("setuid: %w", err)
|
||||
}
|
||||
|
||||
if spec.home != "" {
|
||||
_ = os.Setenv("HOME", spec.home)
|
||||
}
|
||||
if spec.name != "" {
|
||||
_ = os.Setenv("USER", spec.name)
|
||||
_ = os.Setenv("LOGNAME", spec.name)
|
||||
}
|
||||
|
||||
return spec, nil
|
||||
}
|
||||
|
||||
func resolveUserSpec(username string) (*userSpec, error) {
|
||||
u, err := user.Lookup(username)
|
||||
if err == nil {
|
||||
uid, err := strconv.Atoi(u.Uid)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse uid %q: %w", u.Uid, err)
|
||||
}
|
||||
gid, err := strconv.Atoi(u.Gid)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse gid %q: %w", u.Gid, err)
|
||||
}
|
||||
|
||||
var groups []int
|
||||
if gids, err := u.GroupIds(); err == nil {
|
||||
for _, g := range gids {
|
||||
if gidVal, convErr := strconv.Atoi(g); convErr == nil {
|
||||
groups = append(groups, gidVal)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(groups) == 0 {
|
||||
groups = []int{gid}
|
||||
}
|
||||
|
||||
return &userSpec{
|
||||
name: u.Username,
|
||||
uid: uid,
|
||||
gid: gid,
|
||||
groups: groups,
|
||||
home: u.HomeDir,
|
||||
}, nil
|
||||
}
|
||||
|
||||
fallbackSpec, fallbackErr := lookupUserFromPasswd(username)
|
||||
if fallbackErr == nil {
|
||||
return fallbackSpec, nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("lookup user %q failed: %v (fallback: %w)", username, err, fallbackErr)
|
||||
}
|
||||
|
||||
func lookupUserFromPasswd(username string) (*userSpec, error) {
|
||||
f, err := os.Open("/etc/passwd")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open /etc/passwd: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
scanner := bufio.NewScanner(f)
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
if strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
|
||||
fields := strings.Split(line, ":")
|
||||
if len(fields) < 7 {
|
||||
continue
|
||||
}
|
||||
if fields[0] != username {
|
||||
continue
|
||||
}
|
||||
|
||||
uid, err := strconv.Atoi(fields[2])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse uid %q: %w", fields[2], err)
|
||||
}
|
||||
gid, err := strconv.Atoi(fields[3])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse gid %q: %w", fields[3], err)
|
||||
}
|
||||
|
||||
return &userSpec{
|
||||
name: fields[0],
|
||||
uid: uid,
|
||||
gid: gid,
|
||||
groups: []int{gid},
|
||||
home: fields[5],
|
||||
}, nil
|
||||
}
|
||||
|
||||
if err := scanner.Err(); err != nil {
|
||||
return nil, fmt.Errorf("scan /etc/passwd: %w", err)
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("user %q not found in /etc/passwd", username)
|
||||
}
|
||||
|
||||
// Proxy manages the temperature monitoring proxy
|
||||
type Proxy struct {
|
||||
socketPath string
|
||||
sshKeyPath string
|
||||
workDir string
|
||||
knownHosts knownhosts.Manager
|
||||
listener net.Listener
|
||||
rateLimiter *rateLimiter
|
||||
nodeGate *nodeGate
|
||||
router map[string]handlerFunc
|
||||
config *Config
|
||||
metrics *ProxyMetrics
|
||||
audit *auditLogger
|
||||
|
||||
allowedPeerUIDs map[uint32]struct{}
|
||||
allowedPeerGIDs map[uint32]struct{}
|
||||
|
|
@ -161,6 +305,32 @@ func runProxy() {
|
|||
log.Fatal().Err(err).Msg("Failed to load configuration")
|
||||
}
|
||||
|
||||
runAsUser := os.Getenv("PULSE_SENSOR_PROXY_USER")
|
||||
if runAsUser == "" {
|
||||
runAsUser = defaultRunAsUser
|
||||
}
|
||||
|
||||
if spec, err := dropPrivileges(runAsUser); err != nil {
|
||||
log.Fatal().Err(err).Str("user", runAsUser).Msg("Failed to drop privileges")
|
||||
} else if spec != nil {
|
||||
log.Info().
|
||||
Str("user", spec.name).
|
||||
Int("uid", spec.uid).
|
||||
Int("gid", spec.gid).
|
||||
Msg("Running as unprivileged user")
|
||||
}
|
||||
|
||||
auditPath := os.Getenv("PULSE_SENSOR_PROXY_AUDIT_LOG")
|
||||
if auditPath == "" {
|
||||
auditPath = defaultAuditLogPath
|
||||
}
|
||||
|
||||
auditLogger, err := newAuditLogger(auditPath)
|
||||
if err != nil {
|
||||
log.Fatal().Err(err).Str("path", auditPath).Msg("Failed to initialize audit logger")
|
||||
}
|
||||
defer auditLogger.Close()
|
||||
|
||||
// Initialize metrics
|
||||
metrics := NewProxyMetrics(Version)
|
||||
|
||||
|
|
@ -168,16 +338,24 @@ func runProxy() {
|
|||
Str("socket", socketPath).
|
||||
Str("ssh_key_dir", sshKeyPath).
|
||||
Str("config_path", cfgPath).
|
||||
Str("audit_log", auditPath).
|
||||
Str("version", Version).
|
||||
Msg("Starting pulse-sensor-proxy")
|
||||
|
||||
knownHostsManager, err := knownhosts.NewManager(filepath.Join(sshKeyPath, "known_hosts"))
|
||||
if err != nil {
|
||||
log.Fatal().Err(err).Msg("Failed to initialize known hosts manager")
|
||||
}
|
||||
|
||||
proxy := &Proxy{
|
||||
socketPath: socketPath,
|
||||
sshKeyPath: sshKeyPath,
|
||||
rateLimiter: newRateLimiter(),
|
||||
knownHosts: knownHostsManager,
|
||||
rateLimiter: newRateLimiter(metrics),
|
||||
nodeGate: newNodeGate(),
|
||||
config: cfg,
|
||||
metrics: metrics,
|
||||
audit: auditLogger,
|
||||
}
|
||||
|
||||
if wd, err := os.Getwd(); err == nil {
|
||||
|
|
@ -293,6 +471,8 @@ func (p *Proxy) acceptConnections() {
|
|||
func (p *Proxy) handleConnection(conn net.Conn) {
|
||||
defer conn.Close()
|
||||
|
||||
remoteAddr := conn.RemoteAddr().String()
|
||||
|
||||
// Track concurrent requests
|
||||
p.metrics.queueDepth.Inc()
|
||||
defer p.metrics.queueDepth.Dec()
|
||||
|
|
@ -310,6 +490,9 @@ func (p *Proxy) handleConnection(conn net.Conn) {
|
|||
cred, err := extractPeerCredentials(conn)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Msg("Peer credentials unavailable")
|
||||
if p.audit != nil {
|
||||
p.audit.LogConnectionDenied("", nil, remoteAddr, "peer_credentials_unavailable")
|
||||
}
|
||||
p.sendErrorV2(conn, "unauthorized", "")
|
||||
return
|
||||
}
|
||||
|
|
@ -320,22 +503,45 @@ func (p *Proxy) handleConnection(conn net.Conn) {
|
|||
Uint32("uid", cred.uid).
|
||||
Uint32("gid", cred.gid).
|
||||
Msg("Peer authorization failed")
|
||||
if p.audit != nil {
|
||||
p.audit.LogConnectionDenied("", cred, remoteAddr, err.Error())
|
||||
}
|
||||
p.sendErrorV2(conn, "unauthorized", "")
|
||||
return
|
||||
}
|
||||
|
||||
if p.audit != nil {
|
||||
p.audit.LogConnectionAccepted("", cred, remoteAddr)
|
||||
}
|
||||
|
||||
// Check rate limit and concurrency
|
||||
releaseLimiter, ok := p.rateLimiter.allow(peerID{uid: cred.uid, pid: cred.pid})
|
||||
if !ok {
|
||||
p.metrics.rateLimitHits.Inc()
|
||||
peer := peerID{uid: cred.uid}
|
||||
releaseLimiter, limitReason, allowed := p.rateLimiter.allow(peer)
|
||||
if !allowed {
|
||||
log.Warn().
|
||||
Uint32("uid", cred.uid).
|
||||
Uint32("pid", cred.pid).
|
||||
Str("reason", limitReason).
|
||||
Msg("Rate limit exceeded")
|
||||
if p.audit != nil {
|
||||
p.audit.LogRateLimitHit("", cred, remoteAddr, limitReason)
|
||||
}
|
||||
p.sendErrorV2(conn, "rate limit exceeded", "")
|
||||
return
|
||||
}
|
||||
defer releaseLimiter()
|
||||
releaseFn := releaseLimiter
|
||||
defer func() {
|
||||
if releaseFn != nil {
|
||||
releaseFn()
|
||||
}
|
||||
}()
|
||||
applyPenalty := func(reason string) {
|
||||
if releaseFn != nil {
|
||||
releaseFn()
|
||||
releaseFn = nil
|
||||
}
|
||||
p.rateLimiter.penalize(peer, reason)
|
||||
}
|
||||
|
||||
// Read request using newline-delimited framing
|
||||
limited := &io.LimitedReader{R: conn, N: maxRequestBytes}
|
||||
|
|
@ -344,28 +550,48 @@ func (p *Proxy) handleConnection(conn net.Conn) {
|
|||
line, err := reader.ReadBytes('\n')
|
||||
if err != nil {
|
||||
if errors.Is(err, bufio.ErrBufferFull) || limited.N <= 0 {
|
||||
if p.audit != nil {
|
||||
p.audit.LogValidationFailure("", cred, remoteAddr, "", nil, "payload_too_large")
|
||||
}
|
||||
p.sendErrorV2(conn, "payload too large", "")
|
||||
applyPenalty("payload_too_large")
|
||||
return
|
||||
}
|
||||
if errors.Is(err, io.EOF) {
|
||||
if p.audit != nil {
|
||||
p.audit.LogValidationFailure("", cred, remoteAddr, "", nil, "empty_request")
|
||||
}
|
||||
p.sendErrorV2(conn, "empty request", "")
|
||||
applyPenalty("empty_request")
|
||||
return
|
||||
}
|
||||
if p.audit != nil {
|
||||
p.audit.LogValidationFailure("", cred, remoteAddr, "", nil, "read_error")
|
||||
}
|
||||
p.sendErrorV2(conn, "failed to read request", "")
|
||||
applyPenalty("read_error")
|
||||
return
|
||||
}
|
||||
|
||||
// Trim whitespace and validate
|
||||
line = bytes.TrimSpace(line)
|
||||
if len(line) == 0 {
|
||||
if p.audit != nil {
|
||||
p.audit.LogValidationFailure("", cred, remoteAddr, "", nil, "empty_request")
|
||||
}
|
||||
p.sendErrorV2(conn, "empty request", "")
|
||||
applyPenalty("empty_request")
|
||||
return
|
||||
}
|
||||
|
||||
// Parse JSON
|
||||
var req RPCRequest
|
||||
if err := json.Unmarshal(line, &req); err != nil {
|
||||
if p.audit != nil {
|
||||
p.audit.LogValidationFailure("", cred, remoteAddr, "", nil, "invalid_json")
|
||||
}
|
||||
p.sendErrorV2(conn, "invalid request format", "")
|
||||
applyPenalty("invalid_json")
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -389,9 +615,13 @@ func (p *Proxy) handleConnection(conn net.Conn) {
|
|||
// Find handler
|
||||
handler := p.router[req.Method]
|
||||
if handler == nil {
|
||||
if p.audit != nil {
|
||||
p.audit.LogValidationFailure(req.CorrelationID, cred, remoteAddr, req.Method, nil, "unknown_method")
|
||||
}
|
||||
resp.Error = "unknown method"
|
||||
logger.Warn().Msg("Unknown method")
|
||||
p.sendResponse(conn, resp)
|
||||
applyPenalty("unknown_method")
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -407,15 +637,27 @@ func (p *Proxy) handleConnection(conn net.Conn) {
|
|||
Uint32("pid", cred.pid).
|
||||
Str("corr_id", req.CorrelationID).
|
||||
Msg("SECURITY: Container attempted to call privileged method - access denied")
|
||||
if p.audit != nil {
|
||||
p.audit.LogValidationFailure(req.CorrelationID, cred, remoteAddr, req.Method, nil, "privileged_method_denied")
|
||||
}
|
||||
p.sendResponse(conn, resp)
|
||||
p.metrics.rpcRequests.WithLabelValues(req.Method, "unauthorized").Inc()
|
||||
applyPenalty("privileged_method_denied")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if p.audit != nil {
|
||||
p.audit.LogCommandStart(req.CorrelationID, cred, remoteAddr, "", req.Method, nil)
|
||||
}
|
||||
|
||||
// Execute handler
|
||||
result, err := handler(ctx, &req, logger)
|
||||
duration := time.Since(startTime)
|
||||
if err != nil {
|
||||
if p.audit != nil {
|
||||
p.audit.LogCommandResult(req.CorrelationID, cred, remoteAddr, "", req.Method, nil, 1, duration, "", "", err)
|
||||
}
|
||||
resp.Error = err.Error()
|
||||
logger.Warn().Err(err).Msg("Handler failed")
|
||||
// Clear read deadline and set write deadline for error response
|
||||
|
|
@ -431,6 +673,9 @@ func (p *Proxy) handleConnection(conn net.Conn) {
|
|||
// Success
|
||||
resp.Success = true
|
||||
resp.Data = result
|
||||
if p.audit != nil {
|
||||
p.audit.LogCommandResult(req.CorrelationID, cred, remoteAddr, "", req.Method, nil, 0, duration, "", "", nil)
|
||||
}
|
||||
logger.Info().Msg("Request completed")
|
||||
|
||||
// Clear read deadline and set write deadline for response
|
||||
|
|
|
|||
|
|
@ -16,15 +16,19 @@ const defaultMetricsAddr = "127.0.0.1:9127"
|
|||
|
||||
// ProxyMetrics holds Prometheus metrics for the proxy
|
||||
type ProxyMetrics struct {
|
||||
rpcRequests *prometheus.CounterVec
|
||||
rpcLatency *prometheus.HistogramVec
|
||||
sshRequests *prometheus.CounterVec
|
||||
sshLatency *prometheus.HistogramVec
|
||||
queueDepth prometheus.Gauge
|
||||
rateLimitHits prometheus.Counter
|
||||
buildInfo *prometheus.GaugeVec
|
||||
server *http.Server
|
||||
registry *prometheus.Registry
|
||||
rpcRequests *prometheus.CounterVec
|
||||
rpcLatency *prometheus.HistogramVec
|
||||
sshRequests *prometheus.CounterVec
|
||||
sshLatency *prometheus.HistogramVec
|
||||
queueDepth prometheus.Gauge
|
||||
rateLimitHits prometheus.Counter
|
||||
limiterRejects *prometheus.CounterVec
|
||||
globalConcurrency prometheus.Gauge
|
||||
limiterPenalties *prometheus.CounterVec
|
||||
limiterPeers prometheus.Gauge
|
||||
buildInfo *prometheus.GaugeVec
|
||||
server *http.Server
|
||||
registry *prometheus.Registry
|
||||
}
|
||||
|
||||
// NewProxyMetrics creates and registers all metrics
|
||||
|
|
@ -74,6 +78,32 @@ func NewProxyMetrics(version string) *ProxyMetrics {
|
|||
Help: "Number of RPC requests rejected due to rate limiting.",
|
||||
},
|
||||
),
|
||||
limiterRejects: prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Name: "pulse_proxy_limiter_rejections_total",
|
||||
Help: "Limiter rejections by reason.",
|
||||
},
|
||||
[]string{"reason"},
|
||||
),
|
||||
globalConcurrency: prometheus.NewGauge(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "pulse_proxy_global_concurrency_inflight",
|
||||
Help: "Current global concurrency slots in use.",
|
||||
},
|
||||
),
|
||||
limiterPenalties: prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Name: "pulse_proxy_limiter_penalties_total",
|
||||
Help: "Penalty sleeps applied after validation failures.",
|
||||
},
|
||||
[]string{"reason"},
|
||||
),
|
||||
limiterPeers: prometheus.NewGauge(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "pulse_proxy_limiter_active_peers",
|
||||
Help: "Number of peers tracked by the rate limiter.",
|
||||
},
|
||||
),
|
||||
buildInfo: prometheus.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "pulse_proxy_build_info",
|
||||
|
|
@ -91,6 +121,10 @@ func NewProxyMetrics(version string) *ProxyMetrics {
|
|||
pm.sshLatency,
|
||||
pm.queueDepth,
|
||||
pm.rateLimitHits,
|
||||
pm.limiterRejects,
|
||||
pm.globalConcurrency,
|
||||
pm.limiterPenalties,
|
||||
pm.limiterPeers,
|
||||
pm.buildInfo,
|
||||
)
|
||||
|
||||
|
|
@ -165,3 +199,39 @@ func sanitizeNodeLabel(node string) string {
|
|||
|
||||
return out
|
||||
}
|
||||
|
||||
func (m *ProxyMetrics) recordLimiterReject(reason string) {
|
||||
if m == nil {
|
||||
return
|
||||
}
|
||||
m.rateLimitHits.Inc()
|
||||
m.limiterRejects.WithLabelValues(reason).Inc()
|
||||
}
|
||||
|
||||
func (m *ProxyMetrics) incGlobalConcurrency() {
|
||||
if m == nil {
|
||||
return
|
||||
}
|
||||
m.globalConcurrency.Inc()
|
||||
}
|
||||
|
||||
func (m *ProxyMetrics) decGlobalConcurrency() {
|
||||
if m == nil {
|
||||
return
|
||||
}
|
||||
m.globalConcurrency.Dec()
|
||||
}
|
||||
|
||||
func (m *ProxyMetrics) recordPenalty(reason string) {
|
||||
if m == nil {
|
||||
return
|
||||
}
|
||||
m.limiterPenalties.WithLabelValues(reason).Inc()
|
||||
}
|
||||
|
||||
func (m *ProxyMetrics) setLimiterPeers(count int) {
|
||||
if m == nil {
|
||||
return
|
||||
}
|
||||
m.limiterPeers.Set(float64(count))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,16 +2,46 @@ package main
|
|||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
const (
|
||||
tempWrapperPath = "/usr/local/libexec/pulse-sensor-proxy/temp-wrapper.sh"
|
||||
tempWrapperScript = `#!/bin/sh
|
||||
set -eu
|
||||
|
||||
if command -v sensors >/dev/null 2>&1; then
|
||||
OUTPUT="$(sensors -j 2>/dev/null || true)"
|
||||
if [ -n "$OUTPUT" ]; then
|
||||
printf '%s\n' "$OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -r /sys/class/thermal/thermal_zone0/temp ]; then
|
||||
RAW="$(cat /sys/class/thermal/thermal_zone0/temp 2>/dev/null || true)"
|
||||
if [ -n "$RAW" ]; then
|
||||
TEMP="$(awk -v raw="$RAW" 'BEGIN { if (raw == "") exit 1; printf "%.2f", raw / 1000.0 }' 2>/dev/null || true)"
|
||||
if [ -n "$TEMP" ]; then
|
||||
printf '{"rpitemp-virtual":{"temp1":{"temp1_input":%s}}}\n' "$TEMP"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
exit 1
|
||||
`
|
||||
)
|
||||
|
||||
// execCommand executes a shell command and returns output
|
||||
func execCommand(cmd string) (string, error) {
|
||||
out, err := exec.Command("sh", "-c", cmd).CombinedOutput()
|
||||
|
|
@ -47,12 +77,70 @@ func (p *Proxy) buildAuthorizedKey(pubKey string) (string, error) {
|
|||
const comment = "pulse-sensor-proxy"
|
||||
|
||||
// Forced command with all restrictions
|
||||
const forced = `command="sensors -j",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty`
|
||||
forced := fmt.Sprintf(`command="%s",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty`, tempWrapperPath)
|
||||
|
||||
// Format: from="...",command="...",no-* ssh-rsa AAAA... pulse-sensor-proxy
|
||||
return fmt.Sprintf(`%s,%s %s %s`, fromClause, forced, pubKey, comment), nil
|
||||
}
|
||||
|
||||
func (p *Proxy) ensureHostKey(node string) error {
|
||||
if p.knownHosts == nil {
|
||||
return fmt.Errorf("host key manager not configured")
|
||||
}
|
||||
return p.knownHosts.Ensure(context.Background(), node)
|
||||
}
|
||||
|
||||
func (p *Proxy) sshCommonOptions() string {
|
||||
if p.knownHosts == nil {
|
||||
return "-o StrictHostKeyChecking=yes -o BatchMode=yes"
|
||||
}
|
||||
return fmt.Sprintf("-o StrictHostKeyChecking=yes -o BatchMode=yes -o UserKnownHostsFile=%s -o GlobalKnownHostsFile=/dev/null",
|
||||
shellQuote(p.knownHosts.Path()))
|
||||
}
|
||||
|
||||
func shellQuote(arg string) string {
|
||||
if arg == "" {
|
||||
return "''"
|
||||
}
|
||||
if !strings.Contains(arg, "'") {
|
||||
return "'" + arg + "'"
|
||||
}
|
||||
return strconv.Quote(arg)
|
||||
}
|
||||
|
||||
func (p *Proxy) ensureTempWrapper(nodeHost, commonOpts string) error {
|
||||
dir := filepath.Dir(tempWrapperPath)
|
||||
mkdirCmd := fmt.Sprintf(
|
||||
`ssh %s -o ConnectTimeout=10 root@%s "mkdir -p %s && chmod 755 %s"`,
|
||||
commonOpts,
|
||||
nodeHost,
|
||||
dir,
|
||||
dir,
|
||||
)
|
||||
|
||||
if _, err := execCommand(mkdirCmd); err != nil {
|
||||
return fmt.Errorf("failed to prepare temperature wrapper directory on %s: %w", nodeHost, err)
|
||||
}
|
||||
|
||||
uploadCmd := fmt.Sprintf(
|
||||
`ssh %s -o ConnectTimeout=10 root@%s "cat > %s <<'EOF'
|
||||
%s
|
||||
EOF
|
||||
chmod 755 %s"`,
|
||||
commonOpts,
|
||||
nodeHost,
|
||||
tempWrapperPath,
|
||||
tempWrapperScript,
|
||||
tempWrapperPath,
|
||||
)
|
||||
|
||||
if _, err := execCommand(uploadCmd); err != nil {
|
||||
return fmt.Errorf("failed to install temperature wrapper on %s: %w", nodeHost, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// pushSSHKeyFrom pushes a public key from a specific directory to a node
|
||||
func (p *Proxy) pushSSHKeyFrom(nodeHost, keyDir string) error {
|
||||
startTime := time.Now()
|
||||
|
|
@ -73,9 +161,23 @@ func (p *Proxy) pushSSHKeyFrom(nodeHost, keyDir string) error {
|
|||
return fmt.Errorf("failed to build authorized key: %w", err)
|
||||
}
|
||||
|
||||
if err := p.ensureHostKey(nodeHost); err != nil {
|
||||
p.metrics.sshRequests.WithLabelValues(nodeLabel, "error").Inc()
|
||||
p.metrics.sshLatency.WithLabelValues(nodeLabel).Observe(time.Since(startTime).Seconds())
|
||||
return fmt.Errorf("failed to ensure host key for %s: %w", nodeHost, err)
|
||||
}
|
||||
|
||||
commonOpts := p.sshCommonOptions()
|
||||
if err := p.ensureTempWrapper(nodeHost, commonOpts); err != nil {
|
||||
p.metrics.sshRequests.WithLabelValues(nodeLabel, "error").Inc()
|
||||
p.metrics.sshLatency.WithLabelValues(nodeLabel).Observe(time.Since(startTime).Seconds())
|
||||
return fmt.Errorf("failed to stage temperature wrapper on %s: %w", nodeHost, err)
|
||||
}
|
||||
|
||||
// Check if the exact restricted entry already exists
|
||||
checkCmd := fmt.Sprintf(
|
||||
`ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 root@%s "grep -F '%s' /root/.ssh/authorized_keys 2>/dev/null"`,
|
||||
`ssh %s -o ConnectTimeout=10 root@%s "grep -F '%s' /root/.ssh/authorized_keys 2>/dev/null"`,
|
||||
commonOpts,
|
||||
nodeHost,
|
||||
entry,
|
||||
)
|
||||
|
|
@ -89,7 +191,8 @@ func (p *Proxy) pushSSHKeyFrom(nodeHost, keyDir string) error {
|
|||
|
||||
// Remove old pulse-temp-proxy and pulse-sensor-proxy entries (for upgrade path)
|
||||
removeOldCmd := fmt.Sprintf(
|
||||
`ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 root@%s "mkdir -p /root/.ssh && chmod 700 /root/.ssh && grep -v -e 'pulse-temp-proxy$' -e 'pulse-sensor-proxy$' /root/.ssh/authorized_keys > /root/.ssh/authorized_keys.tmp 2>/dev/null || touch /root/.ssh/authorized_keys.tmp"`,
|
||||
`ssh %s -o ConnectTimeout=10 root@%s "mkdir -p /root/.ssh && chmod 700 /root/.ssh && grep -v -e 'pulse-temp-proxy$' -e 'pulse-sensor-proxy$' /root/.ssh/authorized_keys > /root/.ssh/authorized_keys.tmp 2>/dev/null || touch /root/.ssh/authorized_keys.tmp"`,
|
||||
commonOpts,
|
||||
nodeHost,
|
||||
)
|
||||
|
||||
|
|
@ -101,7 +204,8 @@ func (p *Proxy) pushSSHKeyFrom(nodeHost, keyDir string) error {
|
|||
|
||||
// Add the new restricted key and atomically replace the file
|
||||
addCmd := fmt.Sprintf(
|
||||
`ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 root@%s "echo '%s' >> /root/.ssh/authorized_keys.tmp && mv /root/.ssh/authorized_keys.tmp /root/.ssh/authorized_keys && chmod 600 /root/.ssh/authorized_keys"`,
|
||||
`ssh %s -o ConnectTimeout=10 root@%s "echo '%s' >> /root/.ssh/authorized_keys.tmp && mv /root/.ssh/authorized_keys.tmp /root/.ssh/authorized_keys && chmod 600 /root/.ssh/authorized_keys"`,
|
||||
commonOpts,
|
||||
nodeHost,
|
||||
entry,
|
||||
)
|
||||
|
|
@ -135,9 +239,17 @@ func (p *Proxy) testSSHConnection(nodeHost string) error {
|
|||
nodeLabel := sanitizeNodeLabel(nodeHost)
|
||||
|
||||
privKeyPath := filepath.Join(p.sshKeyPath, "id_ed25519")
|
||||
if err := p.ensureHostKey(nodeHost); err != nil {
|
||||
p.metrics.sshRequests.WithLabelValues(nodeLabel, "error").Inc()
|
||||
p.metrics.sshLatency.WithLabelValues(nodeLabel).Observe(time.Since(startTime).Seconds())
|
||||
return fmt.Errorf("failed to ensure host key for %s: %w", nodeHost, err)
|
||||
}
|
||||
|
||||
commonOpts := p.sshCommonOptions()
|
||||
cmd := fmt.Sprintf(
|
||||
`ssh -i %[1]s -T -n -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=5 root@%[2]s "echo test"`,
|
||||
privKeyPath,
|
||||
`ssh %s -i %s -T -n -o LogLevel=ERROR -o ConnectTimeout=5 root@%s ""`,
|
||||
commonOpts,
|
||||
shellQuote(privKeyPath),
|
||||
nodeHost,
|
||||
)
|
||||
|
||||
|
|
@ -162,12 +274,20 @@ func (p *Proxy) getTemperatureViaSSH(nodeHost string) (string, error) {
|
|||
nodeLabel := sanitizeNodeLabel(nodeHost)
|
||||
|
||||
privKeyPath := filepath.Join(p.sshKeyPath, "id_ed25519")
|
||||
if err := p.ensureHostKey(nodeHost); err != nil {
|
||||
p.metrics.sshRequests.WithLabelValues(nodeLabel, "error").Inc()
|
||||
p.metrics.sshLatency.WithLabelValues(nodeLabel).Observe(time.Since(startTime).Seconds())
|
||||
return "", fmt.Errorf("failed to ensure host key for %s: %w", nodeHost, err)
|
||||
}
|
||||
|
||||
// Since we use ForceCommand="sensors -j", any SSH command will run sensors
|
||||
commonOpts := p.sshCommonOptions()
|
||||
|
||||
// Since we use a forced wrapper command, any SSH connection runs the wrapper
|
||||
// We don't need to specify the command
|
||||
cmd := fmt.Sprintf(
|
||||
`ssh -i %[1]s -T -n -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=5 root@%[2]s ""`,
|
||||
privKeyPath,
|
||||
`ssh %s -i %s -T -n -o LogLevel=ERROR -o ConnectTimeout=5 root@%s ""`,
|
||||
commonOpts,
|
||||
shellQuote(privKeyPath),
|
||||
nodeHost,
|
||||
)
|
||||
|
||||
|
|
|
|||
138
cmd/pulse-sensor-proxy/ssh_test.go
Normal file
138
cmd/pulse-sensor-proxy/ssh_test.go
Normal file
|
|
@ -0,0 +1,138 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func setupTempWrapper(t *testing.T) (scriptPath, thermalFile, binDir, baseDir string) {
|
||||
t.Helper()
|
||||
|
||||
baseDir = t.TempDir()
|
||||
|
||||
thermalDir := filepath.Join(baseDir, "sys", "class", "thermal", "thermal_zone0")
|
||||
if err := os.MkdirAll(thermalDir, 0o755); err != nil {
|
||||
t.Fatalf("failed to create thermal zone directory: %v", err)
|
||||
}
|
||||
thermalFile = filepath.Join(thermalDir, "temp")
|
||||
|
||||
scriptContent := strings.ReplaceAll(tempWrapperScript, "/sys/class/thermal/thermal_zone0/temp", thermalFile)
|
||||
scriptPath = filepath.Join(baseDir, "temp-wrapper.sh")
|
||||
if err := os.WriteFile(scriptPath, []byte(scriptContent), 0o755); err != nil {
|
||||
t.Fatalf("failed to write wrapper script: %v", err)
|
||||
}
|
||||
|
||||
binDir = filepath.Join(baseDir, "bin")
|
||||
if err := os.MkdirAll(binDir, 0o755); err != nil {
|
||||
t.Fatalf("failed to create bin directory: %v", err)
|
||||
}
|
||||
|
||||
linkCommand := func(name string) {
|
||||
target, err := exec.LookPath(name)
|
||||
if err != nil {
|
||||
t.Fatalf("required command %q not found on host: %v", name, err)
|
||||
}
|
||||
content := fmt.Sprintf("#!/bin/sh\nexec %s \"$@\"\n", target)
|
||||
if err := os.WriteFile(filepath.Join(binDir, name), []byte(content), 0o755); err != nil {
|
||||
t.Fatalf("failed to create shim for %s: %v", name, err)
|
||||
}
|
||||
}
|
||||
|
||||
linkCommand("awk")
|
||||
linkCommand("cat")
|
||||
|
||||
return scriptPath, thermalFile, binDir, baseDir
|
||||
}
|
||||
|
||||
func runTempWrapper(t *testing.T, scriptPath, binDir string, extraEnv ...string) []byte {
|
||||
t.Helper()
|
||||
cmd := exec.Command("sh", scriptPath)
|
||||
env := []string{"PATH=" + binDir}
|
||||
env = append(env, extraEnv...)
|
||||
cmd.Env = env
|
||||
output, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("temp wrapper failed: %v (output: %s)", err, strings.TrimSpace(string(output)))
|
||||
}
|
||||
return output
|
||||
}
|
||||
|
||||
func TestTempWrapperFallbackWhenSensorsMissing(t *testing.T) {
|
||||
scriptPath, thermalFile, binDir, _ := setupTempWrapper(t)
|
||||
|
||||
if err := os.WriteFile(thermalFile, []byte("51234\n"), 0o644); err != nil {
|
||||
t.Fatalf("failed to write thermal zone temperature: %v", err)
|
||||
}
|
||||
|
||||
output := runTempWrapper(t, scriptPath, binDir)
|
||||
|
||||
var data map[string]map[string]map[string]float64
|
||||
if err := json.Unmarshal(output, &data); err != nil {
|
||||
t.Fatalf("failed to parse wrapper output as JSON: %v (output: %s)", err, strings.TrimSpace(string(output)))
|
||||
}
|
||||
|
||||
temp1, ok := data["rpitemp-virtual"]["temp1"]["temp1_input"]
|
||||
if !ok {
|
||||
t.Fatalf("expected rpitemp-virtual temp1 reading in output: %v", data)
|
||||
}
|
||||
if temp1 != 51.23 {
|
||||
t.Fatalf("expected converted temperature 51.23, got %.2f", temp1)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTempWrapperFallbackWhenSensorsEmpty(t *testing.T) {
|
||||
scriptPath, thermalFile, binDir, _ := setupTempWrapper(t)
|
||||
|
||||
sensorsStub := filepath.Join(binDir, "sensors")
|
||||
content := "#!/bin/sh\nexit 0\n"
|
||||
if err := os.WriteFile(sensorsStub, []byte(content), 0o755); err != nil {
|
||||
t.Fatalf("failed to create sensors stub: %v", err)
|
||||
}
|
||||
|
||||
if err := os.WriteFile(thermalFile, []byte("47890\n"), 0o644); err != nil {
|
||||
t.Fatalf("failed to write thermal zone temperature: %v", err)
|
||||
}
|
||||
|
||||
output := runTempWrapper(t, scriptPath, binDir)
|
||||
|
||||
var data map[string]map[string]map[string]float64
|
||||
if err := json.Unmarshal(output, &data); err != nil {
|
||||
t.Fatalf("failed to parse wrapper output as JSON: %v (output: %s)", err, strings.TrimSpace(string(output)))
|
||||
}
|
||||
|
||||
temp1, ok := data["rpitemp-virtual"]["temp1"]["temp1_input"]
|
||||
if !ok {
|
||||
t.Fatalf("expected rpitemp-virtual temp1 reading in output: %v", data)
|
||||
}
|
||||
if temp1 != 47.89 {
|
||||
t.Fatalf("expected converted temperature 47.89, got %.2f", temp1)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTempWrapperPrefersSensorsOutput(t *testing.T) {
|
||||
scriptPath, thermalFile, binDir, _ := setupTempWrapper(t)
|
||||
|
||||
jsonOutput := `{"cpu_thermal-virtual-0":{"temp1":{"temp1_input":42.5}}}`
|
||||
sensorsStub := filepath.Join(binDir, "sensors")
|
||||
content := fmt.Sprintf("#!/bin/sh\nprintf '%s'\n", jsonOutput)
|
||||
if err := os.WriteFile(sensorsStub, []byte(content), 0o755); err != nil {
|
||||
t.Fatalf("failed to create sensors stub: %v", err)
|
||||
}
|
||||
|
||||
// Ensure thermal zone file exists but should be ignored
|
||||
if err := os.WriteFile(thermalFile, []byte("40000\n"), 0o644); err != nil {
|
||||
t.Fatalf("failed to write thermal zone temperature: %v", err)
|
||||
}
|
||||
|
||||
output := runTempWrapper(t, scriptPath, binDir)
|
||||
|
||||
trimmed := strings.TrimSpace(string(output))
|
||||
if trimmed != jsonOutput {
|
||||
t.Fatalf("expected wrapper to return sensors output %s, got %s", jsonOutput, trimmed)
|
||||
}
|
||||
}
|
||||
|
|
@ -7,62 +7,122 @@ import (
|
|||
"golang.org/x/time/rate"
|
||||
)
|
||||
|
||||
// peerID identifies a connecting process by UID+PID
|
||||
// peerID identifies a connecting principal (grouped by UID)
|
||||
type peerID struct {
|
||||
uid uint32
|
||||
pid uint32
|
||||
}
|
||||
|
||||
// limiterEntry holds rate limiting and concurrency controls for a peer
|
||||
type limiterEntry struct {
|
||||
limiter *rate.Limiter // throughput: 20/min with burst 10
|
||||
semaphore chan struct{} // concurrency: cap 10
|
||||
limiter *rate.Limiter
|
||||
semaphore chan struct{}
|
||||
lastSeen time.Time
|
||||
}
|
||||
|
||||
type limiterPolicy struct {
|
||||
perPeerLimit rate.Limit
|
||||
perPeerBurst int
|
||||
perPeerConcurrency int
|
||||
globalConcurrency int
|
||||
penaltyDuration time.Duration
|
||||
}
|
||||
|
||||
// rateLimiter manages per-peer rate limits and concurrency
|
||||
type rateLimiter struct {
|
||||
mu sync.Mutex
|
||||
entries map[peerID]*limiterEntry
|
||||
quitChan chan struct{}
|
||||
mu sync.Mutex
|
||||
entries map[peerID]*limiterEntry
|
||||
quitChan chan struct{}
|
||||
globalSem chan struct{}
|
||||
policy limiterPolicy
|
||||
metrics *ProxyMetrics
|
||||
}
|
||||
|
||||
const (
|
||||
defaultPerPeerBurst = 2
|
||||
defaultPerPeerConcurrency = 2
|
||||
defaultGlobalConcurrency = 8
|
||||
)
|
||||
|
||||
var (
|
||||
defaultPerPeerRateInterval = 5 * time.Second // 0.2 qps (~12/min)
|
||||
defaultPenaltyDuration = 2 * time.Second
|
||||
defaultPerPeerLimit = rate.Every(defaultPerPeerRateInterval)
|
||||
)
|
||||
|
||||
// newRateLimiter creates a new rate limiter with cleanup loop
|
||||
func newRateLimiter() *rateLimiter {
|
||||
func newRateLimiter(metrics *ProxyMetrics) *rateLimiter {
|
||||
rl := &rateLimiter{
|
||||
entries: make(map[peerID]*limiterEntry),
|
||||
quitChan: make(chan struct{}),
|
||||
entries: make(map[peerID]*limiterEntry),
|
||||
quitChan: make(chan struct{}),
|
||||
globalSem: make(chan struct{}, defaultGlobalConcurrency),
|
||||
policy: limiterPolicy{
|
||||
perPeerLimit: defaultPerPeerLimit,
|
||||
perPeerBurst: defaultPerPeerBurst,
|
||||
perPeerConcurrency: defaultPerPeerConcurrency,
|
||||
globalConcurrency: defaultGlobalConcurrency,
|
||||
penaltyDuration: defaultPenaltyDuration,
|
||||
},
|
||||
metrics: metrics,
|
||||
}
|
||||
if rl.metrics != nil {
|
||||
rl.metrics.setLimiterPeers(0)
|
||||
}
|
||||
go rl.cleanupLoop()
|
||||
return rl
|
||||
}
|
||||
|
||||
// allow checks if a peer is allowed to make a request and reserves a concurrency slot
|
||||
// Returns a release function and whether the request is allowed
|
||||
func (rl *rateLimiter) allow(id peerID) (release func(), allowed bool) {
|
||||
// allow checks if a peer is allowed to make a request and reserves concurrency.
|
||||
// Returns a release function, rejection reason (if any), and whether the request is allowed.
|
||||
func (rl *rateLimiter) allow(id peerID) (release func(), reason string, allowed bool) {
|
||||
rl.mu.Lock()
|
||||
entry := rl.entries[id]
|
||||
if entry == nil {
|
||||
entry = &limiterEntry{
|
||||
limiter: rate.NewLimiter(rate.Every(time.Minute/20), 10), // 20/min, burst 10
|
||||
semaphore: make(chan struct{}, 10), // max 10 concurrent
|
||||
limiter: rate.NewLimiter(rl.policy.perPeerLimit, rl.policy.perPeerBurst),
|
||||
semaphore: make(chan struct{}, rl.policy.perPeerConcurrency),
|
||||
}
|
||||
rl.entries[id] = entry
|
||||
if rl.metrics != nil {
|
||||
rl.metrics.setLimiterPeers(len(rl.entries))
|
||||
}
|
||||
}
|
||||
entry.lastSeen = time.Now()
|
||||
rl.mu.Unlock()
|
||||
|
||||
// Check rate limit
|
||||
if !entry.limiter.Allow() {
|
||||
return nil, false
|
||||
rl.recordRejection("rate")
|
||||
return nil, "rate", false
|
||||
}
|
||||
|
||||
// Try to acquire concurrency slot
|
||||
// Acquire global concurrency
|
||||
select {
|
||||
case rl.globalSem <- struct{}{}:
|
||||
if rl.metrics != nil {
|
||||
rl.metrics.incGlobalConcurrency()
|
||||
}
|
||||
default:
|
||||
rl.recordRejection("global_concurrency")
|
||||
return nil, "global_concurrency", false
|
||||
}
|
||||
|
||||
// Try to acquire per-peer concurrency slot
|
||||
select {
|
||||
case entry.semaphore <- struct{}{}:
|
||||
return func() { <-entry.semaphore }, true
|
||||
return func() {
|
||||
<-entry.semaphore
|
||||
<-rl.globalSem
|
||||
if rl.metrics != nil {
|
||||
rl.metrics.decGlobalConcurrency()
|
||||
}
|
||||
}, "", true
|
||||
default:
|
||||
return nil, false // max concurrent in-flight reached
|
||||
<-rl.globalSem
|
||||
if rl.metrics != nil {
|
||||
rl.metrics.decGlobalConcurrency()
|
||||
}
|
||||
rl.recordRejection("peer_concurrency")
|
||||
return nil, "peer_concurrency", false
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -79,6 +139,9 @@ func (rl *rateLimiter) cleanupLoop() {
|
|||
delete(rl.entries, id)
|
||||
}
|
||||
}
|
||||
if rl.metrics != nil {
|
||||
rl.metrics.setLimiterPeers(len(rl.entries))
|
||||
}
|
||||
rl.mu.Unlock()
|
||||
case <-rl.quitChan:
|
||||
return
|
||||
|
|
@ -91,6 +154,22 @@ func (rl *rateLimiter) shutdown() {
|
|||
close(rl.quitChan)
|
||||
}
|
||||
|
||||
func (rl *rateLimiter) penalize(id peerID, reason string) {
|
||||
if rl.policy.penaltyDuration <= 0 {
|
||||
return
|
||||
}
|
||||
time.Sleep(rl.policy.penaltyDuration)
|
||||
if rl.metrics != nil {
|
||||
rl.metrics.recordPenalty(reason)
|
||||
}
|
||||
}
|
||||
|
||||
func (rl *rateLimiter) recordRejection(reason string) {
|
||||
if rl.metrics != nil {
|
||||
rl.metrics.recordLimiterReject(reason)
|
||||
}
|
||||
}
|
||||
|
||||
// nodeGate controls per-node concurrency for temperature requests
|
||||
type nodeGate struct {
|
||||
mu sync.Mutex
|
||||
|
|
|
|||
43
cmd/pulse-sensor-proxy/throttle_test.go
Normal file
43
cmd/pulse-sensor-proxy/throttle_test.go
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestRateLimiterPenalizeMetrics(t *testing.T) {
|
||||
metrics := NewProxyMetrics("test")
|
||||
rl := newRateLimiter(metrics)
|
||||
rl.policy.penaltyDuration = 10 * time.Millisecond
|
||||
|
||||
start := time.Now()
|
||||
rl.penalize(peerID{uid: 42}, "invalid_json")
|
||||
if time.Since(start) < rl.policy.penaltyDuration {
|
||||
t.Fatalf("expected penalize to sleep at least %v", rl.policy.penaltyDuration)
|
||||
}
|
||||
|
||||
mf, err := metrics.registry.Gather()
|
||||
if err != nil {
|
||||
t.Fatalf("gather metrics: %v", err)
|
||||
}
|
||||
|
||||
found := false
|
||||
for _, fam := range mf {
|
||||
if fam.GetName() != "pulse_proxy_limiter_penalties_total" {
|
||||
continue
|
||||
}
|
||||
for _, metric := range fam.GetMetric() {
|
||||
if metric.GetCounter().GetValue() == 0 {
|
||||
continue
|
||||
}
|
||||
for _, label := range metric.GetLabel() {
|
||||
if label.GetName() == "reason" && label.GetValue() == "invalid_json" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("expected limiter penalty metric for invalid_json")
|
||||
}
|
||||
}
|
||||
|
|
@ -1,8 +1,13 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"regexp"
|
||||
"strings"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
|
@ -19,6 +24,13 @@ var (
|
|||
ipv6Regex = regexp.MustCompile(`^[0-9a-fA-F:]+$`)
|
||||
)
|
||||
|
||||
var (
|
||||
allowedCommands = map[string]struct{}{
|
||||
"sensors": {},
|
||||
"ipmitool": {},
|
||||
}
|
||||
)
|
||||
|
||||
// sanitizeCorrelationID validates and sanitizes a correlation ID
|
||||
// Returns a valid UUID, generating a new one if input is missing or invalid
|
||||
func sanitizeCorrelationID(id string) string {
|
||||
|
|
@ -33,8 +45,162 @@ func sanitizeCorrelationID(id string) string {
|
|||
|
||||
// validateNodeName checks if a node name is in valid format
|
||||
func validateNodeName(name string) error {
|
||||
if !nodeNameRegex.MatchString(name) {
|
||||
if name == "" {
|
||||
return fmt.Errorf("invalid node name")
|
||||
}
|
||||
|
||||
if ipv4Regex.MatchString(name) {
|
||||
return nil
|
||||
}
|
||||
|
||||
candidate := name
|
||||
if strings.HasPrefix(candidate, "[") && strings.HasSuffix(candidate, "]") {
|
||||
candidate = candidate[1 : len(candidate)-1]
|
||||
}
|
||||
|
||||
if ip := net.ParseIP(candidate); ip != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if nodeNameRegex.MatchString(name) {
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("invalid node name")
|
||||
}
|
||||
|
||||
func validateCommand(name string, args []string) error {
|
||||
if err := validateCommandName(name); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, arg := range args {
|
||||
if err := validateCommandArg(arg); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if name == "ipmitool" {
|
||||
if err := validateIPMIToolArgs(args); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateCommandName(name string) error {
|
||||
if name == "" {
|
||||
return errors.New("command required")
|
||||
}
|
||||
|
||||
if strings.Contains(name, "/") {
|
||||
return errors.New("absolute command paths not allowed")
|
||||
}
|
||||
|
||||
if _, ok := allowedCommands[name]; !ok {
|
||||
return fmt.Errorf("command %q not permitted", name)
|
||||
}
|
||||
|
||||
if !isASCII(name) {
|
||||
return errors.New("command must be ASCII")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateCommandArg(arg string) error {
|
||||
if len(arg) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
if len(arg) > 1024 {
|
||||
return errors.New("argument too long")
|
||||
}
|
||||
|
||||
if !utf8.ValidString(arg) {
|
||||
return errors.New("argument contains invalid UTF-8")
|
||||
}
|
||||
|
||||
if hasNullByte(arg) {
|
||||
return errors.New("argument contains null byte")
|
||||
}
|
||||
|
||||
if !isASCII(arg) {
|
||||
return errors.New("argument must be ASCII")
|
||||
}
|
||||
|
||||
if hasShellMeta(arg) {
|
||||
return errors.New("argument contains forbidden shell characters")
|
||||
}
|
||||
|
||||
if strings.Contains(arg, "=") && !strings.HasPrefix(arg, "-") {
|
||||
return errors.New("environment-style arguments not permitted")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateIPMIToolArgs(args []string) error {
|
||||
lowered := make([]string, len(args))
|
||||
for i, arg := range args {
|
||||
lowered[i] = strings.ToLower(arg)
|
||||
}
|
||||
|
||||
for i := 0; i < len(lowered); i++ {
|
||||
token := lowered[i]
|
||||
switch token {
|
||||
case "shell", "raw", "exec", "lanplus", "lanplusciphers":
|
||||
return errors.New("dangerous ipmitool arguments not permitted")
|
||||
case "chassis":
|
||||
if i+1 < len(lowered) {
|
||||
switch lowered[i+1] {
|
||||
case "power", "bootparam", "status", "policy":
|
||||
return errors.New("chassis operations not permitted")
|
||||
}
|
||||
}
|
||||
case "power", "reset", "off", "cycle", "bmc", "mc":
|
||||
return errors.New("power control commands not permitted")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func hasShellMeta(s string) bool {
|
||||
forbidden := []string{";", "|", "&", "$", "`", "\\", ">", "<", "(", ")", "[", "]", "{", "}", "!", "~"}
|
||||
for _, ch := range forbidden {
|
||||
if strings.Contains(s, ch) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
if strings.Contains(s, "..") {
|
||||
return true
|
||||
}
|
||||
|
||||
if strings.ContainsAny(s, "\n\r\t") {
|
||||
return true
|
||||
}
|
||||
|
||||
if strings.HasPrefix(s, "-") && strings.Contains(s, "=") {
|
||||
if strings.Contains(s, "/") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func hasNullByte(s string) bool {
|
||||
return strings.IndexByte(s, 0) >= 0
|
||||
}
|
||||
|
||||
func isASCII(s string) bool {
|
||||
for _, r := range s {
|
||||
if r > unicode.MaxASCII {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
|
|
|||
31
cmd/pulse-sensor-proxy/validation_fuzz_test.go
Normal file
31
cmd/pulse-sensor-proxy/validation_fuzz_test.go
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func FuzzValidateCommand(f *testing.F) {
|
||||
seeds := []string{
|
||||
"sensors -j",
|
||||
"ipmitool sdr",
|
||||
"sensors",
|
||||
"ipmitool lan print",
|
||||
}
|
||||
for _, seed := range seeds {
|
||||
f.Add(seed)
|
||||
}
|
||||
|
||||
f.Fuzz(func(t *testing.T, input string) {
|
||||
fields := strings.Fields(input)
|
||||
if len(fields) == 0 {
|
||||
return
|
||||
}
|
||||
cmd := fields[0]
|
||||
args := []string{}
|
||||
if len(fields) > 1 {
|
||||
args = fields[1:]
|
||||
}
|
||||
validateCommand(cmd, args) // ensure no panics
|
||||
})
|
||||
}
|
||||
123
cmd/pulse-sensor-proxy/validation_test.go
Normal file
123
cmd/pulse-sensor-proxy/validation_test.go
Normal file
|
|
@ -0,0 +1,123 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSanitizeCorrelationID(t *testing.T) {
|
||||
valid := sanitizeCorrelationID("550e8400-e29b-41d4-a716-446655440000")
|
||||
if valid != "550e8400-e29b-41d4-a716-446655440000" {
|
||||
t.Fatalf("expected valid UUID to pass through, got %s", valid)
|
||||
}
|
||||
|
||||
invalid := sanitizeCorrelationID("not-a-uuid")
|
||||
if invalid == "not-a-uuid" {
|
||||
t.Fatalf("expected invalid UUID to be replaced")
|
||||
}
|
||||
|
||||
empty := sanitizeCorrelationID("")
|
||||
if empty == "" {
|
||||
t.Fatalf("expected empty string to be replaced")
|
||||
}
|
||||
|
||||
if invalid == empty {
|
||||
t.Fatalf("expected regenerated UUIDs to differ")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateNodeName(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
wantErr bool
|
||||
desc string
|
||||
}{
|
||||
{name: "node-1", wantErr: false, desc: "alphanumeric"},
|
||||
{name: "example.com", wantErr: false, desc: "dns hostname"},
|
||||
{name: "1.2.3.4", wantErr: false, desc: "ipv4"},
|
||||
{name: "2001:db8::1", wantErr: false, desc: "ipv6 compressed"},
|
||||
{name: "[2001:db8::10]", wantErr: false, desc: "ipv6 bracketed"},
|
||||
{name: "::1", wantErr: false, desc: "ipv6 loopback"},
|
||||
{name: "::", wantErr: false, desc: "ipv6 unspecified"},
|
||||
{name: "::ffff:192.0.2.1", wantErr: false, desc: "ipv4-mapped ipv6 dual stack"},
|
||||
{name: "[::1]", wantErr: false, desc: "ipv6 loopback bracketed"},
|
||||
{name: "fe80::1%eth0", wantErr: true, desc: "ipv6 zone identifier"},
|
||||
{name: "[fe80::1%eth0]", wantErr: true, desc: "ipv6 zone identifier bracketed"},
|
||||
{name: "[2001:db8::1]:22", wantErr: true, desc: "ipv6 with port suffix"},
|
||||
{name: "[2001:db8::1", wantErr: true, desc: "missing closing bracket"},
|
||||
{name: "2001:db8::1]", wantErr: true, desc: "missing opening bracket"},
|
||||
{name: "bad host", wantErr: true, desc: "whitespace disallowed"},
|
||||
{name: "-leadinghyphen", wantErr: true, desc: "leading hyphen disallowed"},
|
||||
{name: "example.com:22", wantErr: true, desc: "dns name with port"},
|
||||
{name: "", wantErr: true, desc: "empty string"},
|
||||
{name: "example_com", wantErr: false, desc: "underscore"},
|
||||
{name: "NODE123", wantErr: false, desc: "uppercase"},
|
||||
{name: strings.Repeat("a", 64), wantErr: false, desc: "64 chars"},
|
||||
{name: strings.Repeat("a", 65), wantErr: true, desc: "65 chars"},
|
||||
{name: "senso\u200Brs", wantErr: true, desc: "zero-width space"},
|
||||
{name: "node\\name", wantErr: true, desc: "backslash"},
|
||||
{name: "/etc/passwd", wantErr: true, desc: "absolute path"},
|
||||
{name: "node\x00", wantErr: true, desc: "null byte"},
|
||||
{name: "example.com;rm", wantErr: true, desc: "semicolon"},
|
||||
{name: "node$(rm)", wantErr: true, desc: "subshell"},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
tc := tc
|
||||
name := tc.desc
|
||||
if name == "" {
|
||||
name = tc.name
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
err := validateNodeName(tc.name)
|
||||
if tc.wantErr && err == nil {
|
||||
t.Fatalf("expected error validating %q", tc.name)
|
||||
}
|
||||
if !tc.wantErr && err != nil {
|
||||
t.Fatalf("unexpected error for %q: %v", tc.name, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateCommand(t *testing.T) {
|
||||
type tc struct {
|
||||
name string
|
||||
args []string
|
||||
wantErr bool
|
||||
desc string
|
||||
}
|
||||
|
||||
cases := []tc{
|
||||
{name: "sensors", args: nil, wantErr: false, desc: "bare sensors"},
|
||||
{name: "sensors", args: []string{"-j"}, wantErr: false, desc: "json flag"},
|
||||
{name: "ipmitool", args: []string{"sdr"}, wantErr: false, desc: "safe ipmitool"},
|
||||
{name: "sensors", args: []string{"; rm -rf /"}, wantErr: true, desc: "shell metachar"},
|
||||
{name: "sensors", args: []string{"$(id)"}, wantErr: true, desc: "subshell"},
|
||||
{name: "ipmitool", args: []string{"-H", "1.2.3.4", "&&", "shutdown"}, wantErr: true, desc: "command chaining"},
|
||||
{name: "sensors", args: []string{">/tmp/out"}, wantErr: true, desc: "redirect"},
|
||||
{name: "senso\u200Brs", wantErr: true, desc: "unicode homoglyph"},
|
||||
{name: "sensors", args: []string{"-" + strings.Repeat("v", 2000)}, wantErr: true, desc: "arg too long"},
|
||||
{name: "sensors", args: []string{"test\x00"}, wantErr: true, desc: "null byte arg"},
|
||||
{name: "ipmitool", args: []string{"chassis", "power", "off"}, wantErr: true, desc: "dangerous ipmitool"},
|
||||
{name: "sensors", args: []string{"LC_ALL=C"}, wantErr: true, desc: "env prefix"},
|
||||
{name: "/usr/bin/sensors", wantErr: true, desc: "absolute path"},
|
||||
{name: "ipmitool", args: []string{"--extraneous=../../etc/passwd"}, wantErr: true, desc: "path traversal"},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
tc := tc
|
||||
if tc.desc == "" {
|
||||
tc.desc = tc.name
|
||||
}
|
||||
t.Run(tc.desc, func(t *testing.T) {
|
||||
err := validateCommand(tc.name, tc.args)
|
||||
if tc.wantErr && err == nil {
|
||||
t.Fatalf("expected error for %s %v", tc.name, tc.args)
|
||||
}
|
||||
if !tc.wantErr && err != nil {
|
||||
t.Fatalf("unexpected error for %s %v: %v", tc.name, tc.args, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -1,23 +1,25 @@
|
|||
version: '3.8'
|
||||
|
||||
services:
|
||||
pulse:
|
||||
image: rcourtman/pulse:latest
|
||||
image: ${PULSE_IMAGE:-rcourtman/pulse:latest}
|
||||
container_name: pulse
|
||||
ports:
|
||||
- "7655:7655" # Web UI and API
|
||||
volumes:
|
||||
- pulse_data:/data
|
||||
environment:
|
||||
- TZ=UTC # Set your timezone
|
||||
# - PUID=1000 # Optional: Set user ID (uncomment and adjust as needed)
|
||||
# - PGID=1000 # Optional: Set group ID (uncomment and adjust as needed)
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "${PULSE_PORT:-7655}:7655"
|
||||
volumes:
|
||||
- pulse-data:/data
|
||||
# Secure temperature monitoring via host-side proxy
|
||||
- /mnt/pulse-proxy:/mnt/pulse-proxy:ro
|
||||
environment:
|
||||
- TZ=${TZ:-UTC}
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:7655"]
|
||||
test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://localhost:7655/api/health"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
start_period: 30s
|
||||
start_period: 10s
|
||||
|
||||
volumes:
|
||||
pulse_data:
|
||||
driver: local
|
||||
pulse-data:
|
||||
driver: local
|
||||
|
|
|
|||
71
docs/PHASE1_SUMMARY.md
Normal file
71
docs/PHASE1_SUMMARY.md
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
# Pulse Sensor Proxy – Phase 1 Summary
|
||||
|
||||
## Executive Summary
|
||||
Phase 1 delivered a complete hardening and observability overhaul for the Pulse sensor proxy. The service now runs under least privilege, exposes tamper-evident audit trails, forwards logs off-host, enforces adaptive rate caps, and ships with comprehensive validation tests plus documentation for ongoing operations and security posture. These improvements dramatically reduce the proxy's attack surface while giving operators clear visibility and controls.
|
||||
|
||||
## Security Improvements
|
||||
- **Host hardening**
|
||||
- SSH daemon locked down (no passwords, no forwarding, `ForceCommand` wrapper).
|
||||
- Dedicated user `pulse-sensor` with minimal home/project directories.
|
||||
- File permissions tightened (0750 binaries, 0600 private keys, 0640 append-only logs).
|
||||
- Privilege drop via `Setuid/Setgid` post-bind; service confirms running as unprivileged UID 995.
|
||||
- **Command execution guardrails**
|
||||
- Whitelist-based command validator for `sensors`/`ipmitool`; rejects shell metacharacters, subshells, dangerous ipmitool subcommands, null bytes, and path traversal.
|
||||
- Enhanced node-name validation covering unicode, length, and absolute path abuse.
|
||||
- **Logging & audit**
|
||||
- Structured audit logger with hash chain + HMAC-style tamper detection.
|
||||
- Remote forwarding via `rsyslog` (RELP/TLS) and local queue for resilience.
|
||||
- **Sandboxing**
|
||||
- Network segmentation documentation with firewall ACLs.
|
||||
- AppArmor profile restricting filesystem/networking; seccomp profile (classic + OCI JSON).
|
||||
- **Rate limiting**
|
||||
- Per-UID token bucket (0.2 QPS burst 2) + global concurrency cap (8) + penalty sleeps.
|
||||
- Audit + metrics instrumentation for limiter decisions and penalties.
|
||||
|
||||
## Key Metrics & Tests
|
||||
- `go test ./cmd/pulse-sensor-proxy/...` passes (including new unit suites for command validation, sanitizer, limiter penalties, audit logging).
|
||||
- 10 hostile-command attack cases covered (metacharacters, subshells, redirects, homoglyphs, null bytes, long args, path traversal, dangerous ipmitool ops, env prefixes, absolute paths).
|
||||
- Fuzz harness (`FuzzValidateCommand`) executing for 24 h (Task #24).
|
||||
- Prometheus metrics validated:
|
||||
- `pulse_proxy_limiter_rejections_total{reason="rate"}` increments under load.
|
||||
- `pulse_proxy_limiter_penalties_total{reason="invalid_json"}` increments on validation failure.
|
||||
- `pulse_proxy_limiter_active_peers` accurate (UID grouping).
|
||||
- Audit log entries verified: connection acceptance, limiter rejections, validation failures, command start/finish with event hash chaining.
|
||||
|
||||
## Deployment Checklist
|
||||
1. **Scripts**
|
||||
- Run `scripts/create-sensor-user.sh`
|
||||
- Run `scripts/harden-sensor-proxy.sh`
|
||||
- Run `scripts/secure-sensor-files.sh`
|
||||
- Run `scripts/setup-log-forwarding.sh`
|
||||
2. **Binaries**
|
||||
- Build/install `/opt/pulse/sensor-proxy/bin/pulse-sensor-proxy` (0750, root:pulse-sensor).
|
||||
3. **Configuration**
|
||||
- `/etc/pulse-sensor-proxy/config.yaml` updated (allowed subnets, UID/GID list, metrics addr).
|
||||
- Systemd unit exports `PULSE_SENSOR_PROXY_USER`, `PULSE_SENSOR_PROXY_SSH_DIR`, `PULSE_SENSOR_PROXY_AUDIT_LOG`.
|
||||
4. **Profiles**
|
||||
- Deploy AppArmor profile (`security/apparmor/pulse-sensor-proxy.apparmor`).
|
||||
- Apply seccomp (systemd `SystemCallFilter` overrides or container JSON profile).
|
||||
5. **Networking**
|
||||
- Implement firewall ACLs per `docs/security/pulse-sensor-proxy-network.md`.
|
||||
6. **Log Forwarding**
|
||||
- Place TLS certs in `/etc/pulse/log-forwarding`.
|
||||
- Verify rsyslog forwarding to remote collector.
|
||||
7. **Restart & Validate**
|
||||
- `systemctl restart pulse-sensor-proxy`.
|
||||
- Confirm metrics endpoint, audit log creation, limiter behaviour.
|
||||
|
||||
## Verification Steps
|
||||
1. **Privilege Drop**: `ps -o user= -p $(pgrep -f pulse-sensor-proxy)` → `pulse-sensor`.
|
||||
2. **Audit Trail**: Trigger RPC (`get_status`) → verify `audit.log` entries with valid `event_hash`.
|
||||
3. **Rate Limiter**: Fire >10 concurrent requests → confirm `pulse_proxy_limiter_rejections_total{reason="rate"}` and audit `limiter.rejection`.
|
||||
4. **Remote Logging**: `logger` or manual append to proxy log → confirm arrival at remote collector.
|
||||
5. **Security Profiles**: `aa-status | grep pulse-sensor-proxy` (enforced), `systemctl show pulse-sensor-proxy -p SystemCallFilter`.
|
||||
6. **App Functionality**: Run `ensure_cluster_keys`, `get_temperature` RPCs, ensure success and no audit warnings.
|
||||
|
||||
## Known Limitations / Deferred to Phase 2
|
||||
- **Adaptive Polling**: still fixed intervals (Phase 2 focuses on controller, backpressure, staleness SLOs).
|
||||
- **Queue Backpressure**: groundwork in rate limiter; full queue-based collector scheduling to be built next.
|
||||
- **External Sentinels**: cross-check monitoring and metric ingestion planned in Phase 3.
|
||||
- **AppArmor/Seccomp Tuning**: profiles may need refinement after real-world observation.
|
||||
- **Long-run Fuzz Results**: Task #24 fuzz campaign active; incorporate findings post-run.
|
||||
58
docs/operations/pulse-sensor-proxy-runbook.md
Normal file
58
docs/operations/pulse-sensor-proxy-runbook.md
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
# Pulse Sensor Proxy Runbook
|
||||
|
||||
## Quick Reference
|
||||
- Binary: `/opt/pulse/sensor-proxy/bin/pulse-sensor-proxy`
|
||||
- Unit: `pulse-sensor-proxy.service`
|
||||
- Logs: `/var/log/pulse/sensor-proxy/proxy.log`
|
||||
- Audit trail: `/var/log/pulse/sensor-proxy/audit.log` (hash chained, forwarded via rsyslog)
|
||||
- Metrics: `http://127.0.0.1:9456/metrics`
|
||||
- Limiters: per-UID token bucket (burst 2) + global concurrency (8)
|
||||
|
||||
## Monitoring Alerts & Response
|
||||
### Rate Limit Hits (`pulse_proxy_limiter_rejections_total`)
|
||||
1. Check audit log entries tagged `limiter.rejection` for offending UID.
|
||||
2. Confirm workload legitimacy; if expected, consider increasing limits via config override.
|
||||
3. If malicious, block source process/user and inspect Pulse audit logs.
|
||||
|
||||
### Penalty Events (`pulse_proxy_limiter_penalties_total`)
|
||||
1. Review corresponding validation failures in audit log (`command.validation_failed`).
|
||||
2. If repeated invalid JSON/unknown methods, inspect caller code for regressions or intrusion attempts.
|
||||
|
||||
### Audit Log Forwarder Down
|
||||
1. `journalctl -u rsyslog` to confirm transmission errors.
|
||||
2. Ensure `/etc/pulse/log-forwarding` certs valid & remote host reachable.
|
||||
3. Forwarding queue stored locally in `/var/log/pulse/sensor-proxy/forwarding.log`; ship manually if outage exceeds 1 hour.
|
||||
|
||||
### Proxy Health Endpoint Fails
|
||||
1. `systemctl status pulse-sensor-proxy`
|
||||
2. Check `/var/log/pulse/sensor-proxy/proxy.log` for panic or limiter exhaustion.
|
||||
3. Inspect `/var/log/pulse/sensor-proxy/audit.log` for recent privileged method denials.
|
||||
|
||||
## Standard Procedures
|
||||
### Restart Proxy Safely
|
||||
```bash
|
||||
sudo systemctl stop pulse-sensor-proxy
|
||||
sudo apparmor_parser -r /etc/apparmor.d/pulse-sensor-proxy # if updating policy
|
||||
sudo systemctl start pulse-sensor-proxy
|
||||
```
|
||||
Verify: `curl -s http://127.0.0.1:9456/metrics | grep pulse_proxy_build_info`.
|
||||
|
||||
### Rotate SSH Keys
|
||||
1. Run `scripts/secure-sensor-files.sh` to regenerate keys (ensure environment locked down).
|
||||
2. Use RPC `ensure_cluster_keys` to distribute new public key.
|
||||
3. Confirm nodes accept `ssh` from proxy host.
|
||||
|
||||
### Adjust Rate Limits
|
||||
1. Update `limiter_policy` environment overrides (future config).
|
||||
2. Restart proxy; monitor limiter metrics to validate new thresholds.
|
||||
3. Document change in security runbook.
|
||||
|
||||
## Incident Handling
|
||||
- **Unauthorized Command Attempt:** audit log shows `command.validation_failed` and limiter penalties; capture correlation ID, check Pulse side for compromised container.
|
||||
- **Excessive Temperature Failures:** refer to `pulse_proxy_ssh_requests_total{result="error"}`; validate network ACLs and node health; escalate to Proxmox team if nodes unreachable.
|
||||
- **Log Tampering Suspected:** verify audit hash chain by replaying `eventHash` values; compare with remote log store (immutable). Trigger security response if mismatch.
|
||||
|
||||
## Postmortem Checklist
|
||||
- Timeline: command audit entries, limiter stats, rsyslog queue depth.
|
||||
- Verify AppArmor/seccomp status (`aa-status`, `systemctl show pulse-sensor-proxy -p AppArmorProfile`).
|
||||
- Ensure firewall ACLs match `docs/security/pulse-sensor-proxy-network.md`.
|
||||
52
docs/security/pulse-sensor-proxy-hardening.md
Normal file
52
docs/security/pulse-sensor-proxy-hardening.md
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
# Pulse Sensor Proxy AppArmor & Seccomp Hardening
|
||||
|
||||
## AppArmor Profile
|
||||
- Profile path: `security/apparmor/pulse-sensor-proxy.apparmor`
|
||||
- Grants read-only access to configs, logs, SSH keys, and binaries; allows outbound TCP/SSH; blocks raw sockets, module loading, ptrace, and absolute command execution outside the allowlist.
|
||||
|
||||
### Installation
|
||||
```bash
|
||||
sudo install -m 0644 security/apparmor/pulse-sensor-proxy.apparmor /etc/apparmor.d/pulse-sensor-proxy
|
||||
sudo apparmor_parser -r /etc/apparmor.d/pulse-sensor-proxy
|
||||
sudo ln -sf /etc/apparmor.d/pulse-sensor-proxy /etc/apparmor.d/force-complain/pulse-sensor-proxy # optional staged mode
|
||||
sudo systemctl restart apparmor
|
||||
```
|
||||
|
||||
### Enforce Mode
|
||||
```bash
|
||||
sudo aa-enforce pulse-sensor-proxy
|
||||
```
|
||||
Monitor `/var/log/syslog` for `DENIED` events and update the profile as needed.
|
||||
|
||||
## Seccomp Filter
|
||||
- OCI-style profile: `security/seccomp/pulse-sensor-proxy.json`
|
||||
- Allows standard Go runtime syscalls, network operations, file IO, and `execve` for whitelisted helpers; other syscalls return `EPERM`.
|
||||
|
||||
### Apply via systemd (classic service)
|
||||
Add to the override:
|
||||
```ini
|
||||
[Service]
|
||||
AppArmorProfile=pulse-sensor-proxy
|
||||
RestrictNamespaces=yes
|
||||
NoNewPrivileges=yes
|
||||
SystemCallFilter=@system-service
|
||||
SystemCallArchitectures=native
|
||||
SystemCallAllow=accept;connect;recvfrom;sendto;recvmsg;sendmsg;sendmmsg;getsockname;getpeername;getsockopt;setsockopt;shutdown
|
||||
```
|
||||
|
||||
Reload and restart:
|
||||
```bash
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl restart pulse-sensor-proxy
|
||||
```
|
||||
|
||||
### Apply seccomp JSON (containerised deployments)
|
||||
- Profile: `security/seccomp/pulse-sensor-proxy.json`
|
||||
- Use with Podman/Docker style runtimes:
|
||||
```bash
|
||||
podman run --seccomp-profile /opt/pulse/security/seccomp/pulse-sensor-proxy.json ...
|
||||
```
|
||||
|
||||
## Operational Notes
|
||||
- Use `journalctl -t auditbeat -g pulse-sensor-proxy` or `aa-status` to confirm profile status.
|
||||
- Pair with network ACLs (see `docs/security/pulse-sensor-proxy-network.md`) and log shipping (`scripts/setup-log-forwarding.sh`).
|
||||
64
docs/security/pulse-sensor-proxy-network.md
Normal file
64
docs/security/pulse-sensor-proxy-network.md
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
# Pulse Sensor Proxy Network Segmentation
|
||||
|
||||
## Overview
|
||||
- **Proxy host** collects temperatures via SSH from Proxmox nodes and serves a Unix socket to the Pulse stack.
|
||||
- Goals: isolate the proxy from production hypervisors, prevent lateral movement, and ensure log forwarding/audit channels remain available.
|
||||
|
||||
## Zones & Connectivity
|
||||
- **Pulse Application Zone (AZ-Pulse)**
|
||||
- Hosts Pulse backend/frontend containers.
|
||||
- Allowed to reach the proxy over Unix socket (local) or loopback if containerised via `socat`.
|
||||
- **Sensor Proxy Zone (AZ-Sensor)**
|
||||
- Dedicated VM/bare-metal host running `pulse-sensor-proxy`.
|
||||
- Maintains outbound SSH to Proxmox management interfaces only.
|
||||
- **Proxmox Management Zone (AZ-Proxmox)**
|
||||
- Hypervisors / BMCs reachable on `tcp/22` (SSH) and optional IPMI UDP.
|
||||
- **Logging/Monitoring Zone (AZ-Logging)**
|
||||
- Receives forwarded audit/application logs (e.g. RELP/TLS on `tcp/6514`).
|
||||
- Exposes Prometheus scrape port (default `tcp/9456`) if remote monitoring required.
|
||||
|
||||
## Recommended Firewall Rules
|
||||
|
||||
| Source Zone | Destination Zone | Protocol/Port | Purpose | Action |
|
||||
|-------------|------------------|---------------|---------|--------|
|
||||
| AZ-Pulse (localhost) | AZ-Sensor (Unix socket) | `unix` | RPC requests from Pulse | Allow (local only) |
|
||||
| AZ-Sensor | AZ-Proxmox nodes | `tcp/22` | SSH for sensors/ipmitool wrapper | Allow (restricted to node list) |
|
||||
| AZ-Sensor | AZ-Proxmox BMC | `udp/623` *(optional)* | IPMI if required for temperature data | Allow if needed |
|
||||
| AZ-Proxmox | AZ-Sensor | `any` | Return SSH traffic | Allow stateful |
|
||||
| AZ-Sensor | AZ-Logging | `tcp/6514` (TLS RELP) | Audit/application log forwarding | Allow |
|
||||
| AZ-Logging | AZ-Sensor | `tcp/9456` *(optional)* | Prometheus scrape of proxy metrics | Allow if scraping remotely |
|
||||
| Any | AZ-Sensor | `tcp/22` | Shell/SSH access | Deny (use management bastion) |
|
||||
| AZ-Sensor | Internet | `any` | Outbound Internet | Deny (except package mirrors via proxy if required) |
|
||||
|
||||
## Implementation Steps
|
||||
1. Place proxy host in dedicated subnet/VLAN with ACLs enforcing the table above.
|
||||
2. Populate `/etc/hosts` or routing so proxy resolves Proxmox nodes to management IPs only (no public networks).
|
||||
3. Configure iptables/nftables on proxy:
|
||||
```bash
|
||||
# Allow SSH to Proxmox nodes
|
||||
iptables -A OUTPUT -p tcp -d <PROXMOX_SUBNET>/24 --dport 22 -m conntrack --ctstate NEW,ESTABLISHED -j ACCEPT
|
||||
iptables -A INPUT -p tcp -s <PROXMOX_SUBNET>/24 --sport 22 -m conntrack --ctstate ESTABLISHED -j ACCEPT
|
||||
|
||||
# Allow log forwarding
|
||||
iptables -A OUTPUT -p tcp -d <LOG_HOST> --dport 6514 -m conntrack --ctstate NEW,ESTABLISHED -j ACCEPT
|
||||
iptables -A INPUT -p tcp -s <LOG_HOST> --sport 6514 -m conntrack --ctstate ESTABLISHED -j ACCEPT
|
||||
|
||||
# (Optional) allow Prometheus scrape
|
||||
iptables -A INPUT -p tcp -s <SCRAPE_HOST> --dport 9456 -m conntrack --ctstate NEW,ESTABLISHED -j ACCEPT
|
||||
iptables -A OUTPUT -p tcp -d <SCRAPE_HOST> --sport 9456 -m conntrack --ctstate ESTABLISHED -j ACCEPT
|
||||
|
||||
# Drop everything else
|
||||
iptables -P OUTPUT DROP
|
||||
iptables -P INPUT DROP
|
||||
```
|
||||
4. Deny inbound SSH to proxy except via management bastion: block `tcp/22` or whitelist bastion IPs.
|
||||
5. Ensure log-forwarding TLS certificates are rotated and stored under `/etc/pulse/log-forwarding`.
|
||||
|
||||
## Monitoring & Alerting
|
||||
- Alert if proxy initiates connections outside permitted subnets (Netflow or host firewall counters).
|
||||
- Monitor `pulse_proxy_limiter_*` metrics for unusual rate-limit hits that might signal abuse.
|
||||
- Track `audit_log` forwarding queue depth and remote availability; on failure, emit alert via rsyslog action queue (set `action.resumeRetryCount=-1` already).
|
||||
|
||||
## Change Management
|
||||
- Document node IP changes and update firewall objects (`PROXMOX_NODES`) before redeploying certificates.
|
||||
- Capture segmentation in infrastructure-as-code (e.g. Terraform/security group definitions) to avoid drift.
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
# Netscape HTTP Cookie File
|
||||
# https://curl.se/docs/http-cookies.html
|
||||
# This file was generated by libcurl! Edit at your own risk.
|
||||
|
||||
File diff suppressed because one or more lines are too long
17
frontend-modern/dist/index.html
vendored
17
frontend-modern/dist/index.html
vendored
|
|
@ -1,17 +0,0 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<meta name="theme-color" content="#000000" />
|
||||
<link rel="icon" type="image/svg+xml" href="/logo.svg" />
|
||||
<title>Pulse</title>
|
||||
<script type="module" crossorigin src="/assets/index-BTYq3hC4.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="/assets/index-C-bZ849w.css">
|
||||
</head>
|
||||
<body>
|
||||
<noscript>You need to enable JavaScript to run this app.</noscript>
|
||||
<div id="root"></div>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
16
frontend-modern/dist/logo.svg
vendored
16
frontend-modern/dist/logo.svg
vendored
|
|
@ -1,16 +0,0 @@
|
|||
<svg width="256" height="256" viewBox="0 0 256 256" xmlns="http://www.w3.org/2000/svg">
|
||||
<title>Pulse Logo</title>
|
||||
<style>
|
||||
.pulse-bg { fill: #2563eb; }
|
||||
.pulse-ring { fill: none; stroke: #ffffff; stroke-width: 14; opacity: 0.92; }
|
||||
.pulse-center { fill: #ffffff; }
|
||||
@media (prefers-color-scheme: dark) {
|
||||
.pulse-bg { fill: #3b82f6; }
|
||||
.pulse-ring { stroke: #dbeafe; }
|
||||
.pulse-center { fill: #dbeafe; }
|
||||
}
|
||||
</style>
|
||||
<circle class="pulse-bg" cx="128" cy="128" r="122"/>
|
||||
<circle class="pulse-ring" cx="128" cy="128" r="84"/>
|
||||
<circle class="pulse-center" cx="128" cy="128" r="26"/>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 625 B |
|
|
@ -938,8 +938,17 @@ function AppLayout(props: {
|
|||
>
|
||||
{platform.icon}
|
||||
<span>{platform.label}</span>
|
||||
<Show when={disabled() && !platform.live}>
|
||||
<span class="ml-1 text-[10px] uppercase tracking-wide text-gray-400 dark:text-gray-600">Add host</span>
|
||||
<Show when={!platform.live}>
|
||||
<button
|
||||
type="button"
|
||||
onClick={(event) => {
|
||||
event.stopPropagation();
|
||||
navigate(platform.settingsRoute);
|
||||
}}
|
||||
class="ml-1 text-[10px] uppercase tracking-wide text-gray-400 dark:text-gray-600 hover:text-blue-500 focus-visible:outline-none focus-visible:ring-0"
|
||||
>
|
||||
Add host
|
||||
</button>
|
||||
</Show>
|
||||
</div>
|
||||
);
|
||||
|
|
|
|||
|
|
@ -1018,48 +1018,49 @@ const snapshotOverridesCount = createMemo(() => {
|
|||
// Get PBS instances from props
|
||||
const pbsInstances = props.pbsInstances || [];
|
||||
|
||||
const pbsServers = pbsInstances
|
||||
.filter((pbs) => (pbs.cpu || 0) > 0 || (pbs.memory || 0) > 0)
|
||||
.map((pbs) => {
|
||||
// PBS IDs already have "pbs-" prefix from backend, don't double it
|
||||
const pbsId = pbs.id;
|
||||
const override = overridesMap.get(pbsId);
|
||||
const pbsServers = pbsInstances.map((pbs) => {
|
||||
// Offline PBS instances report zero metrics; keep them visible so connectivity toggles stay usable
|
||||
// PBS IDs already have "pbs-" prefix from backend, don't double it
|
||||
const pbsId = pbs.id;
|
||||
const override = overridesMap.get(pbsId);
|
||||
|
||||
// Check if any threshold values actually differ from defaults
|
||||
const hasCustomThresholds =
|
||||
override?.thresholds &&
|
||||
Object.keys(override.thresholds).some((key) => {
|
||||
const k = key as keyof typeof override.thresholds;
|
||||
// PBS uses node defaults for CPU/Memory
|
||||
return (
|
||||
override.thresholds[k] !== undefined &&
|
||||
override.thresholds[k] !== props.nodeDefaults[k as keyof typeof props.nodeDefaults]
|
||||
);
|
||||
});
|
||||
// Check if any threshold values actually differ from defaults
|
||||
const hasCustomThresholds =
|
||||
override?.thresholds &&
|
||||
Object.keys(override.thresholds).some((key) => {
|
||||
const k = key as keyof typeof override.thresholds;
|
||||
// PBS uses node defaults for CPU/Memory
|
||||
return (
|
||||
override.thresholds[k] !== undefined &&
|
||||
override.thresholds[k] !== props.nodeDefaults[k as keyof typeof props.nodeDefaults]
|
||||
);
|
||||
});
|
||||
|
||||
const disableConnectivity = override?.disableConnectivity || false;
|
||||
const hasOverride = hasCustomThresholds || disableConnectivity;
|
||||
|
||||
return {
|
||||
id: pbsId,
|
||||
name: pbs.name,
|
||||
type: 'pbs' as const,
|
||||
resourceType: 'PBS',
|
||||
host: pbs.host,
|
||||
status: pbs.status,
|
||||
cpu: pbs.cpu,
|
||||
memory: pbs.memory,
|
||||
memoryUsed: pbs.memoryUsed,
|
||||
memoryTotal: pbs.memoryTotal,
|
||||
uptime: pbs.uptime,
|
||||
hasOverride: hasCustomThresholds || false,
|
||||
disabled: false,
|
||||
disableConnectivity: override?.disableConnectivity || false,
|
||||
thresholds: override?.thresholds || {},
|
||||
defaults: {
|
||||
cpu: props.nodeDefaults.cpu,
|
||||
memory: props.nodeDefaults.memory,
|
||||
},
|
||||
};
|
||||
});
|
||||
return {
|
||||
id: pbsId,
|
||||
name: pbs.name,
|
||||
type: 'pbs' as const,
|
||||
resourceType: 'PBS',
|
||||
host: pbs.host,
|
||||
status: pbs.status,
|
||||
cpu: pbs.cpu,
|
||||
memory: pbs.memory,
|
||||
memoryUsed: pbs.memoryUsed,
|
||||
memoryTotal: pbs.memoryTotal,
|
||||
uptime: pbs.uptime,
|
||||
hasOverride,
|
||||
disabled: false,
|
||||
disableConnectivity,
|
||||
thresholds: override?.thresholds || {},
|
||||
defaults: {
|
||||
cpu: props.nodeDefaults.cpu,
|
||||
memory: props.nodeDefaults.memory,
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
if (search) {
|
||||
return pbsServers.filter(
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { Component, createSignal, Show, For, createMemo, createEffect, onMount } from 'solid-js';
|
||||
import { useNavigate } from '@solidjs/router';
|
||||
import { useWebSocket } from '@/App';
|
||||
import { formatBytes, formatAbsoluteTime, formatRelativeTime, formatUptime } from '@/utils/format';
|
||||
import { createLocalStorageBooleanSignal, STORAGE_KEYS } from '@/utils/localStorage';
|
||||
|
|
@ -24,6 +25,7 @@ interface DateGroup {
|
|||
}
|
||||
|
||||
const UnifiedBackups: Component = () => {
|
||||
const navigate = useNavigate();
|
||||
const { state } = useWebSocket();
|
||||
const pveBackupsState = createMemo(() => state.backups?.pve ?? state.pveBackups);
|
||||
const pbsBackupsState = createMemo(() => state.backups?.pbs ?? state.pbsBackups);
|
||||
|
|
@ -1141,12 +1143,7 @@ const UnifiedBackups: Component = () => {
|
|||
actions={
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => {
|
||||
const settingsTab = document.querySelector(
|
||||
'[role="tab"]:last-child',
|
||||
) as HTMLElement;
|
||||
settingsTab?.click();
|
||||
}}
|
||||
onClick={() => navigate('/settings')}
|
||||
class="inline-flex items-center px-3 py-1.5 border border-transparent text-xs font-medium rounded-md text-white bg-blue-600 hover:bg-blue-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500"
|
||||
>
|
||||
Go to Settings
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { createSignal, createMemo, createEffect, For, Show, onMount } from 'solid-js';
|
||||
import { useNavigate } from '@solidjs/router';
|
||||
import type { VM, Container, Node } from '@/types/api';
|
||||
import { GuestRow } from './GuestRow';
|
||||
import { useWebSocket } from '@/App';
|
||||
|
|
@ -28,6 +29,7 @@ type StatusMode = 'all' | 'running' | 'stopped';
|
|||
type GroupingMode = 'grouped' | 'flat';
|
||||
|
||||
export function Dashboard(props: DashboardProps) {
|
||||
const navigate = useNavigate();
|
||||
const ws = useWebSocket();
|
||||
const { connected, activeAlerts, initialDataReceived, reconnecting, reconnect } = ws;
|
||||
const [search, setSearch] = createSignal('');
|
||||
|
|
@ -639,12 +641,7 @@ export function Dashboard(props: DashboardProps) {
|
|||
actions={
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => {
|
||||
const settingsTab = document.querySelector(
|
||||
'[role="tab"]:last-child',
|
||||
) as HTMLElement;
|
||||
settingsTab?.click();
|
||||
}}
|
||||
onClick={() => navigate('/settings')}
|
||||
class="inline-flex items-center px-3 py-1.5 border border-transparent text-xs font-medium rounded-md text-white bg-blue-600 hover:bg-blue-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500"
|
||||
>
|
||||
Go to Settings
|
||||
|
|
|
|||
|
|
@ -580,7 +580,10 @@ IMPORTANT: Keep these credentials secure!
|
|||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => window.location.reload()}
|
||||
onClick={() => {
|
||||
const basePath = import.meta.env.BASE_URL || '/';
|
||||
window.location.assign(basePath);
|
||||
}}
|
||||
class="flex-1 py-3 px-4 bg-blue-600 hover:bg-blue-700 text-white rounded-lg font-medium transition-colors"
|
||||
>
|
||||
Continue to Login
|
||||
|
|
|
|||
|
|
@ -329,7 +329,6 @@ const Settings: Component<SettingsProps> = (props) => {
|
|||
if (currentTab() !== 'pve') {
|
||||
setCurrentTab('pve');
|
||||
}
|
||||
navigate('/settings/pve', { replace: true });
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { Component, For, Show, createSignal, createMemo, createEffect, onMount } from 'solid-js';
|
||||
import { useNavigate } from '@solidjs/router';
|
||||
import { useWebSocket } from '@/App';
|
||||
import { getAlertStyles } from '@/utils/alerts';
|
||||
import { formatBytes } from '@/utils/format';
|
||||
|
|
@ -14,6 +15,7 @@ import { ProxmoxSectionNav } from '@/components/Proxmox/ProxmoxSectionNav';
|
|||
import { getNodeDisplayName } from '@/utils/nodes';
|
||||
|
||||
const Storage: Component = () => {
|
||||
const navigate = useNavigate();
|
||||
const { state, connected, activeAlerts, initialDataReceived } = useWebSocket();
|
||||
const [viewMode, setViewMode] = createSignal<'node' | 'storage'>('node');
|
||||
const [tabView, setTabView] = createSignal<'pools' | 'disks'>('pools');
|
||||
|
|
@ -671,12 +673,7 @@ const Storage: Component = () => {
|
|||
actions={
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => {
|
||||
const settingsTab = document.querySelector(
|
||||
'[role=\"tab\"]:last-child',
|
||||
) as HTMLElement;
|
||||
settingsTab?.click();
|
||||
}}
|
||||
onClick={() => navigate('/settings')}
|
||||
class="inline-flex items-center px-3 py-1.5 border border-transparent text-xs font-medium rounded-md text-white bg-blue-600 hover:bg-blue-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500"
|
||||
>
|
||||
Go to Settings
|
||||
|
|
|
|||
|
|
@ -766,7 +766,8 @@ func (m *Manager) UpdateConfig(config AlertConfig) {
|
|||
config.TimeThresholds = make(map[string]int)
|
||||
}
|
||||
ensureDelay := func(key string) {
|
||||
if delay, ok := config.TimeThresholds[key]; !ok || delay <= 0 {
|
||||
delay, ok := config.TimeThresholds[key]
|
||||
if !ok || delay < 0 {
|
||||
config.TimeThresholds[key] = defaultDelaySeconds
|
||||
}
|
||||
}
|
||||
|
|
@ -774,7 +775,7 @@ func (m *Manager) UpdateConfig(config AlertConfig) {
|
|||
ensureDelay("node")
|
||||
ensureDelay("storage")
|
||||
ensureDelay("pbs")
|
||||
if delay, ok := config.TimeThresholds["all"]; ok && delay <= 0 {
|
||||
if delay, ok := config.TimeThresholds["all"]; ok && delay < 0 {
|
||||
config.TimeThresholds["all"] = defaultDelaySeconds
|
||||
}
|
||||
config.DockerIgnoredContainerPrefixes = NormalizeDockerIgnoredPrefixes(config.DockerIgnoredContainerPrefixes)
|
||||
|
|
@ -5299,7 +5300,7 @@ func (m *Manager) checkQuarantineMetric(pmg models.PMGInstance, metricType strin
|
|||
}
|
||||
|
||||
// calculateTrimmedBaseline computes a robust baseline from historical samples
|
||||
// using trimmed mean with median fallback as specified by Codex
|
||||
// using trimmed mean with median fallback for statistical robustness
|
||||
func calculateTrimmedBaseline(samples []float64) (baseline float64, trustworthy bool) {
|
||||
sampleCount := len(samples)
|
||||
|
||||
|
|
@ -5488,7 +5489,7 @@ func (m *Manager) checkAnomalyMetric(pmg models.PMGInstance, tracker *pmgAnomaly
|
|||
baseline = 1.0 // Treat as 1 for ratio math
|
||||
}
|
||||
|
||||
// Determine thresholds based on Codex spec
|
||||
// Determine warning and critical thresholds
|
||||
var warnRatio, critRatio float64
|
||||
var warnDelta, critDelta float64
|
||||
|
||||
|
|
|
|||
|
|
@ -3914,11 +3914,14 @@ elif [ "$TEMP_MONITORING_AVAILABLE" = true ]; then
|
|||
IS_RPI=true
|
||||
fi
|
||||
|
||||
TEMPERATURE_SETUP_SUCCESS=false
|
||||
|
||||
# Install lm-sensors if not present (skip on Raspberry Pi)
|
||||
if ! command -v sensors &> /dev/null; then
|
||||
if [ "$IS_RPI" = true ]; then
|
||||
echo " ℹ️ Raspberry Pi detected - using native RPi temperature interface"
|
||||
echo " Pulse will read temperature from /sys/class/thermal/thermal_zone0/temp"
|
||||
TEMPERATURE_SETUP_SUCCESS=true
|
||||
else
|
||||
echo " ✓ Installing lm-sensors..."
|
||||
|
||||
|
|
@ -3939,6 +3942,7 @@ elif [ "$TEMP_MONITORING_AVAILABLE" = true ]; then
|
|||
if apt-get install -y lm-sensors > /dev/null 2>&1; then
|
||||
sensors-detect --auto > /dev/null 2>&1 || true
|
||||
echo " ✓ lm-sensors installed successfully"
|
||||
TEMPERATURE_SETUP_SUCCESS=true
|
||||
else
|
||||
echo ""
|
||||
echo " ⚠️ Could not install lm-sensors"
|
||||
|
|
@ -3957,15 +3961,21 @@ elif [ "$TEMP_MONITORING_AVAILABLE" = true ]; then
|
|||
fi
|
||||
else
|
||||
echo " ✓ lm-sensors package verified"
|
||||
TEMPERATURE_SETUP_SUCCESS=true
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "✓ Temperature monitoring enabled"
|
||||
if [ "$IS_RPI" = true ]; then
|
||||
echo " Using Raspberry Pi native temperature interface"
|
||||
if [ "$TEMPERATURE_SETUP_SUCCESS" = true ]; then
|
||||
echo "✓ Temperature monitoring enabled"
|
||||
if [ "$IS_RPI" = true ]; then
|
||||
echo " Using Raspberry Pi native temperature interface"
|
||||
fi
|
||||
echo " Temperature data will appear in the dashboard within 10 seconds"
|
||||
TEMPERATURE_ENABLED=true
|
||||
else
|
||||
echo "✗ Temperature monitoring could not be enabled"
|
||||
echo " Resolve the installation issues above and rerun this step."
|
||||
fi
|
||||
echo " Temperature data will appear in the dashboard within 10 seconds"
|
||||
TEMPERATURE_ENABLED=true
|
||||
|
||||
# Configure automatic ProxyJump if needed (for containerized Pulse)
|
||||
if [ "$CONFIGURE_PROXYJUMP" = true ] && [ -n "$PROXY_JUMP_HOST" ]; then
|
||||
|
|
|
|||
|
|
@ -6,12 +6,14 @@ import (
|
|||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
|
@ -51,6 +53,8 @@ type Router struct {
|
|||
settingsMu sync.RWMutex
|
||||
cachedAllowEmbedding bool
|
||||
cachedAllowedOrigins string
|
||||
publicURLMu sync.Mutex
|
||||
publicURLDetected bool
|
||||
}
|
||||
|
||||
// NewRouter creates a new router instance
|
||||
|
|
@ -1038,6 +1042,13 @@ func (r *Router) SetMonitor(m *monitoring.Monitor) {
|
|||
if r.systemSettingsHandler != nil {
|
||||
r.systemSettingsHandler.SetMonitor(m)
|
||||
}
|
||||
if m != nil {
|
||||
if url := strings.TrimSpace(r.config.PublicURL); url != "" {
|
||||
if mgr := m.GetNotificationManager(); mgr != nil {
|
||||
mgr.SetPublicURL(url)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// reloadSystemSettings loads system settings from disk and caches them
|
||||
|
|
@ -1077,6 +1088,7 @@ func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
|||
}
|
||||
|
||||
// Get cached system settings (loaded once at startup, not from disk every request)
|
||||
r.capturePublicURLFromRequest(req)
|
||||
r.settingsMu.RLock()
|
||||
allowEmbedding := r.cachedAllowEmbedding
|
||||
allowedEmbedOrigins := r.cachedAllowedOrigins
|
||||
|
|
@ -1307,6 +1319,155 @@ func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
|||
}), allowEmbedding, allowedEmbedOrigins).ServeHTTP(w, req)
|
||||
}
|
||||
|
||||
func (r *Router) capturePublicURLFromRequest(req *http.Request) {
|
||||
if req == nil || r == nil || r.config == nil {
|
||||
return
|
||||
}
|
||||
|
||||
if r.config.EnvOverrides != nil && r.config.EnvOverrides["publicURL"] {
|
||||
return
|
||||
}
|
||||
|
||||
rawHost := firstForwardedValue(req.Header.Get("X-Forwarded-Host"))
|
||||
if rawHost == "" {
|
||||
rawHost = req.Host
|
||||
}
|
||||
hostWithPort, hostOnly := sanitizeForwardedHost(rawHost)
|
||||
if hostWithPort == "" {
|
||||
return
|
||||
}
|
||||
if isLoopbackHost(hostOnly) {
|
||||
return
|
||||
}
|
||||
|
||||
rawProto := firstForwardedValue(req.Header.Get("X-Forwarded-Proto"))
|
||||
if rawProto == "" {
|
||||
rawProto = firstForwardedValue(req.Header.Get("X-Forwarded-Scheme"))
|
||||
}
|
||||
scheme := strings.ToLower(strings.TrimSpace(rawProto))
|
||||
switch scheme {
|
||||
case "https", "http":
|
||||
// supported values
|
||||
default:
|
||||
if req.TLS != nil {
|
||||
scheme = "https"
|
||||
} else {
|
||||
scheme = "http"
|
||||
}
|
||||
}
|
||||
if scheme == "" {
|
||||
scheme = "http"
|
||||
}
|
||||
|
||||
if _, _, err := net.SplitHostPort(hostWithPort); err != nil {
|
||||
if forwardedPort := firstForwardedValue(req.Header.Get("X-Forwarded-Port")); forwardedPort != "" {
|
||||
if shouldAppendForwardedPort(forwardedPort, scheme) {
|
||||
if strings.Contains(hostWithPort, ":") && !strings.HasPrefix(hostWithPort, "[") {
|
||||
hostWithPort = fmt.Sprintf("[%s]", hostWithPort)
|
||||
} else if strings.HasPrefix(hostWithPort, "[") && !strings.Contains(hostWithPort, "]") {
|
||||
hostWithPort = fmt.Sprintf("[%s]", strings.TrimPrefix(hostWithPort, "["))
|
||||
}
|
||||
hostWithPort = fmt.Sprintf("%s:%s", hostWithPort, forwardedPort)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
candidate := fmt.Sprintf("%s://%s", scheme, hostWithPort)
|
||||
normalizedCandidate := strings.TrimRight(strings.TrimSpace(candidate), "/")
|
||||
|
||||
r.publicURLMu.Lock()
|
||||
if r.publicURLDetected {
|
||||
r.publicURLMu.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
current := strings.TrimRight(strings.TrimSpace(r.config.PublicURL), "/")
|
||||
if current != "" && current == normalizedCandidate {
|
||||
r.publicURLDetected = true
|
||||
r.publicURLMu.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
r.config.PublicURL = normalizedCandidate
|
||||
r.publicURLDetected = true
|
||||
r.publicURLMu.Unlock()
|
||||
|
||||
log.Info().
|
||||
Str("publicURL", normalizedCandidate).
|
||||
Msg("Detected public URL from inbound request; using for notifications")
|
||||
|
||||
if r.monitor != nil {
|
||||
if mgr := r.monitor.GetNotificationManager(); mgr != nil {
|
||||
mgr.SetPublicURL(normalizedCandidate)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func firstForwardedValue(header string) string {
|
||||
if header == "" {
|
||||
return ""
|
||||
}
|
||||
parts := strings.Split(header, ",")
|
||||
if len(parts) == 0 {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(parts[0])
|
||||
}
|
||||
|
||||
func sanitizeForwardedHost(raw string) (string, string) {
|
||||
host := strings.TrimSpace(raw)
|
||||
if host == "" {
|
||||
return "", ""
|
||||
}
|
||||
|
||||
host = strings.TrimPrefix(host, "http://")
|
||||
host = strings.TrimPrefix(host, "https://")
|
||||
host = strings.TrimSpace(strings.TrimSuffix(host, "/"))
|
||||
if host == "" {
|
||||
return "", ""
|
||||
}
|
||||
|
||||
hostOnly := host
|
||||
if h, _, err := net.SplitHostPort(hostOnly); err == nil {
|
||||
hostOnly = h
|
||||
}
|
||||
hostOnly = strings.Trim(hostOnly, "[]")
|
||||
|
||||
return host, hostOnly
|
||||
}
|
||||
|
||||
func isLoopbackHost(host string) bool {
|
||||
if host == "" {
|
||||
return true
|
||||
}
|
||||
lower := strings.ToLower(host)
|
||||
if lower == "localhost" {
|
||||
return true
|
||||
}
|
||||
if ip := net.ParseIP(host); ip != nil {
|
||||
if ip.IsLoopback() || ip.IsUnspecified() {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func shouldAppendForwardedPort(port, scheme string) bool {
|
||||
if port == "" {
|
||||
return false
|
||||
}
|
||||
if _, err := strconv.Atoi(port); err != nil {
|
||||
return false
|
||||
}
|
||||
if scheme == "https" && port == "443" {
|
||||
return false
|
||||
}
|
||||
if scheme == "http" && port == "80" {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// detectLegacySSH checks if Pulse is using legacy SSH for temperature monitoring
|
||||
//
|
||||
// ⚠️ MIGRATION SCAFFOLDING - TEMPORARY CODE
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ type integrationServer struct {
|
|||
server *httptest.Server
|
||||
monitor *monitoring.Monitor
|
||||
hub *internalws.Hub
|
||||
config *config.Config
|
||||
}
|
||||
|
||||
func newIntegrationServer(t *testing.T) *integrationServer {
|
||||
|
|
@ -50,6 +51,7 @@ func newIntegrationServerWithConfig(t *testing.T, customize func(*config.Config)
|
|||
DemoMode: false,
|
||||
AllowedOrigins: "*",
|
||||
ConcurrentPolling: true,
|
||||
EnvOverrides: make(map[string]bool),
|
||||
}
|
||||
|
||||
if customize != nil {
|
||||
|
|
@ -92,6 +94,7 @@ func newIntegrationServerWithConfig(t *testing.T, customize func(*config.Config)
|
|||
server: srv,
|
||||
monitor: monitor,
|
||||
hub: hub,
|
||||
config: cfg,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -568,6 +571,67 @@ func TestSessionCookieAllowsAuthenticatedAccess(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestPublicURLDetectionUsesForwardedHeaders(t *testing.T) {
|
||||
srv := newIntegrationServer(t)
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, srv.server.URL+"/api/health", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to build request: %v", err)
|
||||
}
|
||||
req.Header.Set("X-Forwarded-Proto", "https")
|
||||
req.Header.Set("X-Forwarded-Host", "pulse.example.com")
|
||||
req.Header.Set("X-Forwarded-Port", "8443")
|
||||
|
||||
res, err := srv.server.Client().Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("health request failed: %v", err)
|
||||
}
|
||||
res.Body.Close()
|
||||
|
||||
expected := "https://pulse.example.com:8443"
|
||||
if got := srv.config.PublicURL; got != expected {
|
||||
t.Fatalf("expected config public URL %q, got %q", expected, got)
|
||||
}
|
||||
|
||||
if mgr := srv.monitor.GetNotificationManager(); mgr != nil {
|
||||
if actual := mgr.GetPublicURL(); actual != expected {
|
||||
t.Fatalf("expected notification manager public URL %q, got %q", expected, actual)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublicURLDetectionRespectsEnvOverride(t *testing.T) {
|
||||
const overrideURL = "https://from-env.example.com"
|
||||
|
||||
srv := newIntegrationServerWithConfig(t, func(cfg *config.Config) {
|
||||
cfg.PublicURL = overrideURL
|
||||
cfg.EnvOverrides["publicURL"] = true
|
||||
})
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, srv.server.URL+"/api/health", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to build request: %v", err)
|
||||
}
|
||||
req.Header.Set("X-Forwarded-Proto", "https")
|
||||
req.Header.Set("X-Forwarded-Host", "ignored.example.org")
|
||||
|
||||
res, err := srv.server.Client().Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("health request failed: %v", err)
|
||||
}
|
||||
res.Body.Close()
|
||||
|
||||
if got := srv.config.PublicURL; got != overrideURL {
|
||||
t.Fatalf("expected config public URL to remain %q, got %q", overrideURL, got)
|
||||
}
|
||||
|
||||
if mgr := srv.monitor.GetNotificationManager(); mgr != nil {
|
||||
if actual := mgr.GetPublicURL(); actual != overrideURL {
|
||||
t.Fatalf("expected notification manager public URL %q, got %q", overrideURL, actual)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func readExpectedVersion(t *testing.T) string {
|
||||
t.Helper()
|
||||
|
||||
|
|
|
|||
|
|
@ -700,6 +700,7 @@ func Load() (*Config, error) {
|
|||
}
|
||||
if publicURL := os.Getenv("PULSE_PUBLIC_URL"); publicURL != "" {
|
||||
cfg.PublicURL = publicURL
|
||||
cfg.EnvOverrides["publicURL"] = true
|
||||
log.Info().Str("url", publicURL).Msg("Public URL configured from PULSE_PUBLIC_URL env var")
|
||||
} else {
|
||||
// Try to auto-detect public URL if not explicitly configured
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -56,8 +56,8 @@ func convertPoolInfoToModel(poolInfo *proxmox.ZFSPoolInfo) *models.ZFSPool {
|
|||
return modelPool
|
||||
}
|
||||
|
||||
// pollVMsWithNodesOptimized polls VMs from all nodes in parallel using goroutines
|
||||
func (m *Monitor) pollVMsWithNodesOptimized(ctx context.Context, instanceName string, client PVEClientInterface, nodes []proxmox.Node) {
|
||||
// pollVMsWithNodes polls VMs from all nodes in parallel using goroutines
|
||||
func (m *Monitor) pollVMsWithNodes(ctx context.Context, instanceName string, client PVEClientInterface, nodes []proxmox.Node) {
|
||||
startTime := time.Now()
|
||||
|
||||
// Channel to collect VM results from each node
|
||||
|
|
@ -655,8 +655,8 @@ func (m *Monitor) pollVMsWithNodesOptimized(ctx context.Context, instanceName st
|
|||
Msg("Parallel VM polling completed")
|
||||
}
|
||||
|
||||
// pollContainersWithNodesOptimized polls containers from all nodes in parallel using goroutines
|
||||
func (m *Monitor) pollContainersWithNodesOptimized(ctx context.Context, instanceName string, client PVEClientInterface, nodes []proxmox.Node) {
|
||||
// pollContainersWithNodes polls containers from all nodes in parallel using goroutines
|
||||
func (m *Monitor) pollContainersWithNodes(ctx context.Context, instanceName string, client PVEClientInterface, nodes []proxmox.Node) {
|
||||
startTime := time.Now()
|
||||
|
||||
// Channel to collect container results from each node
|
||||
|
|
@ -898,8 +898,8 @@ func (m *Monitor) pollContainersWithNodesOptimized(ctx context.Context, instance
|
|||
Msg("Parallel container polling completed")
|
||||
}
|
||||
|
||||
// pollStorageWithNodesOptimized polls storage from all nodes in parallel using goroutines
|
||||
func (m *Monitor) pollStorageWithNodesOptimized(ctx context.Context, instanceName string, client PVEClientInterface, nodes []proxmox.Node) {
|
||||
// pollStorageWithNodes polls storage from all nodes in parallel using goroutines
|
||||
func (m *Monitor) pollStorageWithNodes(ctx context.Context, instanceName string, client PVEClientInterface, nodes []proxmox.Node) {
|
||||
startTime := time.Now()
|
||||
|
||||
// Get cluster storage configuration first (single call)
|
||||
|
|
@ -1254,9 +1254,19 @@ func (m *Monitor) pollStorageWithNodesOptimized(ctx context.Context, instanceNam
|
|||
}
|
||||
}
|
||||
|
||||
// Check alerts for all storage devices
|
||||
// Record metrics and check alerts for all storage devices
|
||||
for _, storage := range allStorage {
|
||||
m.alertManager.CheckStorage(storage)
|
||||
if m.metricsHistory != nil {
|
||||
timestamp := time.Now()
|
||||
m.metricsHistory.AddStorageMetric(storage.ID, "usage", storage.Usage, timestamp)
|
||||
m.metricsHistory.AddStorageMetric(storage.ID, "used", float64(storage.Used), timestamp)
|
||||
m.metricsHistory.AddStorageMetric(storage.ID, "total", float64(storage.Total), timestamp)
|
||||
m.metricsHistory.AddStorageMetric(storage.ID, "avail", float64(storage.Free), timestamp)
|
||||
}
|
||||
|
||||
if m.alertManager != nil {
|
||||
m.alertManager.CheckStorage(storage)
|
||||
}
|
||||
}
|
||||
|
||||
if !cephDetected {
|
||||
201
internal/monitoring/monitor_storage_test.go
Normal file
201
internal/monitoring/monitor_storage_test.go
Normal file
|
|
@ -0,0 +1,201 @@
|
|||
package monitoring
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
||||
"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
|
||||
)
|
||||
|
||||
// fakeStorageClient provides minimal PVE responses needed by the optimized storage poller.
|
||||
type fakeStorageClient struct {
|
||||
allStorage []proxmox.Storage
|
||||
storageByNode map[string][]proxmox.Storage
|
||||
}
|
||||
|
||||
func (f *fakeStorageClient) GetNodes(ctx context.Context) ([]proxmox.Node, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (f *fakeStorageClient) GetNodeStatus(ctx context.Context, node string) (*proxmox.NodeStatus, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (f *fakeStorageClient) GetNodeRRDData(ctx context.Context, node string, timeframe string, cf string, ds []string) ([]proxmox.NodeRRDPoint, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (f *fakeStorageClient) GetVMs(ctx context.Context, node string) ([]proxmox.VM, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (f *fakeStorageClient) GetContainers(ctx context.Context, node string) ([]proxmox.Container, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (f *fakeStorageClient) GetStorage(ctx context.Context, node string) ([]proxmox.Storage, error) {
|
||||
if storages, ok := f.storageByNode[node]; ok {
|
||||
return storages, nil
|
||||
}
|
||||
return nil, fmt.Errorf("unexpected node: %s", node)
|
||||
}
|
||||
|
||||
func (f *fakeStorageClient) GetAllStorage(ctx context.Context) ([]proxmox.Storage, error) {
|
||||
return f.allStorage, nil
|
||||
}
|
||||
|
||||
func (f *fakeStorageClient) GetBackupTasks(ctx context.Context) ([]proxmox.Task, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (f *fakeStorageClient) GetStorageContent(ctx context.Context, node, storage string) ([]proxmox.StorageContent, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (f *fakeStorageClient) GetVMSnapshots(ctx context.Context, node string, vmid int) ([]proxmox.Snapshot, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (f *fakeStorageClient) GetContainerSnapshots(ctx context.Context, node string, vmid int) ([]proxmox.Snapshot, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (f *fakeStorageClient) GetVMStatus(ctx context.Context, node string, vmid int) (*proxmox.VMStatus, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (f *fakeStorageClient) GetContainerStatus(ctx context.Context, node string, vmid int) (*proxmox.Container, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (f *fakeStorageClient) GetClusterResources(ctx context.Context, resourceType string) ([]proxmox.ClusterResource, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (f *fakeStorageClient) IsClusterMember(ctx context.Context) (bool, error) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (f *fakeStorageClient) GetVMFSInfo(ctx context.Context, node string, vmid int) ([]proxmox.VMFileSystem, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (f *fakeStorageClient) GetVMNetworkInterfaces(ctx context.Context, node string, vmid int) ([]proxmox.VMNetworkInterface, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (f *fakeStorageClient) GetVMAgentInfo(ctx context.Context, node string, vmid int) (map[string]interface{}, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (f *fakeStorageClient) GetZFSPoolStatus(ctx context.Context, node string) ([]proxmox.ZFSPoolStatus, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (f *fakeStorageClient) GetZFSPoolsWithDetails(ctx context.Context, node string) ([]proxmox.ZFSPoolInfo, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (f *fakeStorageClient) GetDisks(ctx context.Context, node string) ([]proxmox.Disk, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (f *fakeStorageClient) GetCephStatus(ctx context.Context) (*proxmox.CephStatus, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (f *fakeStorageClient) GetCephDF(ctx context.Context) (*proxmox.CephDF, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func TestPollStorageWithNodesOptimizedRecordsMetricsAndAlerts(t *testing.T) {
|
||||
t.Setenv("PULSE_DATA_DIR", t.TempDir())
|
||||
|
||||
monitor := &Monitor{
|
||||
state: &models.State{},
|
||||
metricsHistory: NewMetricsHistory(16, time.Hour),
|
||||
alertManager: alerts.NewManager(),
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
monitor.alertManager.Stop()
|
||||
})
|
||||
|
||||
// Ensure storage alerts trigger immediately for the test.
|
||||
cfg := monitor.alertManager.GetConfig()
|
||||
cfg.MinimumDelta = 0
|
||||
if cfg.TimeThresholds == nil {
|
||||
cfg.TimeThresholds = make(map[string]int)
|
||||
}
|
||||
cfg.TimeThresholds["storage"] = 0
|
||||
monitor.alertManager.UpdateConfig(cfg)
|
||||
|
||||
storage := proxmox.Storage{
|
||||
Storage: "local",
|
||||
Type: "dir",
|
||||
Content: "images",
|
||||
Active: 1,
|
||||
Enabled: 1,
|
||||
Shared: 0,
|
||||
Total: 1000,
|
||||
Used: 900,
|
||||
Available: 100,
|
||||
}
|
||||
|
||||
client := &fakeStorageClient{
|
||||
allStorage: []proxmox.Storage{storage},
|
||||
storageByNode: map[string][]proxmox.Storage{
|
||||
"node1": {storage},
|
||||
},
|
||||
}
|
||||
|
||||
nodes := []proxmox.Node{
|
||||
{Node: "node1", Status: "online"},
|
||||
}
|
||||
|
||||
monitor.pollStorageWithNodesOptimized(context.Background(), "inst1", client, nodes)
|
||||
|
||||
metrics := monitor.metricsHistory.GetAllStorageMetrics("inst1-node1-local", time.Minute)
|
||||
if len(metrics["usage"]) != 1 {
|
||||
t.Fatalf("expected one usage metric entry, got %d", len(metrics["usage"]))
|
||||
}
|
||||
if len(metrics["used"]) != 1 {
|
||||
t.Fatalf("expected one used metric entry, got %d", len(metrics["used"]))
|
||||
}
|
||||
if len(metrics["total"]) != 1 {
|
||||
t.Fatalf("expected one total metric entry, got %d", len(metrics["total"]))
|
||||
}
|
||||
if len(metrics["avail"]) != 1 {
|
||||
t.Fatalf("expected one avail metric entry, got %d", len(metrics["avail"]))
|
||||
}
|
||||
|
||||
if diff := math.Abs(metrics["usage"][0].Value - 90); diff > 0.001 {
|
||||
t.Fatalf("expected usage metric 90, diff %.4f", diff)
|
||||
}
|
||||
if diff := math.Abs(metrics["used"][0].Value - 900); diff > 0.001 {
|
||||
t.Fatalf("expected used metric 900, diff %.4f", diff)
|
||||
}
|
||||
if diff := math.Abs(metrics["total"][0].Value - 1000); diff > 0.001 {
|
||||
t.Fatalf("expected total metric 1000, diff %.4f", diff)
|
||||
}
|
||||
if diff := math.Abs(metrics["avail"][0].Value - 100); diff > 0.001 {
|
||||
t.Fatalf("expected avail metric 100, diff %.4f", diff)
|
||||
}
|
||||
|
||||
alerts := monitor.alertManager.GetActiveAlerts()
|
||||
found := false
|
||||
for _, alert := range alerts {
|
||||
if alert.ID == "inst1-node1-local-usage" {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("expected storage usage alert to be active")
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -3,24 +3,43 @@ package monitoring
|
|||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/ssh/knownhosts"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/tempproxy"
|
||||
"github.com/rs/zerolog/log"
|
||||
)
|
||||
|
||||
const (
|
||||
proxyFailureThreshold = 3
|
||||
proxyRetryInterval = 5 * time.Minute
|
||||
)
|
||||
|
||||
type temperatureProxy interface {
|
||||
IsAvailable() bool
|
||||
GetTemperature(nodeHost string) (string, error)
|
||||
}
|
||||
|
||||
// TemperatureCollector handles SSH-based temperature collection from Proxmox nodes
|
||||
type TemperatureCollector struct {
|
||||
sshUser string // SSH user (typically "root" or "pulse-monitor")
|
||||
sshKeyPath string // Path to SSH private key
|
||||
proxyClient *tempproxy.Client // Optional: unix socket client for proxy
|
||||
useProxy bool // Whether to use proxy for temperature collection
|
||||
sshUser string // SSH user (typically "root" or "pulse-monitor")
|
||||
sshKeyPath string // Path to SSH private key
|
||||
proxyClient temperatureProxy // Optional: unix socket client for proxy
|
||||
useProxy bool // Whether to use proxy for temperature collection
|
||||
hostKeys knownhosts.Manager
|
||||
proxyMu sync.Mutex
|
||||
proxyFailures int
|
||||
proxyCooldownUntil time.Time
|
||||
}
|
||||
|
||||
// NewTemperatureCollector creates a new temperature collector
|
||||
|
|
@ -30,6 +49,17 @@ func NewTemperatureCollector(sshUser, sshKeyPath string) *TemperatureCollector {
|
|||
sshKeyPath: sshKeyPath,
|
||||
}
|
||||
|
||||
homeDir := os.Getenv("HOME")
|
||||
if homeDir == "" {
|
||||
homeDir = "/home/pulse"
|
||||
}
|
||||
knownHostsPath := filepath.Join(homeDir, ".ssh", "known_hosts_sensors")
|
||||
if manager, err := knownhosts.NewManager(knownHostsPath); err != nil {
|
||||
log.Warn().Err(err).Str("path", knownHostsPath).Msg("Failed to initialize temperature known_hosts manager")
|
||||
} else {
|
||||
tc.hostKeys = manager
|
||||
}
|
||||
|
||||
// Check if proxy is available
|
||||
proxyClient := tempproxy.NewClient()
|
||||
if proxyClient.IsAvailable() {
|
||||
|
|
@ -53,9 +83,10 @@ func (tc *TemperatureCollector) CollectTemperature(ctx context.Context, nodeHost
|
|||
var err error
|
||||
|
||||
// Use proxy if available, otherwise fall back to direct SSH
|
||||
if tc.useProxy && tc.proxyClient != nil {
|
||||
if tc.isProxyEnabled() {
|
||||
output, err = tc.proxyClient.GetTemperature(host)
|
||||
if err != nil {
|
||||
tc.handleProxyFailure(err)
|
||||
log.Debug().
|
||||
Str("node", nodeName).
|
||||
Str("host", host).
|
||||
|
|
@ -63,8 +94,19 @@ func (tc *TemperatureCollector) CollectTemperature(ctx context.Context, nodeHost
|
|||
Msg("Failed to collect temperature data via proxy")
|
||||
return &models.Temperature{Available: false}, nil
|
||||
}
|
||||
tc.handleProxySuccess()
|
||||
} else {
|
||||
// Direct SSH (legacy method)
|
||||
// SECURITY: Block SSH fallback when running in containers (unless dev mode)
|
||||
// Container compromise = SSH key compromise = root access to infrastructure
|
||||
devModeAllowSSH := os.Getenv("PULSE_DEV_ALLOW_CONTAINER_SSH") == "true"
|
||||
if isRunningInContainer() && !devModeAllowSSH {
|
||||
log.Error().
|
||||
Str("node", nodeName).
|
||||
Msg("SECURITY BLOCK: SSH temperature collection disabled in containers - deploy pulse-sensor-proxy")
|
||||
return &models.Temperature{Available: false}, nil
|
||||
}
|
||||
|
||||
// Direct SSH (legacy method for non-containerized deployments)
|
||||
// Try sensors first, fall back to Raspberry Pi method if that fails
|
||||
output, err = tc.runSSHCommand(ctx, host, "sensors -j 2>/dev/null")
|
||||
if err != nil || strings.TrimSpace(output) == "" {
|
||||
|
|
@ -108,13 +150,33 @@ func (tc *TemperatureCollector) CollectTemperature(ctx context.Context, nodeHost
|
|||
|
||||
// runSSHCommand executes a command on a remote node via SSH
|
||||
func (tc *TemperatureCollector) runSSHCommand(ctx context.Context, host, command string) (string, error) {
|
||||
if err := tc.ensureHostKey(ctx, host); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Build SSH command with appropriate options
|
||||
sshArgs := []string{
|
||||
"-o", "StrictHostKeyChecking=no",
|
||||
"-o", "UserKnownHostsFile=/dev/null",
|
||||
"-o", "ConnectTimeout=5",
|
||||
"-o", "BatchMode=yes", // No password prompts
|
||||
"-o", "StrictHostKeyChecking=yes",
|
||||
"-o", "BatchMode=yes",
|
||||
"-o", "LogLevel=ERROR", // Suppress host key warnings that break JSON parsing
|
||||
"-o", "ConnectTimeout=5",
|
||||
}
|
||||
|
||||
if tc.hostKeys != nil && tc.hostKeys.Path() != "" {
|
||||
sshArgs = append(sshArgs,
|
||||
"-o", fmt.Sprintf("UserKnownHostsFile=%s", tc.hostKeys.Path()),
|
||||
"-o", "GlobalKnownHostsFile=/dev/null",
|
||||
)
|
||||
}
|
||||
|
||||
// Explicitly use SSH config file if it exists (for ProxyJump configuration)
|
||||
homeDir := os.Getenv("HOME")
|
||||
if homeDir == "" {
|
||||
homeDir = "/home/pulse"
|
||||
}
|
||||
sshConfigPath := filepath.Join(homeDir, ".ssh/config")
|
||||
if _, err := os.Stat(sshConfigPath); err == nil {
|
||||
sshArgs = append(sshArgs, "-F", sshConfigPath)
|
||||
}
|
||||
|
||||
// Add key if specified
|
||||
|
|
@ -193,7 +255,8 @@ func (tc *TemperatureCollector) parseSensorsJSON(jsonStr string) (*models.Temper
|
|||
strings.Contains(chipLower, "k8temp") ||
|
||||
strings.Contains(chipLower, "acpitz") ||
|
||||
strings.Contains(chipLower, "it87") ||
|
||||
strings.Contains(chipLower, "cpu_thermal") { // Raspberry Pi CPU temperature
|
||||
strings.Contains(chipLower, "cpu_thermal") || // Raspberry Pi CPU temperature
|
||||
strings.Contains(chipLower, "rpitemp") {
|
||||
foundCPUChip = true
|
||||
tc.parseCPUTemps(chipMap, temp)
|
||||
}
|
||||
|
|
@ -268,8 +331,12 @@ func (tc *TemperatureCollector) parseCPUTemps(chipMap map[string]interface{}, te
|
|||
// Look for generic temperature sensors (e.g., "temp1" on Raspberry Pi)
|
||||
if strings.HasPrefix(sensorName, "temp") || strings.HasPrefix(sensorName, "Temp") {
|
||||
if tempVal := extractTempInput(sensorMap); !math.IsNaN(tempVal) && tempVal > 0 {
|
||||
temp.CPUPackage = tempVal
|
||||
temp.CPUMax = tempVal
|
||||
if temp.CPUPackage <= 0 {
|
||||
temp.CPUPackage = tempVal
|
||||
}
|
||||
if tempVal > temp.CPUMax {
|
||||
temp.CPUMax = tempVal
|
||||
}
|
||||
break // Use the first valid generic temp sensor
|
||||
}
|
||||
}
|
||||
|
|
@ -379,3 +446,88 @@ func extractHostname(hostURL string) string {
|
|||
|
||||
return host
|
||||
}
|
||||
|
||||
func (tc *TemperatureCollector) ensureHostKey(ctx context.Context, host string) error {
|
||||
if tc.hostKeys == nil {
|
||||
return nil
|
||||
}
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
return tc.hostKeys.Ensure(ctx, host)
|
||||
}
|
||||
|
||||
func (tc *TemperatureCollector) isProxyEnabled() bool {
|
||||
if tc.proxyClient == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
tc.proxyMu.Lock()
|
||||
restored := false
|
||||
if !tc.useProxy {
|
||||
now := time.Now()
|
||||
if now.After(tc.proxyCooldownUntil) {
|
||||
if tc.proxyClient.IsAvailable() {
|
||||
tc.useProxy = true
|
||||
tc.proxyFailures = 0
|
||||
tc.proxyCooldownUntil = time.Time{}
|
||||
restored = true
|
||||
} else {
|
||||
tc.proxyCooldownUntil = now.Add(proxyRetryInterval)
|
||||
}
|
||||
}
|
||||
}
|
||||
useProxy := tc.useProxy
|
||||
tc.proxyMu.Unlock()
|
||||
|
||||
if restored {
|
||||
log.Info().Msg("Temperature proxy connection restored; resuming proxy collection")
|
||||
}
|
||||
|
||||
return useProxy
|
||||
}
|
||||
|
||||
func (tc *TemperatureCollector) handleProxySuccess() {
|
||||
if tc.proxyClient == nil {
|
||||
return
|
||||
}
|
||||
tc.proxyMu.Lock()
|
||||
tc.proxyFailures = 0
|
||||
tc.proxyMu.Unlock()
|
||||
}
|
||||
|
||||
func (tc *TemperatureCollector) handleProxyFailure(err error) {
|
||||
if tc.proxyClient == nil || !tc.shouldDisableProxy(err) {
|
||||
return
|
||||
}
|
||||
|
||||
tc.proxyMu.Lock()
|
||||
tc.proxyFailures++
|
||||
disable := tc.proxyFailures >= proxyFailureThreshold && tc.useProxy
|
||||
if disable {
|
||||
tc.useProxy = false
|
||||
tc.proxyCooldownUntil = time.Now().Add(proxyRetryInterval)
|
||||
tc.proxyFailures = 0
|
||||
}
|
||||
tc.proxyMu.Unlock()
|
||||
|
||||
if disable {
|
||||
log.Warn().
|
||||
Err(err).
|
||||
Dur("cooldown", proxyRetryInterval).
|
||||
Msg("Temperature proxy disabled after repeated failures; will retry later")
|
||||
}
|
||||
}
|
||||
|
||||
func (tc *TemperatureCollector) shouldDisableProxy(err error) bool {
|
||||
var proxyErr *tempproxy.ProxyError
|
||||
if errors.As(err, &proxyErr) {
|
||||
switch proxyErr.Type {
|
||||
case tempproxy.ErrorTypeTransport, tempproxy.ErrorTypeTimeout, tempproxy.ErrorTypeSSH:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,59 @@
|
|||
package monitoring
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/tempproxy"
|
||||
)
|
||||
|
||||
type stubProxyResponse struct {
|
||||
output string
|
||||
err error
|
||||
}
|
||||
|
||||
type stubTemperatureProxy struct {
|
||||
mu sync.Mutex
|
||||
available bool
|
||||
responses []stubProxyResponse
|
||||
responseFunc func(call int) stubProxyResponse
|
||||
callCount int
|
||||
}
|
||||
|
||||
func (s *stubTemperatureProxy) IsAvailable() bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.available
|
||||
}
|
||||
|
||||
func (s *stubTemperatureProxy) GetTemperature(host string) (string, error) {
|
||||
s.mu.Lock()
|
||||
call := s.callCount
|
||||
s.callCount++
|
||||
|
||||
resp := stubProxyResponse{}
|
||||
switch {
|
||||
case call < len(s.responses):
|
||||
resp = s.responses[call]
|
||||
case s.responseFunc != nil:
|
||||
resp = s.responseFunc(call)
|
||||
case len(s.responses) > 0:
|
||||
resp = s.responses[len(s.responses)-1]
|
||||
}
|
||||
s.mu.Unlock()
|
||||
|
||||
return resp.output, resp.err
|
||||
}
|
||||
|
||||
func (s *stubTemperatureProxy) setAvailable(v bool) {
|
||||
s.mu.Lock()
|
||||
s.available = v
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
func TestParseSensorsJSON_NoTemperatureData(t *testing.T) {
|
||||
collector := &TemperatureCollector{}
|
||||
|
|
@ -80,6 +133,47 @@ func TestParseSensorsJSON_WithCpuAndNvmeData(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestParseSensorsJSON_RPiWrapper(t *testing.T) {
|
||||
collector := &TemperatureCollector{}
|
||||
|
||||
jsonStr := `{"rpitemp-virtual":{"temp1":{"temp1_input":47.5}}}`
|
||||
|
||||
temp, err := collector.parseSensorsJSON(jsonStr)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error parsing wrapper output: %v", err)
|
||||
}
|
||||
if temp == nil {
|
||||
t.Fatalf("expected temperature struct, got nil")
|
||||
}
|
||||
if !temp.HasCPU {
|
||||
t.Fatalf("expected HasCPU to be true for wrapper output")
|
||||
}
|
||||
if temp.CPUPackage != 47.5 {
|
||||
t.Fatalf("expected cpu package temperature 47.5, got %.2f", temp.CPUPackage)
|
||||
}
|
||||
if !temp.Available {
|
||||
t.Fatalf("expected temperature to be available for wrapper output")
|
||||
}
|
||||
}
|
||||
|
||||
func TestShouldDisableProxy(t *testing.T) {
|
||||
collector := &TemperatureCollector{}
|
||||
|
||||
if !collector.shouldDisableProxy(fmt.Errorf("plain")) {
|
||||
t.Fatalf("expected plain errors to disable proxy")
|
||||
}
|
||||
|
||||
transportErr := &tempproxy.ProxyError{Type: tempproxy.ErrorTypeTransport}
|
||||
if !collector.shouldDisableProxy(transportErr) {
|
||||
t.Fatalf("expected transport errors to disable proxy")
|
||||
}
|
||||
|
||||
sensorErr := &tempproxy.ProxyError{Type: tempproxy.ErrorTypeSensor}
|
||||
if collector.shouldDisableProxy(sensorErr) {
|
||||
t.Fatalf("sensor errors should not disable proxy")
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseSensorsJSON_NVMeOnly tests that NVMe-only systems don't show "No CPU sensor"
|
||||
func TestParseSensorsJSON_NVMeOnly(t *testing.T) {
|
||||
collector := &TemperatureCollector{}
|
||||
|
|
@ -166,3 +260,274 @@ func TestParseSensorsJSON_ZeroTemperature(t *testing.T) {
|
|||
t.Fatalf("expected core temperature to be 0.0, got %.2f", temp.Cores[0].Temp)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseRPiTemperature(t *testing.T) {
|
||||
collector := &TemperatureCollector{}
|
||||
temp, err := collector.parseRPiTemperature("48720\n")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error parsing RPi thermal zone output: %v", err)
|
||||
}
|
||||
if !temp.Available {
|
||||
t.Fatalf("expected temperature to be marked available")
|
||||
}
|
||||
if !temp.HasCPU {
|
||||
t.Fatalf("expected HasCPU to be true for RPi thermal zone output")
|
||||
}
|
||||
expected := 48.72
|
||||
if diff := temp.CPUPackage - expected; diff > 1e-6 || diff < -1e-6 {
|
||||
t.Fatalf("expected cpu package temperature %.2f, got %.2f", expected, temp.CPUPackage)
|
||||
}
|
||||
if temp.CPUMax != temp.CPUPackage {
|
||||
t.Fatalf("expected cpu max to match package temperature %.2f, got %.2f", temp.CPUPackage, temp.CPUMax)
|
||||
}
|
||||
if temp.LastUpdate.IsZero() {
|
||||
t.Fatalf("expected LastUpdate to be set")
|
||||
}
|
||||
if elapsed := time.Since(temp.LastUpdate); elapsed > 2*time.Second {
|
||||
t.Fatalf("expected LastUpdate to be recent, got %s", elapsed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseSensorsJSON_PiPartialSensors(t *testing.T) {
|
||||
collector := &TemperatureCollector{}
|
||||
|
||||
jsonStr := `{
|
||||
"cpu_thermal-virtual-0": {
|
||||
"Adapter": "Virtual device",
|
||||
"temp1": {"temp1_input": 51.625}
|
||||
}
|
||||
}`
|
||||
|
||||
temp, err := collector.parseSensorsJSON(jsonStr)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error parsing Pi sensors output: %v", err)
|
||||
}
|
||||
if !temp.Available {
|
||||
t.Fatalf("expected temperature to be available when cpu_thermal sensor present")
|
||||
}
|
||||
if !temp.HasCPU {
|
||||
t.Fatalf("expected HasCPU to be true when cpu_thermal sensor present")
|
||||
}
|
||||
if temp.CPUPackage != 51.625 {
|
||||
t.Fatalf("expected cpu package temperature 51.625, got %.3f", temp.CPUPackage)
|
||||
}
|
||||
if temp.CPUMax != 51.625 {
|
||||
t.Fatalf("expected cpu max temperature 51.625, got %.3f", temp.CPUMax)
|
||||
}
|
||||
if len(temp.Cores) != 0 {
|
||||
t.Fatalf("expected no per-core temperatures, got %d entries", len(temp.Cores))
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseSensorsJSON_CoretempAndRPiFallback(t *testing.T) {
|
||||
collector := &TemperatureCollector{}
|
||||
|
||||
jsonStr := `{
|
||||
"coretemp-isa-0000": {
|
||||
"Package id 0": {"temp1_input": 65.0},
|
||||
"Core 0": {"temp2_input": 63.0},
|
||||
"Core 1": {"temp3_input": 62.5}
|
||||
},
|
||||
"cpu_thermal-virtual-0": {
|
||||
"temp1": {"temp1_input": 50.0}
|
||||
}
|
||||
}`
|
||||
|
||||
temp, err := collector.parseSensorsJSON(jsonStr)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error parsing mixed sensors output: %v", err)
|
||||
}
|
||||
if temp.CPUPackage != 65.0 {
|
||||
t.Fatalf("expected cpu package temperature 65.0 from coretemp, got %.2f", temp.CPUPackage)
|
||||
}
|
||||
if temp.CPUMax < 63.0 {
|
||||
t.Fatalf("expected cpu max to reflect hottest core (>=63.0), got %.2f", temp.CPUMax)
|
||||
}
|
||||
if !temp.HasCPU {
|
||||
t.Fatalf("expected HasCPU to be true when CPU sensors present")
|
||||
}
|
||||
if !temp.Available {
|
||||
t.Fatalf("expected temperature to be available when CPU sensors present")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTemperatureCollector_DisablesProxyAfterFailures(t *testing.T) {
|
||||
stub := &stubTemperatureProxy{
|
||||
responses: []stubProxyResponse{
|
||||
{err: &tempproxy.ProxyError{Type: tempproxy.ErrorTypeTransport, Message: "transport failure 1"}},
|
||||
{err: &tempproxy.ProxyError{Type: tempproxy.ErrorTypeTransport, Message: "transport failure 2"}},
|
||||
{err: &tempproxy.ProxyError{Type: tempproxy.ErrorTypeTransport, Message: "transport failure 3"}},
|
||||
},
|
||||
}
|
||||
stub.setAvailable(true)
|
||||
|
||||
collector := &TemperatureCollector{
|
||||
proxyClient: stub,
|
||||
useProxy: true,
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
for i := 0; i < proxyFailureThreshold; i++ {
|
||||
temp, err := collector.CollectTemperature(ctx, "https://node.example", "node")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error on proxy failure %d: %v", i+1, err)
|
||||
}
|
||||
if temp.Available {
|
||||
t.Fatalf("expected temperature to be unavailable after proxy failure %d", i+1)
|
||||
}
|
||||
}
|
||||
|
||||
if collector.useProxy {
|
||||
t.Fatalf("expected proxy to be disabled after %d failures", proxyFailureThreshold)
|
||||
}
|
||||
if collector.proxyFailures != 0 {
|
||||
t.Fatalf("expected proxy failure counter to reset after disable, got %d", collector.proxyFailures)
|
||||
}
|
||||
if collector.proxyCooldownUntil.IsZero() {
|
||||
t.Fatalf("expected proxy cooldown to be scheduled after disable")
|
||||
}
|
||||
if time.Until(collector.proxyCooldownUntil) <= 0 {
|
||||
t.Fatalf("expected proxy cooldown to be in the future, got %s", collector.proxyCooldownUntil)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTemperatureCollector_ProxyReenablesAfterCooldown(t *testing.T) {
|
||||
stub := &stubTemperatureProxy{}
|
||||
stub.setAvailable(true)
|
||||
|
||||
collector := &TemperatureCollector{
|
||||
proxyClient: stub,
|
||||
useProxy: false,
|
||||
proxyCooldownUntil: time.Now().Add(-time.Minute),
|
||||
}
|
||||
|
||||
if !collector.isProxyEnabled() {
|
||||
t.Fatalf("expected proxy to re-enable when available after cooldown")
|
||||
}
|
||||
if !collector.useProxy {
|
||||
t.Fatalf("expected useProxy to be true after proxy restored")
|
||||
}
|
||||
if !collector.proxyCooldownUntil.IsZero() {
|
||||
t.Fatalf("expected cooldown to reset after proxy restoration, got %s", collector.proxyCooldownUntil)
|
||||
}
|
||||
if collector.proxyFailures != 0 {
|
||||
t.Fatalf("expected proxy failure counter to reset after restoration, got %d", collector.proxyFailures)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTemperatureCollector_ProxyCooldownExtendsWhenUnavailable(t *testing.T) {
|
||||
stub := &stubTemperatureProxy{}
|
||||
stub.setAvailable(false)
|
||||
|
||||
collector := &TemperatureCollector{
|
||||
proxyClient: stub,
|
||||
useProxy: false,
|
||||
proxyCooldownUntil: time.Now().Add(-time.Minute),
|
||||
}
|
||||
|
||||
before := time.Now()
|
||||
if collector.isProxyEnabled() {
|
||||
t.Fatalf("expected proxy to remain disabled while unavailable")
|
||||
}
|
||||
if collector.useProxy {
|
||||
t.Fatalf("expected useProxy to remain false while proxy unavailable")
|
||||
}
|
||||
if !collector.proxyCooldownUntil.After(before) {
|
||||
t.Fatalf("expected cooldown to be pushed into the future, got %s", collector.proxyCooldownUntil)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTemperatureCollector_SuccessResetsFailureCount(t *testing.T) {
|
||||
successJSON := `{"coretemp-isa-0000":{"Package id 0":{"temp1_input": 45.0}}}`
|
||||
stub := &stubTemperatureProxy{
|
||||
responses: []stubProxyResponse{
|
||||
{err: &tempproxy.ProxyError{Type: tempproxy.ErrorTypeTransport, Message: "transient failure"}},
|
||||
{output: successJSON},
|
||||
},
|
||||
}
|
||||
stub.setAvailable(true)
|
||||
|
||||
collector := &TemperatureCollector{
|
||||
proxyClient: stub,
|
||||
useProxy: true,
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
if temp, err := collector.CollectTemperature(ctx, "https://node.example", "node"); err != nil {
|
||||
t.Fatalf("unexpected error during proxy failure: %v", err)
|
||||
} else if temp.Available {
|
||||
t.Fatalf("expected unavailable temperature on proxy failure")
|
||||
}
|
||||
if collector.proxyFailures != 1 {
|
||||
t.Fatalf("expected proxy failure counter to increment to 1, got %d", collector.proxyFailures)
|
||||
}
|
||||
|
||||
temp, err := collector.CollectTemperature(ctx, "https://node.example", "node")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error on proxy success: %v", err)
|
||||
}
|
||||
if temp == nil || !temp.Available {
|
||||
t.Fatalf("expected valid temperature after proxy success")
|
||||
}
|
||||
if collector.proxyFailures != 0 {
|
||||
t.Fatalf("expected proxy failure counter reset after success, got %d", collector.proxyFailures)
|
||||
}
|
||||
if !collector.useProxy {
|
||||
t.Fatalf("expected proxy to remain enabled after success")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTemperatureCollector_ConcurrentCollectTemperature(t *testing.T) {
|
||||
successJSON := `{"coretemp-isa-0000":{"Package id 0":{"temp1_input": 55.0}}}`
|
||||
var callCounter int32
|
||||
stub := &stubTemperatureProxy{
|
||||
responseFunc: func(int) stubProxyResponse {
|
||||
n := atomic.AddInt32(&callCounter, 1)
|
||||
if n%2 == 1 {
|
||||
return stubProxyResponse{
|
||||
err: &tempproxy.ProxyError{Type: tempproxy.ErrorTypeTransport, Message: "transient transport error"},
|
||||
}
|
||||
}
|
||||
return stubProxyResponse{output: successJSON}
|
||||
},
|
||||
}
|
||||
stub.setAvailable(true)
|
||||
|
||||
collector := &TemperatureCollector{
|
||||
proxyClient: stub,
|
||||
useProxy: true,
|
||||
}
|
||||
|
||||
const goroutines = 16
|
||||
const iterations = 32
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(goroutines)
|
||||
|
||||
ctx := context.Background()
|
||||
for i := 0; i < goroutines; i++ {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for j := 0; j < iterations; j++ {
|
||||
temp, err := collector.CollectTemperature(ctx, "https://node.example", "node")
|
||||
if err != nil {
|
||||
t.Errorf("collect temperature returned error: %v", err)
|
||||
return
|
||||
}
|
||||
if temp == nil {
|
||||
t.Errorf("expected non-nil temperature result")
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
if !collector.useProxy {
|
||||
t.Fatalf("expected proxy to remain enabled during concurrent collection")
|
||||
}
|
||||
if collector.proxyFailures >= proxyFailureThreshold {
|
||||
t.Fatalf("expected proxy failures to stay below disable threshold, got %d", collector.proxyFailures)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -268,8 +268,9 @@ type AppriseConfig struct {
|
|||
|
||||
// NewNotificationManager creates a new notification manager
|
||||
func NewNotificationManager(publicURL string) *NotificationManager {
|
||||
if publicURL != "" {
|
||||
log.Info().Str("publicURL", publicURL).Msg("NotificationManager initialized with public URL")
|
||||
cleanURL := strings.TrimRight(strings.TrimSpace(publicURL), "/")
|
||||
if cleanURL != "" {
|
||||
log.Info().Str("publicURL", cleanURL).Msg("NotificationManager initialized with public URL")
|
||||
} else {
|
||||
log.Info().Msg("NotificationManager initialized without public URL - webhook links may not work")
|
||||
}
|
||||
|
|
@ -290,11 +291,36 @@ func NewNotificationManager(publicURL string) *NotificationManager {
|
|||
groupByGuest: false,
|
||||
webhookHistory: make([]WebhookDelivery, 0, WebhookHistoryMaxSize),
|
||||
webhookRateLimits: make(map[string]*webhookRateLimit),
|
||||
publicURL: publicURL,
|
||||
publicURL: cleanURL,
|
||||
appriseExec: defaultAppriseExec,
|
||||
}
|
||||
}
|
||||
|
||||
// SetPublicURL updates the public URL used for webhook payloads.
|
||||
func (n *NotificationManager) SetPublicURL(publicURL string) {
|
||||
trimmed := strings.TrimRight(strings.TrimSpace(publicURL), "/")
|
||||
if trimmed == "" {
|
||||
return
|
||||
}
|
||||
|
||||
n.mu.Lock()
|
||||
if n.publicURL == trimmed {
|
||||
n.mu.Unlock()
|
||||
return
|
||||
}
|
||||
n.publicURL = trimmed
|
||||
n.mu.Unlock()
|
||||
|
||||
log.Info().Str("publicURL", trimmed).Msg("NotificationManager public URL updated")
|
||||
}
|
||||
|
||||
// GetPublicURL returns the configured public URL for notifications.
|
||||
func (n *NotificationManager) GetPublicURL() string {
|
||||
n.mu.RLock()
|
||||
defer n.mu.RUnlock()
|
||||
return n.publicURL
|
||||
}
|
||||
|
||||
// SetEmailConfig updates email configuration
|
||||
func (n *NotificationManager) SetEmailConfig(config EmailConfig) {
|
||||
n.mu.Lock()
|
||||
|
|
|
|||
279
internal/ssh/knownhosts/manager.go
Normal file
279
internal/ssh/knownhosts/manager.go
Normal file
|
|
@ -0,0 +1,279 @@
|
|||
package knownhosts
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Manager exposes operations for ensuring SSH host keys exist locally.
|
||||
type Manager interface {
|
||||
// Ensure guarantees that the host key for the provided host exists in the
|
||||
// managed known_hosts file.
|
||||
Ensure(ctx context.Context, host string) error
|
||||
// Path returns the absolute path to the managed known_hosts file.
|
||||
Path() string
|
||||
}
|
||||
|
||||
type manager struct {
|
||||
path string
|
||||
cache map[string]struct{}
|
||||
mu sync.Mutex
|
||||
keyscanFn keyscanFunc
|
||||
keyscanTimeout time.Duration
|
||||
}
|
||||
|
||||
type keyscanFunc func(ctx context.Context, host string, timeout time.Duration) ([]byte, error)
|
||||
|
||||
const (
|
||||
defaultKeyscanTimeout = 5 * time.Second
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrNoHostKeys is returned when ssh-keyscan yields no usable entries.
|
||||
ErrNoHostKeys = errors.New("knownhosts: no host keys discovered")
|
||||
)
|
||||
|
||||
// Option allows customizing Manager construction.
|
||||
type Option func(*manager)
|
||||
|
||||
// WithTimeout overrides the ssh-keyscan timeout (defaults to 5 seconds).
|
||||
func WithTimeout(d time.Duration) Option {
|
||||
return func(m *manager) {
|
||||
if d > 0 {
|
||||
m.keyscanTimeout = d
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// WithKeyscanFunc overrides the function used to fetch host keys (mainly for tests).
|
||||
func WithKeyscanFunc(fn keyscanFunc) Option {
|
||||
return func(m *manager) {
|
||||
if fn != nil {
|
||||
m.keyscanFn = fn
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// NewManager returns a Manager writing to the supplied known_hosts path.
|
||||
func NewManager(path string, opts ...Option) (Manager, error) {
|
||||
if strings.TrimSpace(path) == "" {
|
||||
return nil, fmt.Errorf("knownhosts: empty path")
|
||||
}
|
||||
|
||||
m := &manager{
|
||||
path: path,
|
||||
cache: make(map[string]struct{}),
|
||||
keyscanFn: defaultKeyscan,
|
||||
keyscanTimeout: defaultKeyscanTimeout,
|
||||
}
|
||||
|
||||
for _, opt := range opts {
|
||||
opt(m)
|
||||
}
|
||||
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// Ensure implements Manager.Ensure.
|
||||
func (m *manager) Ensure(ctx context.Context, host string) error {
|
||||
if strings.TrimSpace(host) == "" {
|
||||
return fmt.Errorf("knownhosts: missing host")
|
||||
}
|
||||
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
if _, ok := m.cache[host]; ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := m.ensureKnownHostsFile(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
exists, err := hostKeyExists(m.path, host)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if exists {
|
||||
m.cache[host] = struct{}{}
|
||||
return nil
|
||||
}
|
||||
|
||||
keyData, err := m.keyscanFn(ctx, host, m.keyscanTimeout)
|
||||
if err != nil {
|
||||
return fmt.Errorf("knownhosts: ssh-keyscan failed for %s: %w", host, err)
|
||||
}
|
||||
|
||||
entries := sanitizeKeyscanOutput(host, keyData)
|
||||
if len(entries) == 0 {
|
||||
return fmt.Errorf("%w for %s", ErrNoHostKeys, host)
|
||||
}
|
||||
|
||||
if err := appendHostKey(m.path, entries); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
m.cache[host] = struct{}{}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Path implements Manager.Path.
|
||||
func (m *manager) Path() string {
|
||||
return m.path
|
||||
}
|
||||
|
||||
func (m *manager) ensureKnownHostsFile() error {
|
||||
dir := filepath.Dir(m.path)
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return fmt.Errorf("knownhosts: mkdir %s: %w", dir, err)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(m.path); err == nil {
|
||||
return nil
|
||||
} else if !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
|
||||
f, err := os.OpenFile(m.path, os.O_CREATE|os.O_WRONLY, 0o600)
|
||||
if err != nil {
|
||||
return fmt.Errorf("knownhosts: create %s: %w", m.path, err)
|
||||
}
|
||||
return f.Close()
|
||||
}
|
||||
|
||||
func hostKeyExists(path, host string) (bool, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
scanner := bufio.NewScanner(f)
|
||||
for scanner.Scan() {
|
||||
if hostLineMatches(host, scanner.Text()) {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
return false, scanner.Err()
|
||||
}
|
||||
|
||||
func appendHostKey(path string, entries [][]byte) error {
|
||||
f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0o600)
|
||||
if err != nil {
|
||||
return fmt.Errorf("knownhosts: open %s: %w", path, err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
for _, entry := range entries {
|
||||
if len(entry) == 0 {
|
||||
continue
|
||||
}
|
||||
if _, err := f.Write(append(entry, '\n')); err != nil {
|
||||
return fmt.Errorf("knownhosts: write entry: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sanitizeKeyscanOutput(host string, raw []byte) [][]byte {
|
||||
var entries [][]byte
|
||||
|
||||
lines := bytes.Split(raw, []byte{'\n'})
|
||||
for _, line := range lines {
|
||||
line = bytes.TrimSpace(line)
|
||||
if len(line) == 0 {
|
||||
continue
|
||||
}
|
||||
if hostLineMatches(host, string(line)) {
|
||||
entries = append(entries, line)
|
||||
}
|
||||
}
|
||||
|
||||
return entries
|
||||
}
|
||||
|
||||
func hostLineMatches(host, line string) bool {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if trimmed == "" || strings.HasPrefix(trimmed, "#") {
|
||||
return false
|
||||
}
|
||||
if strings.HasPrefix(trimmed, "|") {
|
||||
return false // hashed entry; we only manage clear-text hosts
|
||||
}
|
||||
|
||||
fields := strings.Fields(trimmed)
|
||||
if len(fields) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
return hostFieldMatches(host, fields[0])
|
||||
}
|
||||
|
||||
func hostFieldMatches(host, field string) bool {
|
||||
for _, part := range strings.Split(field, ",") {
|
||||
for _, candidate := range hostCandidates(part) {
|
||||
if strings.EqualFold(candidate, host) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func hostCandidates(part string) []string {
|
||||
part = strings.TrimSpace(part)
|
||||
if part == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
if strings.HasPrefix(part, "[") {
|
||||
if idx := strings.Index(part, "]"); idx != -1 {
|
||||
host := part[1:idx]
|
||||
candidates := []string{part}
|
||||
if host != "" {
|
||||
candidates = append(candidates, host)
|
||||
}
|
||||
return candidates
|
||||
}
|
||||
}
|
||||
|
||||
candidates := []string{part}
|
||||
if strings.Count(part, ":") == 1 {
|
||||
if idx := strings.Index(part, ":"); idx > 0 {
|
||||
candidates = append(candidates, part[:idx])
|
||||
}
|
||||
}
|
||||
|
||||
return candidates
|
||||
}
|
||||
|
||||
func defaultKeyscan(ctx context.Context, host string, timeout time.Duration) ([]byte, error) {
|
||||
seconds := int(timeout.Round(time.Second) / time.Second)
|
||||
if seconds <= 0 {
|
||||
seconds = int(defaultKeyscanTimeout / time.Second)
|
||||
}
|
||||
|
||||
scanCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
|
||||
cmd := exec.CommandContext(scanCtx, "ssh-keyscan", "-T", strconv.Itoa(seconds), host)
|
||||
output, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w (output: %s)", err, strings.TrimSpace(string(output)))
|
||||
}
|
||||
return output, nil
|
||||
}
|
||||
133
internal/ssh/knownhosts/manager_test.go
Normal file
133
internal/ssh/knownhosts/manager_test.go
Normal file
|
|
@ -0,0 +1,133 @@
|
|||
package knownhosts
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestEnsureCreatesFileAndCaches(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "known_hosts")
|
||||
|
||||
var calls int
|
||||
keyscan := func(ctx context.Context, host string, timeout time.Duration) ([]byte, error) {
|
||||
calls++
|
||||
return []byte(host + " ssh-ed25519 AAAA"), nil
|
||||
}
|
||||
|
||||
mgr, err := NewManager(path, WithKeyscanFunc(keyscan))
|
||||
if err != nil {
|
||||
t.Fatalf("NewManager: %v", err)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
if err := mgr.Ensure(ctx, "example.com"); err != nil {
|
||||
t.Fatalf("Ensure: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
t.Fatalf("known_hosts not created: %v", err)
|
||||
}
|
||||
|
||||
if err := mgr.Ensure(ctx, "example.com"); err != nil {
|
||||
t.Fatalf("Ensure second call: %v", err)
|
||||
}
|
||||
if calls != 1 {
|
||||
t.Fatalf("expected keyscan once, got %d", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureUsesSanitizedOutput(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "known_hosts")
|
||||
|
||||
keyscan := func(ctx context.Context, host string, timeout time.Duration) ([]byte, error) {
|
||||
return []byte(`# comment
|
||||
example.com ssh-ed25519 AAAA
|
||||
example.com,192.0.2.10 ssh-rsa BBBB
|
||||
other.com ssh-ed25519 CCCC
|
||||
`), nil
|
||||
}
|
||||
|
||||
mgr, err := NewManager(path, WithKeyscanFunc(keyscan))
|
||||
if err != nil {
|
||||
t.Fatalf("NewManager: %v", err)
|
||||
}
|
||||
|
||||
if err := mgr.Ensure(context.Background(), "example.com"); err != nil {
|
||||
t.Fatalf("Ensure: %v", err)
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile: %v", err)
|
||||
}
|
||||
if want := "example.com ssh-ed25519 AAAA\nexample.com,192.0.2.10 ssh-rsa BBBB\n"; string(data) != want {
|
||||
t.Fatalf("unexpected known_hosts contents\nwant:\n%s\ngot:\n%s", want, data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureReturnsErrorWhenNoEntries(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "known_hosts")
|
||||
|
||||
mgr, err := NewManager(path, WithKeyscanFunc(func(ctx context.Context, host string, timeout time.Duration) ([]byte, error) {
|
||||
return []byte("|1|hash|salt ssh-ed25519 AAAA\n"), nil
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatalf("NewManager: %v", err)
|
||||
}
|
||||
|
||||
err = mgr.Ensure(context.Background(), "example.com")
|
||||
if !errors.Is(err, ErrNoHostKeys) {
|
||||
t.Fatalf("expected ErrNoHostKeys, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureRespectsContextCancellation(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "known_hosts")
|
||||
|
||||
keyscan := func(ctx context.Context, host string, timeout time.Duration) ([]byte, error) {
|
||||
<-ctx.Done()
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
|
||||
mgr, err := NewManager(path, WithKeyscanFunc(keyscan), WithTimeout(50*time.Millisecond))
|
||||
if err != nil {
|
||||
t.Fatalf("NewManager: %v", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
if err := mgr.Ensure(ctx, "example.com"); err == nil {
|
||||
t.Fatalf("expected context error, got nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostCandidates(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
want []string
|
||||
}{
|
||||
{"example.com", []string{"example.com"}},
|
||||
{"example.com:2222", []string{"example.com:2222", "example.com"}},
|
||||
{"[example.com]:2222", []string{"[example.com]:2222", "example.com"}},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
got := hostCandidates(tt.input)
|
||||
if len(got) != len(tt.want) {
|
||||
t.Fatalf("hostCandidates(%q) len = %d, want %d", tt.input, len(got), len(tt.want))
|
||||
}
|
||||
for i := range got {
|
||||
if got[i] != tt.want[i] {
|
||||
t.Fatalf("hostCandidates(%q)[%d] = %q, want %q", tt.input, i, got[i], tt.want[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -4,6 +4,8 @@ import (
|
|||
"context"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
|
@ -22,6 +24,24 @@ type ClusterClient struct {
|
|||
lastHealthCheck map[string]time.Time // Track last health check time
|
||||
lastUsedIndex int // For round-robin
|
||||
config ClientConfig // Base config (auth info)
|
||||
rateLimitUntil map[string]time.Time // Cooldown window for rate-limited endpoints
|
||||
}
|
||||
|
||||
const (
|
||||
rateLimitBaseDelay = 150 * time.Millisecond
|
||||
rateLimitMaxJitter = 200 * time.Millisecond
|
||||
rateLimitRetryBudget = 2
|
||||
)
|
||||
|
||||
var statusCodePattern = regexp.MustCompile(`(?i)(?:api error|status)\s+(\d{3})`)
|
||||
|
||||
var transientRateLimitStatusCodes = map[int]struct{}{
|
||||
408: {},
|
||||
425: {}, // Too Early
|
||||
429: {},
|
||||
502: {},
|
||||
503: {},
|
||||
504: {},
|
||||
}
|
||||
|
||||
// NewClusterClient creates a new cluster-aware client
|
||||
|
|
@ -33,6 +53,7 @@ func NewClusterClient(name string, config ClientConfig, endpoints []string) *Clu
|
|||
nodeHealth: make(map[string]bool),
|
||||
lastHealthCheck: make(map[string]time.Time),
|
||||
config: config,
|
||||
rateLimitUntil: make(map[string]time.Time),
|
||||
}
|
||||
|
||||
// Initialize all endpoints as unknown (will be tested on first use)
|
||||
|
|
@ -163,12 +184,26 @@ func (cc *ClusterClient) getHealthyClient(ctx context.Context) (*Client, error)
|
|||
|
||||
// Get list of healthy endpoints
|
||||
var healthyEndpoints []string
|
||||
var coolingEndpoints []string
|
||||
now := time.Now()
|
||||
for endpoint, healthy := range cc.nodeHealth {
|
||||
if healthy {
|
||||
if cooldown, exists := cc.rateLimitUntil[endpoint]; exists {
|
||||
if now.Before(cooldown) {
|
||||
coolingEndpoints = append(coolingEndpoints, endpoint)
|
||||
continue
|
||||
}
|
||||
delete(cc.rateLimitUntil, endpoint)
|
||||
}
|
||||
healthyEndpoints = append(healthyEndpoints, endpoint)
|
||||
}
|
||||
}
|
||||
|
||||
if len(healthyEndpoints) == 0 && len(coolingEndpoints) > 0 {
|
||||
// Nothing is immediately available, fall back to endpoints that are in cooldown
|
||||
healthyEndpoints = append(healthyEndpoints, coolingEndpoints...)
|
||||
}
|
||||
|
||||
log.Debug().
|
||||
Str("cluster", cc.name).
|
||||
Int("healthy", len(healthyEndpoints)).
|
||||
|
|
@ -238,28 +273,38 @@ func (cc *ClusterClient) getHealthyClient(ctx context.Context) (*Client, error)
|
|||
cancel()
|
||||
|
||||
if testErr != nil {
|
||||
// Check if this is a VM-specific error that shouldn't mark the node unhealthy
|
||||
testErrStr := testErr.Error()
|
||||
if strings.Contains(testErrStr, "No QEMU guest agent") ||
|
||||
strings.Contains(testErrStr, "QEMU guest agent is not running") ||
|
||||
strings.Contains(testErrStr, "guest agent") {
|
||||
// This is a VM-specific issue, not a connectivity problem
|
||||
// The node is actually healthy, so don't mark it unhealthy
|
||||
// Check if this is a transient rate limit error that shouldn't mark the node unhealthy
|
||||
if isRateLimited, _ := isTransientRateLimitError(testErr); isRateLimited {
|
||||
log.Debug().
|
||||
Str("cluster", cc.name).
|
||||
Str("endpoint", selectedEndpoint).
|
||||
Err(testErr).
|
||||
Msg("Ignoring VM-specific error during connectivity test")
|
||||
// Continue with client creation since the node is actually accessible
|
||||
Msg("Ignoring transient rate limit error during connectivity test")
|
||||
// Continue with client creation since the node is accessible, just rate limited
|
||||
} else {
|
||||
// Mark as unhealthy for real connectivity issues
|
||||
cc.nodeHealth[selectedEndpoint] = false
|
||||
log.Warn().
|
||||
Str("cluster", cc.name).
|
||||
Str("endpoint", selectedEndpoint).
|
||||
Err(testErr).
|
||||
Msg("Cluster endpoint failed connectivity test")
|
||||
return nil, fmt.Errorf("endpoint %s failed connectivity test: %w", selectedEndpoint, testErr)
|
||||
// Check if this is a VM-specific error that shouldn't mark the node unhealthy
|
||||
testErrStr := testErr.Error()
|
||||
if strings.Contains(testErrStr, "No QEMU guest agent") ||
|
||||
strings.Contains(testErrStr, "QEMU guest agent is not running") ||
|
||||
strings.Contains(testErrStr, "guest agent") {
|
||||
// This is a VM-specific issue, not a connectivity problem
|
||||
// The node is actually healthy, so don't mark it unhealthy
|
||||
log.Debug().
|
||||
Str("cluster", cc.name).
|
||||
Str("endpoint", selectedEndpoint).
|
||||
Err(testErr).
|
||||
Msg("Ignoring VM-specific error during connectivity test")
|
||||
// Continue with client creation since the node is actually accessible
|
||||
} else {
|
||||
// Mark as unhealthy for real connectivity issues
|
||||
cc.nodeHealth[selectedEndpoint] = false
|
||||
log.Warn().
|
||||
Str("cluster", cc.name).
|
||||
Str("endpoint", selectedEndpoint).
|
||||
Err(testErr).
|
||||
Msg("Cluster endpoint failed connectivity test")
|
||||
return nil, fmt.Errorf("endpoint %s failed connectivity test: %w", selectedEndpoint, testErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -400,7 +445,9 @@ func (cc *ClusterClient) recoverUnhealthyNodes(ctx context.Context) {
|
|||
|
||||
// executeWithFailover executes a function with automatic failover
|
||||
func (cc *ClusterClient) executeWithFailover(ctx context.Context, fn func(*Client) error) error {
|
||||
maxRetries := len(cc.endpoints)
|
||||
baseRetries := len(cc.endpoints)
|
||||
maxRetries := baseRetries + rateLimitRetryBudget
|
||||
var lastErr error
|
||||
|
||||
log.Debug().
|
||||
Str("cluster", cc.name).
|
||||
|
|
@ -434,6 +481,7 @@ func (cc *ClusterClient) executeWithFailover(ctx context.Context, fn func(*Clien
|
|||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
lastErr = err
|
||||
|
||||
// Check error type and content
|
||||
errStr := err.Error()
|
||||
|
|
@ -467,6 +515,34 @@ func (cc *ClusterClient) executeWithFailover(ctx context.Context, fn func(*Clien
|
|||
return err
|
||||
}
|
||||
|
||||
if isRateLimited, statusCode := isTransientRateLimitError(err); isRateLimited {
|
||||
backoff := calculateRateLimitBackoff(i)
|
||||
cc.applyRateLimitCooldown(clientEndpoint, backoff)
|
||||
|
||||
event := log.Warn().
|
||||
Str("cluster", cc.name).
|
||||
Str("endpoint", clientEndpoint).
|
||||
Err(err).
|
||||
Dur("backoff", backoff).
|
||||
Int("attempt", i+1)
|
||||
if statusCode != 0 {
|
||||
event = event.Int("status", statusCode)
|
||||
}
|
||||
event.Msg("Rate limited by cluster node, retrying with backoff")
|
||||
|
||||
timer := time.NewTimer(backoff)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
if !timer.Stop() {
|
||||
<-timer.C
|
||||
}
|
||||
return fmt.Errorf("context canceled while backing off after rate limit: %w", ctx.Err())
|
||||
case <-timer.C:
|
||||
}
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
// Check if it's an auth error - don't retry on auth errors
|
||||
if IsAuthError(err) {
|
||||
return err
|
||||
|
|
@ -483,9 +559,75 @@ func (cc *ClusterClient) executeWithFailover(ctx context.Context, fn func(*Clien
|
|||
Msg("Failed on cluster node, trying next")
|
||||
}
|
||||
|
||||
if lastErr != nil {
|
||||
return fmt.Errorf("all cluster nodes failed for %s: %w", cc.name, lastErr)
|
||||
}
|
||||
|
||||
return fmt.Errorf("all cluster nodes failed for %s", cc.name)
|
||||
}
|
||||
|
||||
func (cc *ClusterClient) applyRateLimitCooldown(endpoint string, backoff time.Duration) {
|
||||
if endpoint == "" {
|
||||
return
|
||||
}
|
||||
|
||||
cc.mu.Lock()
|
||||
defer cc.mu.Unlock()
|
||||
if cc.rateLimitUntil == nil {
|
||||
cc.rateLimitUntil = make(map[string]time.Time)
|
||||
}
|
||||
cc.rateLimitUntil[endpoint] = time.Now().Add(backoff)
|
||||
}
|
||||
|
||||
func calculateRateLimitBackoff(attempt int) time.Duration {
|
||||
// Linear backoff with jitter keeps retries gentle while avoiding thundering herd
|
||||
base := rateLimitBaseDelay * time.Duration(attempt+1)
|
||||
if rateLimitMaxJitter <= 0 {
|
||||
return base
|
||||
}
|
||||
|
||||
jitter := time.Duration(rand.Int63n(rateLimitMaxJitter.Nanoseconds()+1)) * time.Nanosecond
|
||||
return base + jitter
|
||||
}
|
||||
|
||||
func isTransientRateLimitError(err error) (bool, int) {
|
||||
if err == nil {
|
||||
return false, 0
|
||||
}
|
||||
|
||||
errStr := err.Error()
|
||||
statusCode := extractStatusCode(errStr)
|
||||
if statusCode != 0 {
|
||||
if _, ok := transientRateLimitStatusCodes[statusCode]; ok {
|
||||
return true, statusCode
|
||||
}
|
||||
}
|
||||
|
||||
lowerErr := strings.ToLower(errStr)
|
||||
if strings.Contains(lowerErr, "rate limit") || strings.Contains(lowerErr, "too many requests") {
|
||||
if statusCode == 0 {
|
||||
statusCode = 429
|
||||
}
|
||||
return true, statusCode
|
||||
}
|
||||
|
||||
return false, statusCode
|
||||
}
|
||||
|
||||
func extractStatusCode(errStr string) int {
|
||||
matches := statusCodePattern.FindStringSubmatch(errStr)
|
||||
if len(matches) != 2 {
|
||||
return 0
|
||||
}
|
||||
|
||||
code, err := strconv.Atoi(matches[1])
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
return code
|
||||
}
|
||||
|
||||
// GetHealthStatus returns the health status of all nodes
|
||||
func (cc *ClusterClient) GetHealthStatus() map[string]bool {
|
||||
cc.mu.RLock()
|
||||
|
|
|
|||
65
pkg/proxmox/cluster_client_test.go
Normal file
65
pkg/proxmox/cluster_client_test.go
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
package proxmox
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestClusterClientHandlesRateLimitWithoutMarkingUnhealthy(t *testing.T) {
|
||||
var requestCount int32
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/api2/json/nodes":
|
||||
current := atomic.AddInt32(&requestCount, 1)
|
||||
if current == 1 {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusTooManyRequests)
|
||||
fmt.Fprint(w, `{"error":"rate limited"}`)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
fmt.Fprint(w, `{"data":[{"node":"node1","status":"online","cpu":0,"maxcpu":1,"mem":0,"maxmem":1,"disk":0,"maxdisk":1,"uptime":1,"level":"normal"}]}`)
|
||||
default:
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
fmt.Fprint(w, `{"data":{}}`)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
cfg := ClientConfig{
|
||||
Host: server.URL,
|
||||
TokenName: "pulse@pve!token",
|
||||
TokenValue: "sometokenvalue",
|
||||
VerifySSL: false,
|
||||
Timeout: 2 * time.Second,
|
||||
}
|
||||
|
||||
cc := NewClusterClient("test-cluster", cfg, []string{server.URL})
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
defer cancel()
|
||||
|
||||
nodes, err := cc.GetNodes(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("expected GetNodes to succeed after retry, got error: %v", err)
|
||||
}
|
||||
|
||||
if len(nodes) != 1 {
|
||||
t.Fatalf("expected 1 node after retry, got %d", len(nodes))
|
||||
}
|
||||
|
||||
health := cc.GetHealthStatus()
|
||||
if healthy, ok := health[server.URL]; !ok || !healthy {
|
||||
t.Fatalf("expected endpoint %s to remain healthy, got health map: %+v", server.URL, health)
|
||||
}
|
||||
|
||||
if atomic.LoadInt32(&requestCount) < 2 {
|
||||
t.Fatalf("expected at least 2 requests to backend, got %d", requestCount)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
# Identify files that will bloat Claude Code's context window.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
|
||||
cd "$ROOT_DIR"
|
||||
|
||||
echo "Scanning modified files that exceed size thresholds..."
|
||||
echo
|
||||
|
||||
git status --short | awk '{print $2}' | while read -r file; do
|
||||
[ -f "$file" ] || continue
|
||||
bytes=$(wc -c < "$file")
|
||||
if [ "$bytes" -ge 65536 ]; then
|
||||
lines=$(wc -l < "$file")
|
||||
printf "%8d bytes %7d lines %s\n" "$bytes" "$lines" "$file"
|
||||
fi
|
||||
done
|
||||
|
||||
echo
|
||||
echo "Tip: stash or split these files when you do not need Claude to inspect them directly."
|
||||
54
scripts/create-sensor-user.sh
Executable file
54
scripts/create-sensor-user.sh
Executable file
|
|
@ -0,0 +1,54 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if [[ $EUID -ne 0 ]]; then
|
||||
echo "Run as root." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SERVICE_USER="pulse-sensor"
|
||||
SERVICE_GROUP="$SERVICE_USER"
|
||||
HOME_DIR="/opt/pulse/sensor-proxy"
|
||||
BIN_DIR="$HOME_DIR/bin"
|
||||
CONFIG_DIR="$HOME_DIR/etc"
|
||||
SSH_DIR="$HOME_DIR/.ssh"
|
||||
LOG_DIR="/var/log/pulse/sensor-proxy"
|
||||
SUDOERS_FILE="/etc/sudoers.d/pulse-sensor-proxy"
|
||||
NOLOGIN_SHELL="/usr/sbin/nologin"
|
||||
|
||||
if id -u "$SERVICE_USER" >/dev/null 2>&1; then
|
||||
usermod --home "$HOME_DIR" --shell "$NOLOGIN_SHELL" "$SERVICE_USER"
|
||||
else
|
||||
useradd --system --home "$HOME_DIR" --shell "$NOLOGIN_SHELL" --user-group "$SERVICE_USER"
|
||||
fi
|
||||
|
||||
install -d -o "$SERVICE_USER" -g "$SERVICE_GROUP" -m 0750 "$HOME_DIR"
|
||||
install -d -o "$SERVICE_USER" -g "$SERVICE_GROUP" -m 0750 "$BIN_DIR"
|
||||
install -d -o root -g "$SERVICE_GROUP" -m 0750 "$CONFIG_DIR"
|
||||
install -d -o "$SERVICE_USER" -g "$SERVICE_GROUP" -m 0700 "$SSH_DIR"
|
||||
install -d -o "$SERVICE_USER" -g "$SERVICE_GROUP" -m 0750 "$LOG_DIR"
|
||||
|
||||
TMP_SUDOERS="$(mktemp)"
|
||||
trap 'rm -f "$TMP_SUDOERS"' EXIT
|
||||
|
||||
cat >"$TMP_SUDOERS" <<'EOF'
|
||||
pulse-sensor ALL=(root) NOPASSWD: /usr/bin/sensors, /usr/sbin/ipmitool
|
||||
EOF
|
||||
|
||||
install -o root -g root -m 0440 "$TMP_SUDOERS" "$SUDOERS_FILE"
|
||||
|
||||
if ! visudo -cf "$SUDOERS_FILE" >/dev/null; then
|
||||
echo "sudoers validation failed" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "User $(id "$SERVICE_USER")"
|
||||
namei -om "$BIN_DIR" >/dev/null
|
||||
namei -om "$CONFIG_DIR" >/dev/null
|
||||
namei -om "$SSH_DIR" >/dev/null
|
||||
namei -om "$LOG_DIR" >/dev/null
|
||||
|
||||
echo "Sudo privileges for $SERVICE_USER:"
|
||||
sudo -l -U "$SERVICE_USER" || true
|
||||
|
||||
echo "pulse-sensor service account ready."
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
#!/bin/bash
|
||||
# Dev Environment Orchestrator
|
||||
# Provides complete state detection and control for Claude Code
|
||||
# Provides complete state detection and control for development tools
|
||||
|
||||
set -eo pipefail
|
||||
|
||||
|
|
@ -239,7 +239,7 @@ cmd_status() {
|
|||
echo -e " Built: $([ "$frontend_built" = "true" ] && echo -e "${GREEN}Yes${NC}" || echo -e "${RED}No${NC}")"
|
||||
fi
|
||||
|
||||
# JSON output for Claude
|
||||
# JSON output for automation tools
|
||||
if [ "$1" = "--json" ]; then
|
||||
echo ""
|
||||
echo "$state"
|
||||
|
|
|
|||
9
scripts/docker-build.sh
Executable file
9
scripts/docker-build.sh
Executable file
|
|
@ -0,0 +1,9 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Simple wrapper that enables BuildKit and forwards all arguments.
|
||||
# To skip building multi-arch agents set BUILD_AGENT=0 before invoking.
|
||||
|
||||
export DOCKER_BUILDKIT=1
|
||||
|
||||
docker build "$@"
|
||||
38
scripts/harden-sensor-proxy.sh
Executable file
38
scripts/harden-sensor-proxy.sh
Executable file
|
|
@ -0,0 +1,38 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
FORCE_COMMAND="${FORCE_COMMAND:-/opt/pulse/bin/sensor-proxy-wrapper}"
|
||||
CONF_PATH="/etc/ssh/sshd_config.d/pulse-sensor-proxy.conf"
|
||||
|
||||
if [[ ! -x "$FORCE_COMMAND" ]]; then
|
||||
echo "Error: FORCE_COMMAND '$FORCE_COMMAND' not found or not executable" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
TMP_CONF="$(mktemp)"
|
||||
trap 'rm -f "$TMP_CONF"' EXIT
|
||||
|
||||
cat >"$TMP_CONF" <<EOF
|
||||
# Hardening for Pulse sensor proxy access
|
||||
PasswordAuthentication no
|
||||
KbdInteractiveAuthentication no
|
||||
ChallengeResponseAuthentication no
|
||||
PermitRootLogin no
|
||||
AllowAgentForwarding no
|
||||
AllowTcpForwarding no
|
||||
PermitTunnel no
|
||||
X11Forwarding no
|
||||
PermitUserEnvironment no
|
||||
ForceCommand $FORCE_COMMAND
|
||||
EOF
|
||||
|
||||
install -o root -g root -m 0644 "$TMP_CONF" "$CONF_PATH"
|
||||
|
||||
sshd -t
|
||||
systemctl reload sshd
|
||||
|
||||
echo "sshd hardening applied to $CONF_PATH"
|
||||
|
||||
# Verification
|
||||
echo "Verifying hardening settings:"
|
||||
sshd -T | grep -E 'passwordauthentication|allowagentforwarding|allowtcpforwarding|x11forwarding|permittunnel' || true
|
||||
|
|
@ -47,6 +47,14 @@ export FRONTEND_PORT PORT
|
|||
export FRONTEND_DEV_HOST FRONTEND_DEV_PORT
|
||||
export PULSE_DEV_API_HOST PULSE_DEV_API_PORT PULSE_DEV_API_URL PULSE_DEV_WS_URL
|
||||
|
||||
# Auto-detect pulse-sensor-proxy socket if available
|
||||
if [[ -z ${PULSE_SENSOR_PROXY_SOCKET:-} ]]; then
|
||||
if [[ -S /mnt/pulse-proxy/pulse-sensor-proxy.sock ]]; then
|
||||
export PULSE_SENSOR_PROXY_SOCKET=/mnt/pulse-proxy/pulse-sensor-proxy.sock
|
||||
printf "[hot-dev] Detected pulse-sensor-proxy socket at %s\n" "${PULSE_SENSOR_PROXY_SOCKET}"
|
||||
fi
|
||||
fi
|
||||
|
||||
EXTRA_CLEANUP_PORT=$((PULSE_DEV_API_PORT + 1))
|
||||
|
||||
cat <<BANNER
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ echo " Pulse Turnkey Docker Installation"
|
|||
echo "============================================"
|
||||
echo ""
|
||||
|
||||
# Check if running as root (early check per Codex feedback)
|
||||
# Check if running as root (early check for better error messages)
|
||||
if [ "$EUID" -ne 0 ]; then
|
||||
echo "❌ ERROR: This script must be run as root"
|
||||
echo ""
|
||||
|
|
|
|||
|
|
@ -236,6 +236,7 @@ fi
|
|||
print_info "Setting up directories with proper ownership..."
|
||||
install -d -o pulse-sensor-proxy -g pulse-sensor-proxy -m 0750 /var/lib/pulse-sensor-proxy
|
||||
install -d -o pulse-sensor-proxy -g pulse-sensor-proxy -m 0700 "$SSH_DIR"
|
||||
install -m 0600 -o pulse-sensor-proxy -g pulse-sensor-proxy /dev/null "$SSH_DIR/known_hosts"
|
||||
install -d -o pulse-sensor-proxy -g pulse-sensor-proxy -m 0755 /etc/pulse-sensor-proxy
|
||||
|
||||
# Create config file with ACL for Docker containers (standalone mode)
|
||||
|
|
|
|||
89
scripts/secure-sensor-files.sh
Executable file
89
scripts/secure-sensor-files.sh
Executable file
|
|
@ -0,0 +1,89 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if [[ $EUID -ne 0 ]]; then
|
||||
echo "Run as root." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SERVICE_USER="pulse-sensor"
|
||||
SERVICE_GROUP="$SERVICE_USER"
|
||||
HOME_DIR="/opt/pulse/sensor-proxy"
|
||||
BIN_PATH="$HOME_DIR/bin/pulse-sensor-proxy"
|
||||
SSH_DIR="$HOME_DIR/.ssh"
|
||||
PRIVATE_KEY="$SSH_DIR/id_ed25519"
|
||||
PUBLIC_KEY="$SSH_DIR/id_ed25519.pub"
|
||||
KNOWN_HOSTS="$SSH_DIR/known_hosts"
|
||||
LOG_DIR="/var/log/pulse/sensor-proxy"
|
||||
LOG_FILE="$LOG_DIR/proxy.log"
|
||||
AUDIT_LOG="$LOG_DIR/audit.log"
|
||||
|
||||
umask 077
|
||||
|
||||
install -d -o "$SERVICE_USER" -g "$SERVICE_GROUP" -m 0700 "$SSH_DIR"
|
||||
|
||||
if [[ ! -f "$PRIVATE_KEY" ]]; then
|
||||
sudo -u "$SERVICE_USER" ssh-keygen -t ed25519 -N '' -C "pulse-sensor@$(hostname -f)" -f "$PRIVATE_KEY"
|
||||
else
|
||||
chown "$SERVICE_USER:$SERVICE_GROUP" "$PRIVATE_KEY"
|
||||
chmod 0600 "$PRIVATE_KEY"
|
||||
fi
|
||||
|
||||
chown "$SERVICE_USER:$SERVICE_GROUP" "$PRIVATE_KEY"
|
||||
chmod 0600 "$PRIVATE_KEY"
|
||||
|
||||
if [[ -f "$PUBLIC_KEY" ]]; then
|
||||
chown "$SERVICE_USER:$SERVICE_GROUP" "$PUBLIC_KEY"
|
||||
chmod 0640 "$PUBLIC_KEY"
|
||||
else
|
||||
sudo -u "$SERVICE_USER" ssh-keygen -y -f "$PRIVATE_KEY" >"$PUBLIC_KEY"
|
||||
chown "$SERVICE_USER:$SERVICE_GROUP" "$PUBLIC_KEY"
|
||||
chmod 0640 "$PUBLIC_KEY"
|
||||
fi
|
||||
|
||||
if [[ ! -f "$KNOWN_HOSTS" ]]; then
|
||||
install -o "$SERVICE_USER" -g "$SERVICE_GROUP" -m 0640 /dev/null "$KNOWN_HOSTS"
|
||||
else
|
||||
chown "$SERVICE_USER:$SERVICE_GROUP" "$KNOWN_HOSTS"
|
||||
chmod 0640 "$KNOWN_HOSTS"
|
||||
fi
|
||||
|
||||
install -d -o "$SERVICE_USER" -g "$SERVICE_GROUP" -m 0750 "$LOG_DIR"
|
||||
|
||||
for log_path in "$LOG_FILE" "$AUDIT_LOG"; do
|
||||
if [[ ! -f "$log_path" ]]; then
|
||||
install -o "$SERVICE_USER" -g "$SERVICE_GROUP" -m 0640 /dev/null "$log_path"
|
||||
else
|
||||
chown "$SERVICE_USER:$SERVICE_GROUP" "$log_path"
|
||||
chmod 0640 "$log_path"
|
||||
fi
|
||||
|
||||
if command -v chattr >/dev/null 2>&1; then
|
||||
if ! lsattr "$log_path" 2>/dev/null | grep -q 'a'; then
|
||||
chattr +a "$log_path" || echo "Warning: could not set append-only attribute on $log_path" >&2
|
||||
fi
|
||||
else
|
||||
echo "Warning: chattr not available; skipping append-only for $log_path." >&2
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ -f "$BIN_PATH" ]]; then
|
||||
chown root:"$SERVICE_GROUP" "$BIN_PATH"
|
||||
chmod 0750 "$BIN_PATH"
|
||||
fi
|
||||
|
||||
echo "SSH artifacts:"
|
||||
ls -l "$PRIVATE_KEY" "$PUBLIC_KEY" "$KNOWN_HOSTS"
|
||||
|
||||
echo "Log files:"
|
||||
ls -l "$LOG_FILE" "$AUDIT_LOG"
|
||||
if command -v lsattr >/dev/null 2>&1; then
|
||||
lsattr "$LOG_FILE" "$AUDIT_LOG" || true
|
||||
fi
|
||||
|
||||
if [[ -f "$BIN_PATH" ]]; then
|
||||
echo "Binary permissions:"
|
||||
ls -l "$BIN_PATH"
|
||||
fi
|
||||
|
||||
echo "sensor proxy file permissions secured."
|
||||
62
scripts/setup-log-forwarding.sh
Executable file
62
scripts/setup-log-forwarding.sh
Executable file
|
|
@ -0,0 +1,62 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if [[ $EUID -ne 0 ]]; then
|
||||
echo "Run as root" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
REMOTE_HOST=${REMOTE_HOST:-logs.pulse.example}
|
||||
REMOTE_PORT=${REMOTE_PORT:-6514}
|
||||
CERT_DIR=${CERT_DIR:-/etc/pulse/log-forwarding}
|
||||
CA_CERT=${CA_CERT:-$CERT_DIR/ca.crt}
|
||||
CLIENT_CERT=${CLIENT_CERT:-$CERT_DIR/client.crt}
|
||||
CLIENT_KEY=${CLIENT_KEY:-$CERT_DIR/client.key}
|
||||
|
||||
install -d -m 0750 "$CERT_DIR"
|
||||
|
||||
CONF_PATH=/etc/rsyslog.d/pulse-sensor-proxy.conf
|
||||
cat <<EOF >"$CONF_PATH"
|
||||
module(load="imfile" PollingInterval="5")
|
||||
|
||||
input(type="imfile"
|
||||
File="/var/log/pulse/sensor-proxy/audit.log"
|
||||
Tag="pulse.audit"
|
||||
Facility="local4"
|
||||
Severity="notice"
|
||||
PersistStateInterval="100"
|
||||
addMetadata="on")
|
||||
|
||||
input(type="imfile"
|
||||
File="/var/log/pulse/sensor-proxy/proxy.log"
|
||||
Tag="pulse.app"
|
||||
Facility="local4"
|
||||
Severity="info"
|
||||
PersistStateInterval="100"
|
||||
addMetadata="on")
|
||||
|
||||
action(type="omfile"
|
||||
File="/var/log/pulse/sensor-proxy/forwarding.log"
|
||||
Template="RSYSLOG_TraditionalFileFormat"
|
||||
DirCreateMode="0750"
|
||||
FileCreateMode="0640")
|
||||
|
||||
if (\$programname == 'pulse.audit' or \$programname == 'pulse.app') then {
|
||||
action(type="omrelp"
|
||||
target="$REMOTE_HOST"
|
||||
port="$REMOTE_PORT"
|
||||
tls="on"
|
||||
tls.caCert="$CA_CERT"
|
||||
tls.myCert="$CLIENT_CERT"
|
||||
tls.myPrivKey="$CLIENT_KEY"
|
||||
queue.type="LinkedList"
|
||||
queue.size="50000"
|
||||
queue.dequeuebatchsize="500"
|
||||
queue.workerthreads="2"
|
||||
action.resumeRetryCount="-1")
|
||||
stop
|
||||
}
|
||||
EOF
|
||||
|
||||
systemctl restart rsyslog
|
||||
echo "Log forwarding enabled to $REMOTE_HOST:$REMOTE_PORT"
|
||||
75
security/apparmor/pulse-sensor-proxy.apparmor
Normal file
75
security/apparmor/pulse-sensor-proxy.apparmor
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
#include <tunables/global>
|
||||
|
||||
profile pulse-sensor-proxy /opt/pulse/sensor-proxy/bin/pulse-sensor-proxy flags=(attach_disconnected,mediate_deleted) {
|
||||
capability chown,
|
||||
capability dac_override,
|
||||
capability dac_read_search,
|
||||
capability setgid,
|
||||
capability setuid,
|
||||
|
||||
network inet stream,
|
||||
network inet6 stream,
|
||||
network unix stream,
|
||||
deny network raw,
|
||||
|
||||
@{PROC}/@{pid}/fd/** r,
|
||||
@{PROC}/@{pid}/cmdline r,
|
||||
@{PROC}/@{pid}/stat r,
|
||||
@{PROC}/@{pid}/status r,
|
||||
|
||||
/opt/pulse/sensor-proxy/bin/pulse-sensor-proxy mr,
|
||||
/opt/pulse/sensor-proxy/bin/* mr,
|
||||
/opt/pulse/sensor-proxy/.ssh/** rwk,
|
||||
/opt/pulse/sensor-proxy/etc/** r,
|
||||
/opt/pulse/sensor-proxy/** r,
|
||||
/var/log/pulse/sensor-proxy/** rw,
|
||||
/run/pulse-sensor-proxy/** rw,
|
||||
|
||||
/etc/hosts r,
|
||||
/etc/hostname r,
|
||||
/etc/resolv.conf r,
|
||||
/etc/pulse-sensor-proxy/** r,
|
||||
|
||||
/usr/bin/ssh mr,
|
||||
/usr/bin/socat mr,
|
||||
/usr/bin/sensors mr,
|
||||
/usr/sbin/ipmitool mr,
|
||||
/bin/bash mr,
|
||||
/bin/sh mr,
|
||||
/bin/cat mr,
|
||||
/bin/echo mr,
|
||||
/usr/bin/tee mr,
|
||||
|
||||
/usr/lib/** mr,
|
||||
/lib/** mr,
|
||||
|
||||
deny /etc/shadow rwl,
|
||||
deny /root/** rwxl,
|
||||
|
||||
ptrace (read) peer=pulse-sensor-proxy,
|
||||
|
||||
signal (receive) set=(hup term int usr1 usr2),
|
||||
signal (send) set=(term) peer=pulse-sensor-proxy,
|
||||
|
||||
/usr/bin/ssh ixr,
|
||||
/usr/bin/sensors ixr,
|
||||
/usr/sbin/ipmitool ixr,
|
||||
/bin/sh ixr,
|
||||
/bin/cat ixr,
|
||||
/bin/echo ixr,
|
||||
/usr/bin/tee ixr,
|
||||
|
||||
deny mount,
|
||||
deny ptrace,
|
||||
deny sys_module,
|
||||
deny sys_rawio,
|
||||
deny sys_admin,
|
||||
|
||||
@{HOME}/.cache/** rw,
|
||||
|
||||
include <abstractions/base>
|
||||
include <abstractions/authentication>
|
||||
include <abstractions/nameservice>
|
||||
include <abstractions/openssl>
|
||||
include <abstractions/user-tmp>
|
||||
}
|
||||
102
security/seccomp/pulse-sensor-proxy.json
Normal file
102
security/seccomp/pulse-sensor-proxy.json
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
{
|
||||
"defaultAction": "SCMP_ACT_ERRNO",
|
||||
"architectures": [
|
||||
"SCMP_ARCH_X86_64",
|
||||
"SCMP_ARCH_AARCH64"
|
||||
],
|
||||
"syscalls": [
|
||||
{
|
||||
"names": [
|
||||
"accept",
|
||||
"accept4",
|
||||
"access",
|
||||
"bind",
|
||||
"brk",
|
||||
"capget",
|
||||
"capset",
|
||||
"chdir",
|
||||
"chmod",
|
||||
"chown",
|
||||
"clock_gettime",
|
||||
"close",
|
||||
"connect",
|
||||
"dup",
|
||||
"dup2",
|
||||
"epoll_create1",
|
||||
"epoll_ctl",
|
||||
"epoll_wait",
|
||||
"eventfd2",
|
||||
"execve",
|
||||
"exit",
|
||||
"exit_group",
|
||||
"fchmod",
|
||||
"fchown",
|
||||
"fcntl",
|
||||
"fdatasync",
|
||||
"fstat",
|
||||
"fsync",
|
||||
"ftruncate",
|
||||
"futex",
|
||||
"getdents64",
|
||||
"getegid",
|
||||
"geteuid",
|
||||
"getgid",
|
||||
"getpeername",
|
||||
"getpid",
|
||||
"getppid",
|
||||
"getrandom",
|
||||
"getrlimit",
|
||||
"getsockname",
|
||||
"getsockopt",
|
||||
"gettid",
|
||||
"getuid",
|
||||
"ioctl",
|
||||
"lseek",
|
||||
"madvise",
|
||||
"mkdir",
|
||||
"mmap",
|
||||
"mprotect",
|
||||
"munmap",
|
||||
"newfstatat",
|
||||
"open",
|
||||
"openat",
|
||||
"pipe2",
|
||||
"prctl",
|
||||
"pread64",
|
||||
"pwrite64",
|
||||
"read",
|
||||
"readlink",
|
||||
"recvfrom",
|
||||
"recvmmsg",
|
||||
"recvmsg",
|
||||
"rename",
|
||||
"rt_sigaction",
|
||||
"rt_sigprocmask",
|
||||
"rt_sigreturn",
|
||||
"sendmmsg",
|
||||
"sendmsg",
|
||||
"sendto",
|
||||
"setgid",
|
||||
"setgroups",
|
||||
"setrlimit",
|
||||
"setsid",
|
||||
"setsockopt",
|
||||
"setuid",
|
||||
"shutdown",
|
||||
"sigaltstack",
|
||||
"socket",
|
||||
"socketpair",
|
||||
"stat",
|
||||
"statx",
|
||||
"symlink",
|
||||
"tgkill",
|
||||
"unlink",
|
||||
"unlinkat",
|
||||
"wait4",
|
||||
"write",
|
||||
"writev"
|
||||
],
|
||||
"action": "SCMP_ACT_ALLOW"
|
||||
}
|
||||
]
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue