mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-19 10:33:29 +00:00
782 lines
32 KiB
Go
782 lines
32 KiB
Go
package api
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/config"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/relay"
|
|
unified "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
|
|
"github.com/rcourtman/pulse-go-rewrite/pkg/aicontracts"
|
|
"github.com/rcourtman/pulse-go-rewrite/pkg/auth"
|
|
)
|
|
|
|
type barrierPatrolExecutor struct {
|
|
calls atomic.Int32
|
|
entered chan struct{}
|
|
release chan struct{}
|
|
}
|
|
|
|
func (e *barrierPatrolExecutor) ExecuteAction(_ context.Context, _ unified.ActionAuditRecord) (*unified.ExecutionResult, error) {
|
|
if e.calls.Add(1) == 1 {
|
|
close(e.entered)
|
|
}
|
|
<-e.release
|
|
return &unified.ExecutionResult{Success: true}, nil
|
|
}
|
|
|
|
func configurePatrolAutoAuthorization(t *testing.T, h *ResourceHandlers) {
|
|
t.Helper()
|
|
store, err := h.getStore("default")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := store.SetResourceOperatorState(unified.ResourceOperatorState{CanonicalID: "vm:42", AutoRemediationPolicy: unified.AutoRemediationPolicy{Enabled: true, CapabilityNames: []string{"restart"}}, SetAt: time.Now().UTC()}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestPatrolActionBrokerBarrierReplayAdmitsExecutorExactlyOnce(t *testing.T) {
|
|
h, _ := newPatrolBrokerTestHandlers(t, unified.ApprovalAdmin)
|
|
executor := &barrierPatrolExecutor{entered: make(chan struct{}), release: make(chan struct{})}
|
|
h.SetActionExecutor(executor)
|
|
configurePatrolAutoAuthorization(t, h)
|
|
broker := NewPatrolActionBroker("default", h, func(context.Context, string) (PatrolActionPolicySnapshot, error) {
|
|
return PatrolActionPolicySnapshot{EffectiveAutonomyLevel: "assisted"}, nil
|
|
})
|
|
first := make(chan error, 1)
|
|
go func() { _, err := broker.Submit(context.Background(), patrolTestProposal()); first <- err }()
|
|
<-executor.entered
|
|
secondDisposition, secondErr := broker.Submit(context.Background(), patrolTestProposal())
|
|
if secondErr != nil {
|
|
t.Fatalf("second Submit: %v", secondErr)
|
|
}
|
|
if secondDisposition.State != string(unified.ActionStateExecuting) {
|
|
t.Fatalf("second state=%q", secondDisposition.State)
|
|
}
|
|
close(executor.release)
|
|
if err := <-first; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if executor.calls.Load() != 1 {
|
|
t.Fatalf("executor calls=%d, want 1", executor.calls.Load())
|
|
}
|
|
}
|
|
|
|
func TestPatrolActionBrokerReplayDuringExecutionReturnsCurrentDisposition(t *testing.T) {
|
|
TestPatrolActionBrokerBarrierReplayAdmitsExecutorExactlyOnce(t)
|
|
}
|
|
|
|
func TestPatrolActionBrokerTerminalReplayPreservesAuditAndEvents(t *testing.T) {
|
|
h, executor := newPatrolBrokerTestHandlers(t, unified.ApprovalAdmin)
|
|
configurePatrolAutoAuthorization(t, h)
|
|
broker := NewPatrolActionBroker("default", h, func(context.Context, string) (PatrolActionPolicySnapshot, error) {
|
|
return PatrolActionPolicySnapshot{EffectiveAutonomyLevel: "assisted"}, nil
|
|
})
|
|
first, err := broker.Submit(context.Background(), patrolTestProposal())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
store, _ := h.getStore("default")
|
|
before, err := store.GetActionLifecycleEvents(first.ActionID, time.Time{}, 100)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
second, err := broker.Submit(context.Background(), patrolTestProposal())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
after, err := store.GetActionLifecycleEvents(first.ActionID, time.Time{}, 100)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if second.State != string(unified.ActionStateCompleted) || len(after) != len(before) || executor.calls != 1 {
|
|
t.Fatalf("second=%#v events=%d/%d calls=%d", second, len(before), len(after), executor.calls)
|
|
}
|
|
}
|
|
|
|
func TestPatrolActionBrokerSnapshotsTenantResourceAndCapabilityPolicyAtPlanTime(t *testing.T) {
|
|
h, executor := newPatrolBrokerTestHandlers(t, unified.ApprovalAdmin)
|
|
store, err := h.getStore("default")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
state := unified.ResourceOperatorState{CanonicalID: "vm:42", NeverAutoRemediate: true, AutoRemediationPolicy: unified.AutoRemediationPolicy{Enabled: true, CapabilityNames: []string{"restart"}}, SetAt: time.Date(2026, 7, 11, 20, 0, 0, 0, time.UTC)}
|
|
if err := store.SetResourceOperatorState(state); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
broker := NewPatrolActionBroker("default", h, func(context.Context, string) (PatrolActionPolicySnapshot, error) {
|
|
return PatrolActionPolicySnapshot{EffectiveAutonomyLevel: "monitor", EmergencyStop: true}, nil
|
|
})
|
|
disposition, err := broker.Submit(context.Background(), patrolTestProposal())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
record, found, err := store.GetActionAudit(disposition.ActionID)
|
|
if err != nil || !found {
|
|
t.Fatalf("record: found=%v err=%v", found, err)
|
|
}
|
|
provenance := record.Plan.PolicyDecision
|
|
if provenance.Version != unified.ActionPolicyDecisionVersion || len(provenance.Authorities) != 3 {
|
|
t.Fatalf("provenance=%#v", provenance)
|
|
}
|
|
if provenance.Authorities[0].Kind != unified.ActionPolicyAuthorityCapability || provenance.Authorities[1].Kind != unified.ActionPolicyAuthorityTenant || provenance.Authorities[2].Kind != unified.ActionPolicyAuthorityResource {
|
|
t.Fatalf("authority order=%#v", provenance.Authorities)
|
|
}
|
|
if !containsPolicyReason(provenance.Authorities[1].ReasonCodes, unified.PolicyReasonTenantEmergencyStop) || !containsPolicyReason(provenance.Authorities[1].ReasonCodes, unified.PolicyReasonTenantModeMonitor) || !containsPolicyReason(provenance.Authorities[2].ReasonCodes, unified.PolicyReasonResourceNeverAuto) {
|
|
t.Fatalf("policy reasons=%#v", provenance.Authorities)
|
|
}
|
|
if executor.calls != 0 {
|
|
t.Fatalf("descriptive provenance authorized dispatch: calls=%d", executor.calls)
|
|
}
|
|
}
|
|
|
|
func TestPatrolApprovalRefusesLostReadinessBeforeRecordingDecision(t *testing.T) {
|
|
h, executor := newPatrolBrokerTestHandlers(t, unified.ApprovalAdmin)
|
|
disposition, err := NewPatrolActionBroker("default", h).Submit(context.Background(), patrolTestProposal())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
executor.readiness = &unified.ResourceActionReadiness{
|
|
Name: "restart",
|
|
Available: false,
|
|
ReasonCode: "command_agent_disconnected",
|
|
Reason: "Connect the command agent for web-42.",
|
|
}
|
|
recorder := httptest.NewRecorder()
|
|
request := httptest.NewRequest(http.MethodPost, "/api/actions/"+disposition.ActionID+"/decision", bytes.NewBufferString(`{"outcome":"approved"}`))
|
|
request.SetPathValue("id", disposition.ActionID)
|
|
h.HandleDecideAction(recorder, actionHandlerTestRequest(request, "operator@example.com"))
|
|
if recorder.Code != http.StatusConflict {
|
|
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
|
|
}
|
|
var envelope struct {
|
|
Error string `json:"error"`
|
|
Details map[string]string `json:"details"`
|
|
}
|
|
if err := json.Unmarshal(recorder.Body.Bytes(), &envelope); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if envelope.Error != "action_execution_unavailable" || envelope.Details["reasonCode"] != "command_agent_disconnected" || envelope.Details["reason"] != "Connect the command agent for web-42." {
|
|
t.Fatalf("envelope=%#v", envelope)
|
|
}
|
|
record, found, err := h.ActionLifecycle().Get("default", disposition.ActionID)
|
|
if err != nil || !found || record.State != unified.ActionStatePending || len(record.Approvals) != 0 || executor.calls != 0 {
|
|
t.Fatalf("record=%#v found=%v err=%v calls=%d", record, found, err, executor.calls)
|
|
}
|
|
}
|
|
|
|
func TestPatrolRefreshRebindsCurrentPolicyAndTrustedOrigin(t *testing.T) {
|
|
h, _ := newPatrolBrokerTestHandlers(t, unified.ApprovalAdmin)
|
|
policyVersion := "tenant-v1"
|
|
policy := func(context.Context, string) (PatrolActionPolicySnapshot, error) {
|
|
return PatrolActionPolicySnapshot{EffectiveAutonomyLevel: "monitor", PolicyVersion: policyVersion}, nil
|
|
}
|
|
disposition, err := NewPatrolActionBroker("default", h, policy).Submit(context.Background(), patrolTestProposal())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
previous, found, err := h.ActionLifecycle().Get("default", disposition.ActionID)
|
|
if err != nil || !found {
|
|
t.Fatalf("previous found=%v err=%v", found, err)
|
|
}
|
|
registry, err := h.buildRegistry("default")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
resource, found := registry.Get(previous.Request.ResourceID)
|
|
if !found || resource == nil {
|
|
t.Fatal("planned resource missing")
|
|
}
|
|
changed := *resource
|
|
changed.UpdatedAt = changed.UpdatedAt.Add(time.Second)
|
|
changed.Name = changed.Name + " refreshed"
|
|
h.SetStateProvider(resourceUnifiedSeedProvider{
|
|
snapshot: models.StateSnapshot{LastUpdate: changed.UpdatedAt},
|
|
resources: []unified.Resource{changed},
|
|
})
|
|
h.cacheMu.Lock()
|
|
h.registryCache = make(map[string]registryCacheEntry)
|
|
h.cacheMu.Unlock()
|
|
policyVersion = "tenant-v2"
|
|
h.SetActionRefreshPlanner(NewActionRefreshPlanner(h, policy))
|
|
|
|
recorder := httptest.NewRecorder()
|
|
request := httptest.NewRequest(http.MethodPost, "/api/actions/"+previous.ID+"/refresh", bytes.NewBufferString(`{"planHash":"`+previous.Plan.PlanHash+`"}`))
|
|
request.SetPathValue("id", previous.ID)
|
|
h.HandleRefreshAction(recorder, actionHandlerTestRequest(request, "operator@example.com"))
|
|
if recorder.Code != http.StatusOK {
|
|
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
|
|
}
|
|
var detail actionDetailResponse
|
|
if err := json.Unmarshal(recorder.Body.Bytes(), &detail); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
replacement := detail.Audit.ActionAuditRecord
|
|
if replacement.ID == previous.ID || replacement.Request.RequestID != "refresh:"+previous.ID || replacement.Request.Actor.SubjectID != patrolActionBrokerActor {
|
|
t.Fatalf("replacement=%#v", replacement)
|
|
}
|
|
if replacement.Origin == nil || replacement.Origin.FindingID != previous.Origin.FindingID || replacement.Origin.InvestigationID != previous.Origin.InvestigationID {
|
|
t.Fatalf("origin=%#v", replacement.Origin)
|
|
}
|
|
if !detail.Readiness.Ready || len(replacement.Plan.PolicyDecision.Authorities) != 3 || replacement.Plan.PolicyDecision.Authorities[1].Revision == previous.Plan.PolicyDecision.Authorities[1].Revision {
|
|
t.Fatalf("readiness=%#v policy=%#v", detail.Readiness, replacement.Plan.PolicyDecision)
|
|
}
|
|
}
|
|
|
|
func containsPolicyReason(reasons []unified.ActionPolicyReasonCode, target unified.ActionPolicyReasonCode) bool {
|
|
for _, reason := range reasons {
|
|
if reason == target {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func TestPatrolActionBrokerConflictingOriginReplayFailsWithoutDispatch(t *testing.T) {
|
|
h, executor := newPatrolBrokerTestHandlers(t, unified.ApprovalAdmin)
|
|
broker := NewPatrolActionBroker("default", h)
|
|
if _, err := broker.Submit(context.Background(), patrolTestProposal()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
conflict := patrolTestProposal()
|
|
conflict.FindingID = "finding-2"
|
|
conflict.InvestigationID = "inv-2"
|
|
if _, err := broker.Submit(context.Background(), conflict); !errors.Is(err, unified.ErrActionIdentityConflict) {
|
|
t.Fatalf("error=%v", err)
|
|
}
|
|
if executor.calls != 0 {
|
|
t.Fatalf("executor calls=%d", executor.calls)
|
|
}
|
|
}
|
|
|
|
func newPatrolBrokerTestHandlers(t *testing.T, minimumApproval unified.ActionApprovalLevel) (*ResourceHandlers, *stubActionExecutor) {
|
|
return newPatrolBrokerTestHandlersWithEligibility(t, minimumApproval, unified.AutoAuthorizeLowRisk)
|
|
}
|
|
|
|
func newPatrolBrokerTestHandlersWithEligibility(t *testing.T, minimumApproval unified.ActionApprovalLevel, eligibility unified.ActionAutoAuthorizationClass) (*ResourceHandlers, *stubActionExecutor) {
|
|
t.Helper()
|
|
now := time.Now().UTC()
|
|
h := newActionTestResourceHandlers(t, &config.Config{DataPath: t.TempDir()})
|
|
h.SetStateProvider(resourceUnifiedSeedProvider{
|
|
snapshot: models.StateSnapshot{LastUpdate: now},
|
|
resources: []unified.Resource{
|
|
{
|
|
ID: "vm:42",
|
|
Type: unified.ResourceTypeVM,
|
|
Name: "web-42",
|
|
Status: unified.StatusWarning,
|
|
LastSeen: now,
|
|
UpdatedAt: now,
|
|
Sources: []unified.DataSource{unified.SourceProxmox},
|
|
Capabilities: []unified.ResourceCapability{
|
|
{
|
|
Name: "restart",
|
|
Type: unified.CapabilityTypeCommon,
|
|
Description: "Restart the VM",
|
|
MinimumApprovalLevel: minimumApproval,
|
|
AutoAuthorization: eligibility,
|
|
InternalHandler: "proxmox.vm.restart",
|
|
Params: []unified.CapabilityParam{
|
|
{Name: "mode", Type: "string", Required: true, Enum: []string{"graceful", "force"}},
|
|
},
|
|
},
|
|
{
|
|
Name: "join_cluster",
|
|
Type: unified.CapabilityTypeCommon,
|
|
Description: "Join an authenticated cluster",
|
|
MinimumApprovalLevel: unified.ApprovalAdmin,
|
|
InternalHandler: "proxmox.vm.join",
|
|
Params: []unified.CapabilityParam{
|
|
{Name: "join_token", Type: "string", Required: true, IsSensitive: true},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
})
|
|
executor := &stubActionExecutor{result: &unified.ExecutionResult{Success: true}}
|
|
h.SetActionExecutor(executor)
|
|
return h, executor
|
|
}
|
|
|
|
func patrolTestProposal() aicontracts.ActionProposal {
|
|
return aicontracts.ActionProposal{
|
|
ProposalID: "prop-1",
|
|
FindingID: "finding-1",
|
|
InvestigationID: "inv-1",
|
|
ResourceID: "vm:42",
|
|
CapabilityName: "restart",
|
|
Params: map[string]any{"mode": "graceful"},
|
|
Reason: "Recover after confirmed outage",
|
|
}
|
|
}
|
|
|
|
func TestPatrolActionBrokerSubmitPlansThroughCanonicalLifecycle(t *testing.T) {
|
|
h, executor := newPatrolBrokerTestHandlers(t, unified.ApprovalAdmin)
|
|
broker := NewPatrolActionBroker("default", h)
|
|
|
|
proposal := patrolTestProposal()
|
|
proposal.EvidenceIDs = []string{" evidence-2 ", "evidence-1", "evidence-1", ""}
|
|
disposition, err := broker.Submit(context.Background(), proposal)
|
|
if err != nil {
|
|
t.Fatalf("Submit: %v", err)
|
|
}
|
|
if disposition.ActionID == "" {
|
|
t.Fatal("disposition has no action id")
|
|
}
|
|
if disposition.State != string(unified.ActionStatePending) {
|
|
t.Fatalf("state = %q, want pending_approval", disposition.State)
|
|
}
|
|
if !disposition.Plan.RequiresApproval || disposition.Plan.ApprovalPolicy != string(unified.ApprovalAdmin) {
|
|
t.Fatalf("plan projection = %#v", disposition.Plan)
|
|
}
|
|
if executor.calls != 0 {
|
|
t.Fatalf("submit without core policy must not execute, executor calls = %d", executor.calls)
|
|
}
|
|
|
|
store, err := h.getStore("default")
|
|
if err != nil {
|
|
t.Fatalf("getStore: %v", err)
|
|
}
|
|
record, ok, err := store.GetActionAudit(disposition.ActionID)
|
|
if err != nil || !ok {
|
|
t.Fatalf("GetActionAudit: ok=%v err=%v", ok, err)
|
|
}
|
|
if record.Request.RequestedBy != patrolActionBrokerActor {
|
|
t.Fatalf("requestedBy = %q, want %q", record.Request.RequestedBy, patrolActionBrokerActor)
|
|
}
|
|
if record.Origin == nil ||
|
|
record.Origin.Surface != patrolActionOriginSurface ||
|
|
record.Origin.FindingID != "finding-1" ||
|
|
record.Origin.InvestigationID != "inv-1" ||
|
|
record.Origin.ProposalID != "prop-1" ||
|
|
fmt.Sprint(record.Origin.EvidenceIDs) != "[evidence-1 evidence-2]" {
|
|
t.Fatalf("audit origin = %#v", record.Origin)
|
|
}
|
|
}
|
|
|
|
func TestPatrolActionBrokerApprovalNoneDoesNotExecuteWithoutCorePolicy(t *testing.T) {
|
|
h, executor := newPatrolBrokerTestHandlers(t, unified.ApprovalNone)
|
|
broker := NewPatrolActionBroker("default", h)
|
|
|
|
disposition, err := broker.Submit(context.Background(), patrolTestProposal())
|
|
if err != nil {
|
|
t.Fatalf("Submit: %v", err)
|
|
}
|
|
if disposition.State != string(unified.ActionStatePlanned) {
|
|
t.Fatalf("state = %q, want planned", disposition.State)
|
|
}
|
|
if disposition.Plan.RequiresApproval {
|
|
t.Fatal("ApprovalNone capability should not require approval")
|
|
}
|
|
if executor.calls != 0 {
|
|
t.Fatalf("submit without core policy must not execute, executor calls = %d", executor.calls)
|
|
}
|
|
}
|
|
|
|
func TestPatrolActionBrokerRejectsSensitiveParams(t *testing.T) {
|
|
h, executor := newPatrolBrokerTestHandlers(t, unified.ApprovalAdmin)
|
|
broker := NewPatrolActionBroker("default", h)
|
|
|
|
proposal := patrolTestProposal()
|
|
proposal.CapabilityName = "join_cluster"
|
|
proposal.Params = map[string]any{"join_token": "sekret-token-value"}
|
|
_, err := broker.Submit(context.Background(), proposal)
|
|
if !errors.Is(err, aicontracts.ErrSensitiveParamsRequireOperator) {
|
|
t.Fatalf("error = %v, want ErrSensitiveParamsRequireOperator", err)
|
|
}
|
|
if executor.calls != 0 {
|
|
t.Fatalf("executor calls = %d, want 0", executor.calls)
|
|
}
|
|
|
|
store, err := h.getStore("default")
|
|
if err != nil {
|
|
t.Fatalf("getStore: %v", err)
|
|
}
|
|
audits, err := store.GetActionAudits("vm:42", time.Time{}, 10)
|
|
if err != nil {
|
|
t.Fatalf("GetActionAudits: %v", err)
|
|
}
|
|
if len(audits) != 0 {
|
|
t.Fatalf("sensitive-param proposals must not persist audits, got %d", len(audits))
|
|
}
|
|
}
|
|
|
|
func TestPatrolActionBrokerCapabilitiesCatalog(t *testing.T) {
|
|
h, _ := newPatrolBrokerTestHandlers(t, unified.ApprovalAdmin)
|
|
broker := NewPatrolActionBroker("default", h)
|
|
|
|
catalog, err := broker.Capabilities(context.Background(), "vm:42")
|
|
if err != nil {
|
|
t.Fatalf("Capabilities: %v", err)
|
|
}
|
|
if catalog.ResourceID != "vm:42" || len(catalog.Capabilities) != 2 {
|
|
t.Fatalf("catalog = %#v", catalog)
|
|
}
|
|
byName := map[string]aicontracts.ActionCapabilityInfo{}
|
|
for _, capability := range catalog.Capabilities {
|
|
byName[capability.Name] = capability
|
|
}
|
|
restart := byName["restart"]
|
|
if restart.MinimumApprovalLevel != string(unified.ApprovalAdmin) || restart.AutoAuthorization != string(unified.AutoAuthorizeLowRisk) || len(restart.Params) != 1 || restart.Params[0].Sensitive {
|
|
t.Fatalf("restart capability = %#v", restart)
|
|
}
|
|
join := byName["join_cluster"]
|
|
if len(join.Params) != 1 || !join.Params[0].Sensitive {
|
|
t.Fatalf("join_cluster capability must mark its token sensitive: %#v", join)
|
|
}
|
|
|
|
if _, err := broker.Capabilities(context.Background(), "vm:404"); err == nil {
|
|
t.Fatal("unknown resource must error")
|
|
}
|
|
}
|
|
|
|
func TestPatrolActionBrokerOptionalResourceScopeStillAuthorizesEligibleCapability(t *testing.T) {
|
|
h, executor := newPatrolBrokerTestHandlers(t, unified.ApprovalAdmin)
|
|
store, err := h.getStore("default")
|
|
if err != nil {
|
|
t.Fatalf("getStore: %v", err)
|
|
}
|
|
if err := store.SetResourceOperatorState(unified.ResourceOperatorState{
|
|
CanonicalID: "vm:42",
|
|
AutoRemediationPolicy: unified.AutoRemediationPolicy{
|
|
Enabled: true,
|
|
CapabilityNames: []string{"restart"},
|
|
},
|
|
SetAt: time.Now().UTC(),
|
|
}); err != nil {
|
|
t.Fatalf("SetResourceOperatorState: %v", err)
|
|
}
|
|
broker := NewPatrolActionBroker("default", h, func(context.Context, string) (PatrolActionPolicySnapshot, error) {
|
|
return PatrolActionPolicySnapshot{EffectiveAutonomyLevel: "assisted"}, nil
|
|
})
|
|
|
|
disposition, err := broker.Submit(context.Background(), patrolTestProposal())
|
|
if err != nil {
|
|
t.Fatalf("Submit: %v", err)
|
|
}
|
|
if disposition.State != string(unified.ActionStateCompleted) {
|
|
t.Fatalf("state = %q, want completed", disposition.State)
|
|
}
|
|
if executor.calls != 1 {
|
|
t.Fatalf("executor calls = %d, want 1", executor.calls)
|
|
}
|
|
audit, found, err := store.GetActionAudit(disposition.ActionID)
|
|
if err != nil || !found {
|
|
t.Fatalf("GetActionAudit: found=%v err=%v", found, err)
|
|
}
|
|
if len(audit.Approvals) != 1 || audit.Approvals[0].Actor != patrolActionPolicyActor || audit.Approvals[0].Method != unified.MethodPolicy {
|
|
t.Fatalf("policy approval = %#v", audit.Approvals)
|
|
}
|
|
|
|
// An idempotent proposal replay returns the terminal audit and cannot
|
|
// rewind or execute the action a second time.
|
|
replayed, err := broker.Submit(context.Background(), patrolTestProposal())
|
|
if err != nil || replayed.State != string(unified.ActionStateCompleted) {
|
|
t.Fatalf("replay disposition=%#v err=%v", replayed, err)
|
|
}
|
|
if executor.calls != 1 {
|
|
t.Fatalf("replay executor calls = %d, want 1", executor.calls)
|
|
}
|
|
}
|
|
|
|
func TestPatrolActionBrokerGlobalAutonomyDoesNotRequirePerResourceOptIn(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
eligibility unified.ActionAutoAuthorizationClass
|
|
snapshot PatrolActionPolicySnapshot
|
|
}{
|
|
{name: "assisted low risk", eligibility: unified.AutoAuthorizeLowRisk, snapshot: PatrolActionPolicySnapshot{EffectiveAutonomyLevel: "assisted"}},
|
|
{name: "unlocked full elevated", eligibility: unified.AutoAuthorizeElevated, snapshot: PatrolActionPolicySnapshot{EffectiveAutonomyLevel: "full", FullModeUnlocked: true}},
|
|
}
|
|
for _, tc := range tests {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
h, executor := newPatrolBrokerTestHandlersWithEligibility(t, unified.ApprovalAdmin, tc.eligibility)
|
|
broker := NewPatrolActionBroker("default", h, func(context.Context, string) (PatrolActionPolicySnapshot, error) {
|
|
return tc.snapshot, nil
|
|
})
|
|
|
|
disposition, err := broker.Submit(context.Background(), patrolTestProposal())
|
|
if err != nil {
|
|
t.Fatalf("Submit: %v", err)
|
|
}
|
|
if disposition.State != string(unified.ActionStateCompleted) || executor.calls != 1 {
|
|
t.Fatalf("global autonomy did not execute without a resource override: disposition=%#v calls=%d", disposition, executor.calls)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestPatrolActionBrokerPolicyScopeAndWindowFailClosed(t *testing.T) {
|
|
h, executor := newPatrolBrokerTestHandlers(t, unified.ApprovalAdmin)
|
|
store, err := h.getStore("default")
|
|
if err != nil {
|
|
t.Fatalf("getStore: %v", err)
|
|
}
|
|
if err := store.SetResourceOperatorState(unified.ResourceOperatorState{
|
|
CanonicalID: "vm:42",
|
|
AutoRemediationPolicy: unified.AutoRemediationPolicy{
|
|
Enabled: true,
|
|
CapabilityNames: []string{"restart"},
|
|
Window: &unified.AutoRemediationWindow{
|
|
Timezone: "UTC", StartMinute: 60, EndMinute: 120,
|
|
},
|
|
},
|
|
SetAt: time.Now().UTC(),
|
|
}); err != nil {
|
|
t.Fatalf("SetResourceOperatorState: %v", err)
|
|
}
|
|
broker := NewPatrolActionBroker("default", h, func(context.Context, string) (PatrolActionPolicySnapshot, error) {
|
|
return PatrolActionPolicySnapshot{EffectiveAutonomyLevel: "assisted"}, nil
|
|
}).(*patrolActionBroker)
|
|
broker.now = func() time.Time { return time.Date(2026, 7, 10, 3, 0, 0, 0, time.UTC) }
|
|
|
|
disposition, err := broker.Submit(context.Background(), patrolTestProposal())
|
|
if err != nil {
|
|
t.Fatalf("Submit: %v", err)
|
|
}
|
|
if disposition.State != string(unified.ActionStatePending) || executor.calls != 0 {
|
|
t.Fatalf("outside-window disposition=%#v calls=%d", disposition, executor.calls)
|
|
}
|
|
}
|
|
|
|
func TestPatrolActionBrokerAutonomyModeIsAnUpperBound(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
eligibility unified.ActionAutoAuthorizationClass
|
|
snapshot PatrolActionPolicySnapshot
|
|
wantCalls int
|
|
}{
|
|
{name: "monitor denies low risk", eligibility: unified.AutoAuthorizeLowRisk, snapshot: PatrolActionPolicySnapshot{EffectiveAutonomyLevel: "monitor"}},
|
|
{name: "approval denies low risk", eligibility: unified.AutoAuthorizeLowRisk, snapshot: PatrolActionPolicySnapshot{EffectiveAutonomyLevel: "approval"}},
|
|
{name: "assisted denies elevated", eligibility: unified.AutoAuthorizeElevated, snapshot: PatrolActionPolicySnapshot{EffectiveAutonomyLevel: "assisted"}},
|
|
{name: "locked full denies elevated", eligibility: unified.AutoAuthorizeElevated, snapshot: PatrolActionPolicySnapshot{EffectiveAutonomyLevel: "full"}},
|
|
{name: "unlocked full admits elevated", eligibility: unified.AutoAuthorizeElevated, snapshot: PatrolActionPolicySnapshot{EffectiveAutonomyLevel: "full", FullModeUnlocked: true}, wantCalls: 1},
|
|
}
|
|
for _, tc := range tests {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
h, executor := newPatrolBrokerTestHandlersWithEligibility(t, unified.ApprovalAdmin, tc.eligibility)
|
|
store, err := h.getStore("default")
|
|
if err != nil {
|
|
t.Fatalf("getStore: %v", err)
|
|
}
|
|
if err := store.SetResourceOperatorState(unified.ResourceOperatorState{
|
|
CanonicalID: "vm:42",
|
|
AutoRemediationPolicy: unified.AutoRemediationPolicy{
|
|
Enabled: true,
|
|
CapabilityNames: []string{"restart"},
|
|
},
|
|
SetAt: time.Now().UTC(),
|
|
}); err != nil {
|
|
t.Fatalf("SetResourceOperatorState: %v", err)
|
|
}
|
|
broker := NewPatrolActionBroker("default", h, func(context.Context, string) (PatrolActionPolicySnapshot, error) {
|
|
return tc.snapshot, nil
|
|
})
|
|
disposition, err := broker.Submit(context.Background(), patrolTestProposal())
|
|
if err != nil {
|
|
t.Fatalf("Submit: %v", err)
|
|
}
|
|
if executor.calls != tc.wantCalls {
|
|
t.Fatalf("executor calls = %d, want %d (disposition=%#v)", executor.calls, tc.wantCalls, disposition)
|
|
}
|
|
if tc.wantCalls == 0 && disposition.State != string(unified.ActionStatePending) {
|
|
t.Fatalf("denied state = %q, want pending_approval", disposition.State)
|
|
}
|
|
if tc.wantCalls == 1 && disposition.State != string(unified.ActionStateCompleted) {
|
|
t.Fatalf("authorized state = %q, want completed", disposition.State)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestPatrolActionBrokerDispatchTimePolicyRevocation(t *testing.T) {
|
|
h, executor := newPatrolBrokerTestHandlers(t, unified.ApprovalAdmin)
|
|
store, err := h.getStore("default")
|
|
if err != nil {
|
|
t.Fatalf("getStore: %v", err)
|
|
}
|
|
if err := store.SetResourceOperatorState(unified.ResourceOperatorState{
|
|
CanonicalID: "vm:42",
|
|
AutoRemediationPolicy: unified.AutoRemediationPolicy{
|
|
Enabled: true,
|
|
CapabilityNames: []string{"restart"},
|
|
},
|
|
SetAt: time.Now().UTC(),
|
|
}); err != nil {
|
|
t.Fatalf("SetResourceOperatorState: %v", err)
|
|
}
|
|
policyReads := 0
|
|
broker := NewPatrolActionBroker("default", h, func(context.Context, string) (PatrolActionPolicySnapshot, error) {
|
|
policyReads++
|
|
if policyReads == 1 {
|
|
return PatrolActionPolicySnapshot{EffectiveAutonomyLevel: "assisted"}, nil
|
|
}
|
|
return PatrolActionPolicySnapshot{EffectiveAutonomyLevel: "approval"}, nil
|
|
})
|
|
|
|
disposition, err := broker.Submit(context.Background(), patrolTestProposal())
|
|
if err != nil {
|
|
t.Fatalf("Submit: %v", err)
|
|
}
|
|
if disposition.State != string(unified.ActionStateFailed) || executor.calls != 0 {
|
|
t.Fatalf("revoked disposition=%#v calls=%d", disposition, executor.calls)
|
|
}
|
|
record, found, err := store.GetActionAudit(disposition.ActionID)
|
|
if err != nil || !found {
|
|
t.Fatalf("GetActionAudit: found=%v err=%v", found, err)
|
|
}
|
|
if len(record.Approvals) != 0 || record.Result == nil || !strings.HasPrefix(record.Result.ErrorMessage, "policy_authorization_revoked:") {
|
|
t.Fatalf("revoked policy left reusable authority: approvals=%#v result=%#v", record.Approvals, record.Result)
|
|
}
|
|
}
|
|
|
|
func TestPatrolActionBrokerRequiresCorrelationIdentity(t *testing.T) {
|
|
h, _ := newPatrolBrokerTestHandlers(t, unified.ApprovalAdmin)
|
|
broker := NewPatrolActionBroker("default", h)
|
|
|
|
missingFinding := patrolTestProposal()
|
|
missingFinding.FindingID = " "
|
|
if _, err := broker.Submit(context.Background(), missingFinding); err == nil {
|
|
t.Fatal("proposal without finding id must be refused")
|
|
}
|
|
|
|
missingInvestigation := patrolTestProposal()
|
|
missingInvestigation.InvestigationID = ""
|
|
if _, err := broker.Submit(context.Background(), missingInvestigation); err == nil {
|
|
t.Fatal("proposal without investigation id must be refused")
|
|
}
|
|
|
|
store, err := h.getStore("default")
|
|
if err != nil {
|
|
t.Fatalf("getStore: %v", err)
|
|
}
|
|
audits, err := store.GetActionAudits("vm:42", time.Time{}, 10)
|
|
if err != nil {
|
|
t.Fatalf("GetActionAudits: %v", err)
|
|
}
|
|
if len(audits) != 0 {
|
|
t.Fatalf("identity-less proposals must not persist audits, got %d", len(audits))
|
|
}
|
|
}
|
|
|
|
func TestPatrolActionBrokerSubmitPublishesOrgScopedTransition(t *testing.T) {
|
|
h, _ := newPatrolBrokerTestHandlers(t, unified.ApprovalAdmin)
|
|
var gotOrg string
|
|
var gotState unified.ActionState
|
|
var gotOrigin *unified.ActionOrigin
|
|
h.SetActionTransitionPublisher(func(orgID string, record unified.ActionAuditRecord) {
|
|
gotOrg = orgID
|
|
gotState = record.State
|
|
gotOrigin = record.Origin
|
|
})
|
|
broker := NewPatrolActionBroker("default", h)
|
|
|
|
if _, err := broker.Submit(context.Background(), patrolTestProposal()); err != nil {
|
|
t.Fatalf("Submit: %v", err)
|
|
}
|
|
if gotOrg != "default" {
|
|
t.Fatalf("transition org = %q, want default", gotOrg)
|
|
}
|
|
if gotState != unified.ActionStatePending {
|
|
t.Fatalf("transition state = %q, want pending_approval", gotState)
|
|
}
|
|
if gotOrigin == nil || gotOrigin.FindingID != "finding-1" {
|
|
t.Fatalf("transition origin = %#v", gotOrigin)
|
|
}
|
|
}
|
|
|
|
func TestPatrolTypedActionJourneyDetectPlanApproveExecuteVerifyAndReconcile(t *testing.T) {
|
|
resources, executor := newPatrolBrokerTestHandlers(t, unified.ApprovalAdmin)
|
|
executor.result = &unified.ExecutionResult{
|
|
Success: true,
|
|
Output: "restart dispatched",
|
|
Verification: &unified.ActionVerificationResult{
|
|
Ran: true, Success: true, RanAt: time.Now().UTC(), Note: "workload health confirmed",
|
|
},
|
|
}
|
|
aiHandler, patrol, _, _ := setupAIHandlerWithPatrol(t)
|
|
pushes := make(chan relay.PushNotificationPayload, 1)
|
|
patrol.SetPushNotifyCallback(func(payload relay.PushNotificationPayload) { pushes <- payload })
|
|
finding := addPatrolFindingForResource(t, patrol, "finding-1", time.Now().UTC(), "vm:42", "Unhealthy workload")
|
|
investigations := newTestInvestigationStore()
|
|
investigation := investigations.Create(finding.ID, "session-1")
|
|
if investigation.ID != "inv-1" {
|
|
t.Fatalf("investigation id = %q, journey proposal expects inv-1", investigation.ID)
|
|
}
|
|
aiHandler.investigationStores = map[string]aicontracts.InvestigationStore{"default": investigations}
|
|
aiHandler.SetResourceStoreProvider(resources.getStore)
|
|
resources.SetActionTransitionPublisher(aiHandler.ReconcilePatrolActionTransition)
|
|
|
|
disposition, err := NewPatrolActionBroker("default", resources).Submit(context.Background(), patrolTestProposal())
|
|
if err != nil {
|
|
t.Fatalf("Submit: %v", err)
|
|
}
|
|
planned := investigations.Get(investigation.ID)
|
|
if planned.Action == nil || planned.Action.ActionID != disposition.ActionID || planned.Action.State != string(unified.ActionStatePending) {
|
|
t.Fatalf("planned investigation action = %#v", planned.Action)
|
|
}
|
|
if planned.Outcome != aicontracts.OutcomeFixQueued {
|
|
t.Fatalf("planned outcome = %q", planned.Outcome)
|
|
}
|
|
|
|
decisionRec := httptest.NewRecorder()
|
|
decisionReq := httptest.NewRequest(http.MethodPost, "/api/actions/"+disposition.ActionID+"/decision", bytes.NewBufferString(`{"outcome":"approved","reason":"maintenance window"}`))
|
|
decisionReq.SetPathValue("id", disposition.ActionID)
|
|
decisionReq = decisionReq.WithContext(auth.WithUser(decisionReq.Context(), "operator@example.com"))
|
|
resources.HandleDecideAction(decisionRec, actionHandlerTestRequest(decisionReq, ""))
|
|
if decisionRec.Code != http.StatusOK {
|
|
t.Fatalf("decision status = %d body=%s", decisionRec.Code, decisionRec.Body.String())
|
|
}
|
|
|
|
executionRec := httptest.NewRecorder()
|
|
executionReq := httptest.NewRequest(http.MethodPost, "/api/actions/"+disposition.ActionID+"/execute", bytes.NewBufferString(`{"reason":"approved maintenance window"}`))
|
|
executionReq.SetPathValue("id", disposition.ActionID)
|
|
executionReq = executionReq.WithContext(auth.WithUser(executionReq.Context(), "operator@example.com"))
|
|
resources.HandleExecuteAction(executionRec, actionHandlerTestRequest(executionReq, ""))
|
|
if executionRec.Code != http.StatusOK {
|
|
t.Fatalf("execution status = %d body=%s", executionRec.Code, executionRec.Body.String())
|
|
}
|
|
|
|
completed := investigations.Get(investigation.ID)
|
|
if completed.Action == nil || completed.Action.State != string(unified.ActionStateCompleted) {
|
|
t.Fatalf("completed investigation action = %#v", completed.Action)
|
|
}
|
|
if completed.Outcome != aicontracts.OutcomeFixVerificationUnknown {
|
|
t.Fatalf("completed outcome = %q, want verification unknown", completed.Outcome)
|
|
}
|
|
updatedFinding := patrol.GetFindings().Get(finding.ID)
|
|
if updatedFinding == nil || updatedFinding.InvestigationOutcome != string(aicontracts.OutcomeFixVerificationUnknown) || updatedFinding.ResolvedAt != nil {
|
|
t.Fatalf("reconciled finding = %#v", updatedFinding)
|
|
}
|
|
store, err := resources.getStore("default")
|
|
if err != nil {
|
|
t.Fatalf("getStore: %v", err)
|
|
}
|
|
audit, found, err := store.GetActionAudit(disposition.ActionID)
|
|
if err != nil || !found || audit.VerificationOutcome.Status != unified.VerificationVerified {
|
|
t.Fatalf("terminal audit: found=%v err=%v audit=%#v", found, err, audit)
|
|
}
|
|
if executor.calls != 1 {
|
|
t.Fatalf("executor calls = %d, want exactly one", executor.calls)
|
|
}
|
|
select {
|
|
case push := <-pushes:
|
|
if push.ActionType != relay.PushActionViewFixResult || push.ActionID != finding.ID || push.Body != "Action completed; verification was inconclusive" {
|
|
t.Fatalf("terminal push = %#v", push)
|
|
}
|
|
default:
|
|
t.Fatal("agent-attested lifecycle did not publish an honest terminal mobile notification")
|
|
}
|
|
}
|