Fix no-op relationship_change spam from registry rebuilds

Every state update rebuilds the resource registry from scratch, which
reconstructs all relationship edges with fresh ObservedAt/LastSeenAt
stamps and fresh metadata maps. recordRegistryChanges compared the old
and new slices with reflect.DeepEqual, so every relationship-bearing
resource emitted a relationship_change row on every rebuild cycle even
when nothing changed. On the public demo this wrote roughly 450k rows
per day (1.2M of 1.6M rows were literal from==to no-ops), grew
unified_resources.db to 1.6GB in three days, starved the store's single
connection until retention pruning failed with SQLITE_BUSY, and drove
the droplet into the swap-thrash outage on 2026-07-08. Same mechanism
as issue #1496.

Compare relationship sets by edge identity instead: canonical source,
canonical target, type, and active state, order-insensitive. Volatile
provenance fields no longer count as change.

Also cap resource_changes at 200k rows during retention pruning so a
pathological writer can never grow the table unbounded inside the
30-day retention window, and record both invariants in the
unified-resources subsystem contract.
This commit is contained in:
rcourtman 2026-07-08 08:10:15 +01:00
parent c728539f07
commit 74131e56e3
6 changed files with 210 additions and 3 deletions

View file

@ -3335,6 +3335,18 @@ for provider-read breadcrumbs such as VMware tasks and events, plus the
change model instead of introducing a second event shape, and `RecordChange`
must stay idempotent by canonical change ID so poller refreshes and replayed
supplemental snapshots do not duplicate resource history.
Change emission over registry rebuilds must diff relationships by edge
identity only: canonical source, canonical target, type, and active state,
order-insensitive (`relationshipsEquivalent` in `change_emission.go`).
Volatile provenance fields (`ObservedAt`, `LastSeenAt`, `Confidence`,
`Metadata`, `Discoverer`) refresh on every rebuild cycle and must never
count as a relationship change; comparing rebuilt slices structurally
emitted a no-op `relationship_change` row per relationship-bearing resource
per cycle and grew `resource_changes` without bound (issue #1496, demo
outage 2026-07-08). Retention pruning must also enforce a hard row cap on
`resource_changes` (`maxResourceChangesRows` in `store.go`) so a
pathological writer cannot grow the table unbounded inside the time-based
retention window.
Action plans in `actions.go` still keep stale-plan protection to the canonical
`resourceVersion`, `policyVersion`, and `planHash` fields, so stale execution
checks stay in the shared resource action model rather than provider-local

View file

@ -125,7 +125,7 @@ func buildResourceChange(before Resource, beforeOK bool, after Resource, afterOK
change.From = resourceStateSummary(before)
change.To = resourceStateSummary(after)
change.Reason = "resource state changed"
case !equalStringPtr(before.ParentID, after.ParentID) || !reflect.DeepEqual(before.Relationships, after.Relationships):
case !equalStringPtr(before.ParentID, after.ParentID) || !relationshipsEquivalent(before.Relationships, after.Relationships):
change.Kind = ChangeRelationship
change.From = resourceRelationSummary(before)
change.To = resourceRelationSummary(after)
@ -168,7 +168,7 @@ func resourceChangedFields(before, after Resource) []string {
if !equalStringPtr(before.ParentID, after.ParentID) {
changed = append(changed, "parentId")
}
if !reflect.DeepEqual(before.Relationships, after.Relationships) {
if !relationshipsEquivalent(before.Relationships, after.Relationships) {
changed = append(changed, "relationships")
}
if !reflect.DeepEqual(before.Capabilities, after.Capabilities) {
@ -205,6 +205,44 @@ func resourceChangedFields(before, after Resource) []string {
return changed
}
// relationshipsEquivalent reports whether two relationship sets describe the
// same edges. Registry rebuilds reconstruct every relationship with fresh
// ObservedAt/LastSeenAt stamps and metadata maps, so comparing with
// reflect.DeepEqual emitted a no-op relationship_change row for every
// relationship-bearing resource on every rebuild cycle (the unbounded
// unified_resources.db growth behind issue #1496). Only edge identity —
// canonical source, canonical target, type, and active state — counts as
// change.
func relationshipsEquivalent(a, b []ResourceRelationship) bool {
if len(a) != len(b) {
return false
}
if len(a) == 0 {
return true
}
edgeKeys := func(relationships []ResourceRelationship) []string {
keys := make([]string, 0, len(relationships))
for _, relationship := range relationships {
keys = append(keys, strings.Join([]string{
CanonicalResourceID(relationship.SourceID),
CanonicalResourceID(relationship.TargetID),
string(relationship.Type),
fmt.Sprintf("%t", relationship.Active),
}, "\x1f"))
}
sort.Strings(keys)
return keys
}
aKeys := edgeKeys(a)
bKeys := edgeKeys(b)
for i := range aKeys {
if aKeys[i] != bKeys[i] {
return false
}
}
return true
}
func dockerCommandChanged(before, after Resource) bool {
var beforeCommand, afterCommand any
if before.Docker != nil {

View file

@ -18,6 +18,90 @@ func TestBuildResourceChange_ReturnsNilWhenUnchanged(t *testing.T) {
}
}
func TestBuildResourceChange_IgnoresVolatileRelationshipFields(t *testing.T) {
baseTime := time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC)
before := Resource{
ID: "app-container:1",
Type: ResourceTypeAppContainer,
Name: "web",
Status: StatusOnline,
Relationships: []ResourceRelationship{
{
SourceID: "app-container:1", TargetID: "docker-network:1", Type: RelAttachedTo,
Confidence: 1, Active: true, Discoverer: "docker_adapter",
ObservedAt: baseTime, LastSeenAt: baseTime,
Metadata: map[string]any{"network": "bridge"},
},
{
SourceID: "app-container:1", TargetID: "agent:1", Type: RelRunsOn,
Confidence: 1, Active: true, Discoverer: "docker_adapter",
ObservedAt: baseTime, LastSeenAt: baseTime,
},
},
}
// Registry rebuilds reconstruct the same edges with fresh timestamps,
// fresh metadata maps, and possibly different ordering.
after := before
rebuiltTime := baseTime.Add(time.Hour)
after.Relationships = []ResourceRelationship{
{
SourceID: "app-container:1", TargetID: "agent:1", Type: RelRunsOn,
Confidence: 1, Active: true, Discoverer: "docker_adapter",
ObservedAt: rebuiltTime, LastSeenAt: rebuiltTime,
},
{
SourceID: "app-container:1", TargetID: "docker-network:1", Type: RelAttachedTo,
Confidence: 1, Active: true, Discoverer: "docker_adapter",
ObservedAt: rebuiltTime, LastSeenAt: rebuiltTime,
Metadata: map[string]any{"network": "bridge"},
},
}
if change := buildResourceChange(before, true, after, true, rebuiltTime, nil, SourcePulseDiff, ""); change != nil {
t.Fatalf("expected nil change for rebuilt-but-identical relationships, got %+v", change)
}
}
func TestBuildResourceChange_DetectsRealRelationshipChanges(t *testing.T) {
edge := ResourceRelationship{
SourceID: "app-container:1", TargetID: "docker-network:1", Type: RelAttachedTo,
Confidence: 1, Active: true, Discoverer: "docker_adapter",
}
before := Resource{
ID: "app-container:1",
Type: ResourceTypeAppContainer,
Name: "web",
Status: StatusOnline,
Relationships: []ResourceRelationship{edge},
}
retargeted := edge
retargeted.TargetID = "docker-network:2"
deactivated := edge
deactivated.Active = false
cases := []struct {
name string
after []ResourceRelationship
}{
{"edge added", []ResourceRelationship{edge, {SourceID: "app-container:1", TargetID: "agent:1", Type: RelRunsOn, Confidence: 1, Active: true}}},
{"edge removed", nil},
{"edge retargeted", []ResourceRelationship{retargeted}},
{"edge deactivated", []ResourceRelationship{deactivated}},
}
for _, tc := range cases {
after := before
after.Relationships = tc.after
change := buildResourceChange(before, true, after, true, time.Now().UTC(), nil, SourcePulseDiff, "")
if change == nil {
t.Fatalf("%s: expected relationship change, got nil", tc.name)
}
if change.Kind != ChangeRelationship {
t.Fatalf("%s: Kind = %q, want %q", tc.name, change.Kind, ChangeRelationship)
}
}
}
func TestBuildResourceChange_ClassifiesStateTransition(t *testing.T) {
before := Resource{
ID: "vm:1",

View file

@ -1153,8 +1153,9 @@ func TestResourceChangeEmissionCoversRelationshipAndCapabilityChanges(t *testing
"change.RelatedResources = relatedResourceIDs(change.ResourceID, before, after)",
"case resourceRestartChanged(before, after):",
"case resourceIncidentChanged(before, after):",
"if !reflect.DeepEqual(before.Relationships, after.Relationships) {",
"if !relationshipsEquivalent(before.Relationships, after.Relationships) {",
"changed = append(changed, \"relationships\")",
"func relationshipsEquivalent(a, b []ResourceRelationship) bool {",
"if resourceIncidentChanged(before, after) {",
"changed = append(changed, \"incidents\")",
"if dockerRestartChanged(before, after) {",

View file

@ -955,6 +955,12 @@ const (
retentionInterval = 1 * time.Hour
initialRetentionDelay = 30 * time.Second
maxUnifiedReclaimPages = 50000
// maxResourceChangesRows bounds resource_changes even inside the
// retention window: time-based pruning alone cannot contain a
// pathological writer (the demo hit 1.6M rows in three days), and an
// unbounded table starves the store's single connection until prunes
// themselves fail with SQLITE_BUSY.
maxResourceChangesRows = 200000
)
// migrateAutoVacuum ensures the database uses incremental auto-vacuum so that
@ -1038,6 +1044,24 @@ func (s *SQLiteResourceStore) startRetentionLoop() chan struct{} {
return stop
}
// capResourceChanges deletes the oldest resource_changes rows beyond limit,
// keeping the newest rows by observed_at.
func (s *SQLiteResourceStore) capResourceChanges(limit int) (int64, error) {
res, err := s.db.Exec(
`DELETE FROM resource_changes WHERE rowid IN (
SELECT rowid FROM resource_changes
ORDER BY observed_at DESC
LIMIT -1 OFFSET ?
)`,
limit,
)
if err != nil {
return 0, err
}
affected, _ := res.RowsAffected()
return affected, nil
}
func (s *SQLiteResourceStore) pruneOldRecords() {
now := time.Now()
changesCutoff := now.Add(-resourceChangesRetention)
@ -1059,6 +1083,12 @@ func (s *SQLiteResourceStore) pruneOldRecords() {
totalDeleted += affected
}
if affected, err := s.capResourceChanges(maxResourceChangesRows); err != nil {
log.Printf("unifiedresources: failed to cap resource_changes: %v", err)
} else if affected > 0 {
totalDeleted += affected
}
res, err = s.db.Exec(
`DELETE FROM action_audits WHERE created_at < ?`,
auditsCutoff.UTC().Format(tsFmt),

View file

@ -2498,3 +2498,45 @@ func TestPruneOldRecords_DeletesExpiredChangesAndAudits(t *testing.T) {
t.Fatalf("expected only audit-recent to survive, got %d: %+v", len(auditResults), auditResults)
}
}
func TestCapResourceChanges_KeepsNewestRows(t *testing.T) {
store := newTestStore(t)
base := time.Now().Add(-10 * time.Hour)
for i := 0; i < 10; i++ {
change := ResourceChange{
ID: fmt.Sprintf("change-%d", i),
ObservedAt: base.Add(time.Duration(i) * time.Hour),
ResourceID: "vm:100",
Kind: ChangeStateTransition,
SourceType: SourcePulseDiff,
Confidence: ConfidenceHigh,
}
if err := store.RecordChange(change); err != nil {
t.Fatalf("RecordChange(%s): %v", change.ID, err)
}
}
deleted, err := store.capResourceChanges(3)
if err != nil {
t.Fatalf("capResourceChanges: %v", err)
}
if deleted != 7 {
t.Fatalf("deleted = %d, want 7", deleted)
}
remaining, err := store.GetRecentChanges("vm:100", time.Time{}, 0)
if err != nil {
t.Fatalf("GetRecentChanges after cap: %v", err)
}
if len(remaining) != 3 {
t.Fatalf("expected 3 surviving rows, got %d", len(remaining))
}
for _, change := range remaining {
switch change.ID {
case "change-7", "change-8", "change-9":
default:
t.Fatalf("unexpected survivor %s; want the newest three", change.ID)
}
}
}