Pulse/internal/alerts/migration_characterization_test.go

package alerts

import (
	"testing"
	"time"

	alertspecs "github.com/rcourtman/pulse-go-rewrite/internal/alerts/specs"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)

func characterizationBaseConfig() AlertConfig {
	return AlertConfig{
		Enabled:         true,
		ActivationState: ActivationActive,
		GuestDefaults: ThresholdConfig{
			CPU:    &HysteresisThreshold{Trigger: 80, Clear: 75},
			Memory: &HysteresisThreshold{Trigger: 85, Clear: 80},
			Disk:   &HysteresisThreshold{Trigger: 90, Clear: 85},
		},
		NodeDefaults: ThresholdConfig{
			CPU:    &HysteresisThreshold{Trigger: 80, Clear: 75},
			Memory: &HysteresisThreshold{Trigger: 85, Clear: 80},
			Disk:   &HysteresisThreshold{Trigger: 90, Clear: 85},
		},
		AgentDefaults: ThresholdConfig{
			CPU:    &HysteresisThreshold{Trigger: 80, Clear: 75},
			Memory: &HysteresisThreshold{Trigger: 85, Clear: 80},
			Disk:   &HysteresisThreshold{Trigger: 90, Clear: 85},
		},
		PBSDefaults: ThresholdConfig{
			CPU:    &HysteresisThreshold{Trigger: 80, Clear: 75},
			Memory: &HysteresisThreshold{Trigger: 85, Clear: 80},
		},
		StorageDefault: HysteresisThreshold{Trigger: 85, Clear: 80},
		Overrides:      map[string]ThresholdConfig{},
		TimeThresholds: map[string]int{},
	}
}

func newCharacterizationManager(t *testing.T, cfg AlertConfig) *Manager {
	t.Helper()

	m := newTestManager(t)
	m.UpdateConfig(cfg)

	// Force deterministic immediate evaluation for characterization tests.
	m.mu.Lock()
	m.config.TimeThresholds = map[string]int{}
	m.config.MetricTimeThresholds = nil
	m.mu.Unlock()

	m.ClearActiveAlerts()
	return m
}

func activeAlert(t *testing.T, m *Manager, alertID string) *Alert {
	t.Helper()

	m.mu.RLock()
	alert := testRequireActiveAlert(t, m, alertID)
	m.mu.RUnlock()
	if alert != nil {
		return alert.Clone()
	}

	t.Fatalf("expected active alert %q, got active alerts %v", alertID, alertKeys(m))
	return nil
}

func testVM(resourceID string, vmID int, name, node, instance, status string, cpu float64, tags ...string) models.VM {
	return models.VM{
		ID:       resourceID,
		VMID:     vmID,
		Name:     name,
		Node:     node,
		Instance: instance,
		Status:   status,
		CPU:      cpu,
		Tags:     tags,
	}
}

func TestAlertCharacterizationCanonicalGuestIdentityAcrossTypedAndUnifiedChecks(t *testing.T) {
	resourceID := BuildGuestKey("pve1", "node1", 101)
	specID := canonicalMetricSpecID(resourceID, "cpu")
	alertID := buildCanonicalStateID(resourceID, specID)

	m := newCharacterizationManager(t, characterizationBaseConfig())
	m.CheckGuest(testVM(resourceID, 101, "app01", "node1", "pve1", "running", 0.85), "pve1")
	typedAlert := activeAlert(t, m, alertID)

	m.ClearActiveAlerts()
	m.CheckUnifiedResource(&UnifiedResourceInput{
		ID:       resourceID,
		Type:     "vm",
		Name:     "app01",
		Node:     "node1",
		Instance: "pve1",
		CPU:      &UnifiedResourceMetric{Percent: 85},
	})
	unifiedAlert := activeAlert(t, m, alertID)

	if typedAlert.ResourceID != resourceID {
		t.Fatalf("typed ResourceID = %q, want %q", typedAlert.ResourceID, resourceID)
	}
	if unifiedAlert.ResourceID != resourceID {
		t.Fatalf("unified ResourceID = %q, want %q", unifiedAlert.ResourceID, resourceID)
	}
	if typedAlert.ID != alertID || unifiedAlert.ID != alertID {
		t.Fatalf("expected stable alert ID %q, got typed=%q unified=%q", alertID, typedAlert.ID, unifiedAlert.ID)
	}
	if typedAlert.CanonicalSpecID != specID || unifiedAlert.CanonicalSpecID != specID {
		t.Fatalf("expected canonical spec id %q, got typed=%q unified=%q", specID, typedAlert.CanonicalSpecID, unifiedAlert.CanonicalSpecID)
	}
	if typedAlert.CanonicalState != alertID || unifiedAlert.CanonicalState != alertID {
		t.Fatalf("expected canonical state %q, got typed=%q unified=%q", alertID, typedAlert.CanonicalState, unifiedAlert.CanonicalState)
	}
}

func TestAlertCharacterizationGetActiveAlertsExportsCanonicalIdentity(t *testing.T) {
	resourceID := BuildGuestKey("pve1", "node1", 101)
	canonicalState := canonicalMetricStateID(resourceID, "cpu")

	m := newCharacterizationManager(t, characterizationBaseConfig())
	m.CheckGuest(testVM(resourceID, 101, "app01", "node1", "pve1", "running", 0.85), "pve1")

	alerts := m.GetActiveAlerts()
	if len(alerts) != 1 {
		t.Fatalf("expected 1 active alert, got %d", len(alerts))
	}
	if alerts[0].ID != canonicalState {
		t.Fatalf("GetActiveAlerts() ID = %q, want canonical ID %q", alerts[0].ID, canonicalState)
	}
}

func TestAlertCharacterizationAcknowledgmentSurvivesAlertIDChangeForSameCanonicalState(t *testing.T) {
	resourceID := BuildGuestKey("pve1", "node1", 101)
	oldAlertID := "legacy-" + resourceID + "-cpu"
	newAlertID := resourceID + "-cpu"

	m := newCharacterizationManager(t, characterizationBaseConfig())

	m.mu.Lock()
	oldAlert := &Alert{
		ID:         oldAlertID,
		Type:       "cpu",
		Level:      AlertLevelWarning,
		ResourceID: resourceID,
		Message:    "legacy alert",
		StartTime:  time.Now().Add(-5 * time.Minute),
		LastSeen:   time.Now().Add(-1 * time.Minute),
	}
	applyCanonicalIdentity(oldAlert, newAlertID, "metric-threshold")
	m.activeAlerts[oldAlertID] = oldAlert
	m.mu.Unlock()

	if err := m.AcknowledgeAlert(oldAlertID, "alice"); err != nil {
		t.Fatalf("AcknowledgeAlert() error = %v", err)
	}

	m.mu.Lock()
	m.removeActiveAlertNoLock(oldAlertID)
	replacement := &Alert{
		ID:         newAlertID,
		Type:       "cpu",
		Level:      AlertLevelWarning,
		ResourceID: resourceID,
		Message:    "canonical alert",
		StartTime:  time.Now(),
		LastSeen:   time.Now(),
	}
	applyCanonicalIdentity(replacement, newAlertID, "metric-threshold")
	m.preserveAlertState(newAlertID, replacement)
	m.activeAlerts[newAlertID] = replacement
	m.mu.Unlock()

	alert := activeAlert(t, m, newAlertID)
	if !alert.Acknowledged {
		t.Fatal("expected acknowledgment to survive alert ID change")
	}
	if alert.AckUser != "alice" {
		t.Fatalf("AckUser = %q, want alice", alert.AckUser)
	}
	if alert.CanonicalState != resourceID+"::"+newAlertID {
		t.Fatalf("CanonicalState = %q, want %q", alert.CanonicalState, resourceID+"::"+newAlertID)
	}
}

func TestAlertCharacterizationHistoryUpdateUsesCanonicalStateAcrossAlertIDChange(t *testing.T) {
	resourceID := BuildGuestKey("pve1", "node1", 101)
	oldAlertID := "legacy-" + resourceID + "-cpu"
	newAlertID := resourceID + "-cpu"
	lastSeen := time.Now()

	m := newCharacterizationManager(t, characterizationBaseConfig())

	oldHistoryAlert := Alert{
		ID:         oldAlertID,
		Type:       "cpu",
		ResourceID: resourceID,
		StartTime:  time.Now().Add(-10 * time.Minute),
		LastSeen:   time.Now().Add(-5 * time.Minute),
	}
	applyCanonicalIdentity(&oldHistoryAlert, newAlertID, "metric-threshold")
	m.historyManager.AddAlert(oldHistoryAlert)

	current := &Alert{
		ID:         newAlertID,
		Type:       "cpu",
		ResourceID: resourceID,
		StartTime:  time.Now().Add(-2 * time.Minute),
		LastSeen:   lastSeen,
	}
	applyCanonicalIdentity(current, newAlertID, "metric-threshold")

	m.historyManager.UpdateAlertLastSeenForAlert(current, lastSeen)

	history := m.GetAlertHistory(10)
	if len(history) == 0 {
		t.Fatalf("expected history entry")
	}
	if !history[0].LastSeen.Equal(lastSeen) {
		t.Fatalf("LastSeen = %v, want %v", history[0].LastSeen, lastSeen)
	}
	expectedCanonicalState := buildCanonicalStateID(resourceID, newAlertID)
	if history[0].ID != expectedCanonicalState {
		t.Fatalf("history alert ID = %q, want canonical ID %q", history[0].ID, expectedCanonicalState)
	}
}

func TestAlertCharacterizationRecentSuppressionSurvivesAlertIDChangeForSameCanonicalState(t *testing.T) {
	resourceID := BuildGuestKey("pve1", "node1", 101)
	oldAlertID := "legacy-" + resourceID + "-cpu"
	newAlertID := resourceID + "-cpu"
	clear := 75.0
	critical := 90.0

	m := newCharacterizationManager(t, characterizationBaseConfig())
	m.mu.Lock()
	m.config.MinimumDelta = 5
	m.config.SuppressionWindow = 10
	recent := &Alert{
		ID:         oldAlertID,
		Type:       "cpu",
		Level:      AlertLevelWarning,
		ResourceID: resourceID,
		Value:      90,
		StartTime:  time.Now().Add(-1 * time.Minute),
		LastSeen:   time.Now().Add(-30 * time.Second),
	}
	applyCanonicalIdentity(recent, newAlertID, "metric-threshold")
	m.recentAlerts[recent.CanonicalState] = recent
	m.mu.Unlock()

	spec := alertspecs.ResourceAlertSpec{
		ID:           newAlertID,
		ResourceID:   resourceID,
		ResourceType: unifiedresources.ResourceTypeVM,
		Kind:         alertspecs.AlertSpecKindMetricThreshold,
		Severity:     alertspecs.AlertSeverityWarning,
		MetricThreshold: &alertspecs.MetricThresholdSpec{
			Metric:    "cpu",
			Direction: alertspecs.ThresholdDirectionAbove,
			Trigger:   80,
			Recovery:  &clear,
			Critical:  &critical,
		},
	}

	m.evaluateCanonicalMetricAlert(spec, "app01", "node1", "pve1", "cpu", 91, &HysteresisThreshold{Trigger: 80, Clear: 75}, nil)

	assertAlertMissing(t, m, newAlertID)

	m.mu.RLock()
	suppressedUntil, ok := m.suppressedUntil[resourceID+"::"+newAlertID]
	m.mu.RUnlock()
	if !ok {
		t.Fatal("expected canonical suppression window to be recorded")
	}
	if time.Until(suppressedUntil) <= 0 {
		t.Fatal("expected canonical suppression window to be in the future")
	}
}

func TestAlertCharacterizationAcknowledgeByCanonicalStateAlias(t *testing.T) {
	resourceID := BuildGuestKey("pve1", "node1", 101)
	alertID := canonicalMetricStateID(resourceID, "cpu")
	canonicalState := alertID

	m := newCharacterizationManager(t, characterizationBaseConfig())
	m.CheckGuest(testVM(resourceID, 101, "app01", "node1", "pve1", "running", 0.95), "pve1")

	if err := m.AcknowledgeAlert(canonicalState, "alice"); err != nil {
		t.Fatalf("AcknowledgeAlert(%q) error = %v", canonicalState, err)
	}

	alert := activeAlert(t, m, alertID)
	if !alert.Acknowledged {
		t.Fatal("expected alert to be acknowledged through canonical state alias")
	}
	if alert.AckUser != "alice" {
		t.Fatalf("AckUser = %q, want alice", alert.AckUser)
	}

	m.mu.RLock()
	_, legacyAck := m.ackState[alertID]
	record, canonicalAck := m.ackStateByCanonical[canonicalState]
	m.mu.RUnlock()
	if legacyAck {
		t.Fatalf("expected canonical alert acknowledgment to be keyed by canonical state, not legacy alert ID")
	}
	if !canonicalAck || !record.acknowledged || record.user != "alice" {
		t.Fatalf("expected canonical ack record for %q, got %+v exists=%t", canonicalState, record, canonicalAck)
	}
}

func TestAlertCharacterizationGuestThresholdPrecedence(t *testing.T) {
	overrideGuestID := BuildGuestKey("pve1", "node1", 101)
	ruleGuestID := BuildGuestKey("pve1", "node1", 102)

	cfg := characterizationBaseConfig()
	cfg.GuestDefaults.CPU = &HysteresisThreshold{Trigger: 90, Clear: 85}
	cfg.CustomRules = []CustomAlertRule{
		{
			Name:     "named-apps",
			Enabled:  true,
			Priority: 10,
			FilterConditions: FilterStack{
				LogicalOperator: "AND",
				Filters: []FilterCondition{
					{Type: "text", Field: "name", Value: "app"},
				},
			},
			Thresholds: ThresholdConfig{
				CPU: &HysteresisThreshold{Trigger: 70, Clear: 65},
			},
		},
	}
	cfg.Overrides[overrideGuestID] = ThresholdConfig{
		CPU: &HysteresisThreshold{Trigger: 95, Clear: 90},
	}

	m := newCharacterizationManager(t, cfg)
	m.CheckGuest(testVM(ruleGuestID, 102, "app-rule", "node1", "pve1", "running", 0.80), "pve1")
	m.CheckGuest(testVM(overrideGuestID, 101, "app-override", "node1", "pve1", "running", 0.80), "pve1")

	assertAlertPresent(t, m, canonicalMetricStateID(ruleGuestID, "cpu"))
	assertAlertMissing(t, m, canonicalMetricStateID(overrideGuestID, "cpu"))
}

func TestAlertCharacterizationDisableConnectivitySuppressesPoweredOffButNotMetrics(t *testing.T) {
	resourceID := BuildGuestKey("pve1", "node1", 101)
	cfg := characterizationBaseConfig()
	cfg.Overrides[resourceID] = ThresholdConfig{DisableConnectivity: true}

	m := newCharacterizationManager(t, cfg)
	stopped := testVM(resourceID, 101, "app01", "node1", "pve1", "stopped", 0)

	m.CheckGuest(stopped, "pve1")
	m.CheckGuest(stopped, "pve1")

	assertAlertMissing(t, m, "guest-powered-off-"+resourceID)

	m.mu.RLock()
	_, hasConfirmations := m.offlineConfirmations[resourceID]
	m.mu.RUnlock()
	if hasConfirmations {
		t.Fatalf("expected powered-off tracking to stay clear when connectivity is disabled")
	}

	m.CheckGuest(testVM(resourceID, 101, "app01", "node1", "pve1", "running", 0.95), "pve1")
	assertAlertPresent(t, m, canonicalMetricStateID(resourceID, "cpu"))
}

func TestAlertCharacterizationReevaluatesAlertsWhenConfigChanges(t *testing.T) {
	resourceID := BuildGuestKey("pve1", "node1", 101)
	alertID := canonicalMetricStateID(resourceID, "cpu")
	cfg := characterizationBaseConfig()

	m := newCharacterizationManager(t, cfg)
	resolved := make(chan string, 1)
	m.SetResolvedCallback(func(id string) {
		select {
		case resolved <- id:
		default:
		}
	})

	m.CheckUnifiedResource(&UnifiedResourceInput{
		ID:       resourceID,
		Type:     "vm",
		Name:     "app01",
		Node:     "node1",
		Instance: "pve1",
		CPU:      &UnifiedResourceMetric{Percent: 87},
	})
	assertAlertPresent(t, m, alertID)

	updated := cfg
	updated.GuestDefaults.CPU = &HysteresisThreshold{Trigger: 90, Clear: 85}
	m.UpdateConfig(updated)

	select {
	case got := <-resolved:
		expectedResolvedID := alertID
		if got != expectedResolvedID {
			t.Fatalf("resolved callback = %q, want %q", got, expectedResolvedID)
		}
	case <-time.After(250 * time.Millisecond):
		t.Fatalf("expected alert %q to resolve after config change", alertID)
	}

	assertAlertMissing(t, m, alertID)

	m.resolvedMutex.RLock()
	_, wasResolved := testLookupResolvedAlert(t, m, alertID)
	m.resolvedMutex.RUnlock()
	if !wasResolved {
		t.Fatalf("expected %q in recently resolved after config change", alertID)
	}
}

func TestAlertCharacterizationPoweredOffLifecycle(t *testing.T) {
	resourceID := BuildGuestKey("pve1", "node1", 101)
	alertID := "guest-powered-off-" + resourceID
	cfg := characterizationBaseConfig()
	m := newCharacterizationManager(t, cfg)

	running := testVM(resourceID, 101, "app01", "node1", "pve1", "running", 0.95)
	stopped := testVM(resourceID, 101, "app01", "node1", "pve1", "stopped", 0)
	paused := testVM(resourceID, 101, "app01", "node1", "pve1", "paused", 0)

	m.CheckGuest(running, "pve1")
	assertAlertPresent(t, m, canonicalMetricStateID(resourceID, "cpu"))

	m.CheckGuest(stopped, "pve1")
	m.CheckGuest(stopped, "pve1")

	assertAlertMissing(t, m, canonicalMetricStateID(resourceID, "cpu"))
	assertAlertPresent(t, m, alertID)

	m.CheckGuest(paused, "pve1")
	assertAlertMissing(t, m, alertID)

	m.CheckGuest(stopped, "pve1")
	m.CheckGuest(stopped, "pve1")
	assertAlertPresent(t, m, alertID)

	m.CheckGuest(running, "pve1")
	assertAlertMissing(t, m, alertID)
}

func TestAlertCharacterizationAcknowledgmentSurvivesPoweredOffReappearance(t *testing.T) {
	resourceID := BuildGuestKey("pve1", "node1", 101)
	alertID := canonicalPoweredStateStateID(resourceID)
	m := newCharacterizationManager(t, characterizationBaseConfig())

	stopped := testVM(resourceID, 101, "app01", "node1", "pve1", "stopped", 0)
	running := testVM(resourceID, 101, "app01", "node1", "pve1", "running", 0.10)

	m.CheckGuest(stopped, "pve1")
	m.CheckGuest(stopped, "pve1")
	if err := m.AcknowledgeAlert(alertID, "alice"); err != nil {
		t.Fatalf("AcknowledgeAlert(%q) failed: %v", alertID, err)
	}

	acknowledged := activeAlert(t, m, alertID)
	if acknowledged.AckTime == nil {
		t.Fatalf("expected acknowledged alert to record ack time")
	}

	m.CheckGuest(running, "pve1")
	assertAlertMissing(t, m, alertID)

	m.CheckGuest(stopped, "pve1")
	m.CheckGuest(stopped, "pve1")

	reappeared := activeAlert(t, m, alertID)
	if !reappeared.Acknowledged {
		t.Fatalf("expected acknowledgment to be restored when the same powered-off identity reappears")
	}
	if reappeared.AckUser != "alice" {
		t.Fatalf("AckUser = %q, want %q", reappeared.AckUser, "alice")
	}
	if reappeared.AckTime == nil || !reappeared.AckTime.Equal(*acknowledged.AckTime) {
		t.Fatalf("expected AckTime to be preserved across clear/recreate, got %v want %v", reappeared.AckTime, acknowledged.AckTime)
	}
}

func TestAlertCharacterizationSuppressTagClearsMetricSuppressionIdentity(t *testing.T) {
	resourceID := BuildGuestKey("pve1", "node1", 101)
	alertID := canonicalMetricStateID(resourceID, "cpu")
	trackingKey := alertID
	cfg := characterizationBaseConfig()
	cfg.SuppressionWindow = 30
	cfg.MinimumDelta = 5

	m := newCharacterizationManager(t, cfg)
	running := testVM(resourceID, 101, "app01", "node1", "pve1", "running", 0.90)
	cleared := testVM(resourceID, 101, "app01", "node1", "pve1", "running", 0.70)
	similarSpike := testVM(resourceID, 101, "app01", "node1", "pve1", "running", 0.91)
	suppressedByTag := testVM(resourceID, 101, "app01", "node1", "pve1", "running", 0.91, "pulse-no-alerts")

	m.CheckGuest(running, "pve1")
	assertAlertPresent(t, m, alertID)

	m.CheckGuest(cleared, "pve1")
	assertAlertMissing(t, m, alertID)

	m.CheckGuest(similarSpike, "pve1")
	assertAlertMissing(t, m, alertID)

	m.mu.RLock()
	_, isSuppressed := m.suppressedUntil[trackingKey]
	m.mu.RUnlock()
	if !isSuppressed {
		t.Fatalf("expected similar retrigger to be suppressed for canonical tracking key %q", trackingKey)
	}

	m.CheckGuest(suppressedByTag, "pve1")

	m.mu.RLock()
	_, isSuppressed = m.suppressedUntil[trackingKey]
	m.mu.RUnlock()
	if isSuppressed {
		t.Fatalf("expected pulse-no-alerts suppression to clear stale suppression state for %q", trackingKey)
	}

	m.CheckGuest(similarSpike, "pve1")
	assertAlertPresent(t, m, alertID)
}

func TestAlertCharacterizationResolvedLookupByCanonicalStateAlias(t *testing.T) {
	resourceID := BuildGuestKey("pve1", "node1", 101)
	alertID := canonicalMetricStateID(resourceID, "cpu")
	canonicalState := alertID

	m := newCharacterizationManager(t, characterizationBaseConfig())
	m.CheckGuest(testVM(resourceID, 101, "app01", "node1", "pve1", "running", 0.95), "pve1")
	m.CheckGuest(testVM(resourceID, 101, "app01", "node1", "pve1", "running", 0.70), "pve1")

	resolved := m.GetResolvedAlert(canonicalState)
	if resolved == nil || resolved.Alert == nil {
		t.Fatalf("expected resolved alert lookup by canonical state %q", canonicalState)
	}
	if resolved.Alert.ID != canonicalState {
		t.Fatalf("resolved alert ID = %q, want canonical ID %q", resolved.Alert.ID, canonicalState)
	}
}

func TestAlertCharacterizationGuestMetricAlertMigratesAcrossNodeMove(t *testing.T) {
	oldResourceID := BuildGuestKey("pve1", "node1", 101)
	newResourceID := BuildGuestKey("pve1", "node2", 101)
	oldState := canonicalMetricStateID(oldResourceID, "cpu")
	newState := canonicalMetricStateID(newResourceID, "cpu")

	m := newCharacterizationManager(t, characterizationBaseConfig())
	m.CheckGuest(testVM(oldResourceID, 101, "app01", "node1", "pve1", "running", 0.95), "pve1")

	if err := m.AcknowledgeAlert(oldState, "alice"); err != nil {
		t.Fatalf("AcknowledgeAlert(%q) error = %v", oldState, err)
	}

	m.mu.Lock()
	m.suppressedUntil[oldState] = time.Now().Add(time.Hour)
	m.alertRateLimit[oldState] = []time.Time{time.Now()}
	m.flappingHistory[oldState] = []time.Time{time.Now()}
	m.flappingActive[oldState] = true
	m.mu.Unlock()

	m.CheckGuest(testVM(newResourceID, 101, "app01", "node2", "pve1", "running", 0.92), "pve1")

	assertAlertMissing(t, m, oldState)
	alert := activeAlert(t, m, newState)
	if alert.ResourceID != newResourceID {
		t.Fatalf("ResourceID = %q, want %q", alert.ResourceID, newResourceID)
	}
	if alert.Node != "node2" {
		t.Fatalf("Node = %q, want node2", alert.Node)
	}
	if !alert.Acknowledged || alert.AckUser != "alice" {
		t.Fatalf("expected acknowledgment to follow migrated alert, got acknowledged=%t user=%q", alert.Acknowledged, alert.AckUser)
	}

	m.mu.RLock()
	_, oldAck := m.ackStateByCanonical[oldState]
	record, newAck := m.ackStateByCanonical[newState]
	_, oldSuppression := m.suppressedUntil[oldState]
	_, newSuppression := m.suppressedUntil[newState]
	_, oldRate := m.alertRateLimit[oldState]
	_, newRate := m.alertRateLimit[newState]
	_, oldFlapping := m.flappingHistory[oldState]
	_, newFlapping := m.flappingHistory[newState]
	m.mu.RUnlock()

	if oldAck || !newAck || !record.acknowledged || record.user != "alice" {
		t.Fatalf("expected canonical ack record to migrate, old=%t new=%t record=%+v", oldAck, newAck, record)
	}
	if oldSuppression || !newSuppression || oldRate || !newRate || oldFlapping || !newFlapping {
		t.Fatalf(
			"expected tracking state to migrate, suppression old=%t new=%t rate old=%t new=%t flapping old=%t new=%t",
			oldSuppression,
			newSuppression,
			oldRate,
			newRate,
			oldFlapping,
			newFlapping,
		)
	}

	history := m.GetAlertHistory(5)
	if len(history) != 1 || history[0].ID != newState {
		t.Fatalf("expected migrated history entry under %q, got %#v", newState, history)
	}
}

func TestAlertCharacterizationGuestMetricResolutionUsesCurrentNodeIdentityAfterMove(t *testing.T) {
	oldResourceID := BuildGuestKey("pve1", "node1", 101)
	newResourceID := BuildGuestKey("pve1", "node2", 101)
	newState := canonicalMetricStateID(newResourceID, "cpu")

	m := newCharacterizationManager(t, characterizationBaseConfig())
	m.CheckGuest(testVM(oldResourceID, 101, "app01", "node1", "pve1", "running", 0.95), "pve1")
	m.CheckGuest(testVM(newResourceID, 101, "app01", "node2", "pve1", "running", 0.70), "pve1")

	assertAlertMissing(t, m, canonicalMetricStateID(oldResourceID, "cpu"))
	assertAlertMissing(t, m, newState)

	resolved := m.GetResolvedAlert(newState)
	if resolved == nil || resolved.Alert == nil {
		t.Fatalf("expected resolved alert lookup by migrated canonical state %q", newState)
	}
	if resolved.Alert.ResourceID != newResourceID {
		t.Fatalf("resolved alert resource ID = %q, want %q", resolved.Alert.ResourceID, newResourceID)
	}

	history := m.GetAlertHistory(5)
	if len(history) != 1 || history[0].ID != newState {
		t.Fatalf("expected resolved history entry under %q, got %#v", newState, history)
	}
}

func TestAlertCharacterizationGuestPerDiskMetricAlertMigratesAcrossNodeMove(t *testing.T) {
	oldResourceID := BuildGuestKey("pve1", "node1", 101) + "-disk-root"
	newResourceID := BuildGuestKey("pve1", "node2", 101) + "-disk-root"

	threshold := &HysteresisThreshold{Trigger: 90, Clear: 85}
	oldSpec, err := buildCanonicalMetricSpec(oldResourceID, "app01", unifiedresources.ResourceTypeVM, "disk", threshold)
	if err != nil {
		t.Fatalf("buildCanonicalMetricSpec(oldResourceID) error = %v", err)
	}
	newSpec, err := buildCanonicalMetricSpec(newResourceID, "app01", unifiedresources.ResourceTypeVM, "disk", threshold)
	if err != nil {
		t.Fatalf("buildCanonicalMetricSpec(newResourceID) error = %v", err)
	}

	oldState := buildCanonicalStateID(oldResourceID, oldSpec.ID)
	newState := buildCanonicalStateID(newResourceID, newSpec.ID)

	m := newCharacterizationManager(t, characterizationBaseConfig())
	m.checkMetricWithCanonicalSpec(oldSpec, "app01", "node1", "pve1", "VM", 95, threshold, &metricOptions{Message: "VM disk (/root) at 95%"})
	m.checkMetricWithCanonicalSpec(newSpec, "app01", "node2", "pve1", "VM", 95, threshold, &metricOptions{Message: "VM disk (/root) at 95%"})

	assertAlertMissing(t, m, oldState)
	alert := activeAlert(t, m, newState)
	if alert.ResourceID != newResourceID {
		t.Fatalf("ResourceID = %q, want %q", alert.ResourceID, newResourceID)
	}

	history := m.GetAlertHistory(5)
	if len(history) != 1 || history[0].ID != newState {
		t.Fatalf("expected per-disk history entry under %q, got %#v", newState, history)
	}
}

func TestAlertCharacterizationManualClearRemovesCanonicalTrackingState(t *testing.T) {
	resourceID := BuildGuestKey("pve1", "node1", 101)
	specID := canonicalMetricSpecID(resourceID, "cpu")
	alertID := buildCanonicalStateID(resourceID, specID)
	trackingKey := alertID

	m := newCharacterizationManager(t, characterizationBaseConfig())
	m.CheckGuest(testVM(resourceID, 101, "app01", "node1", "pve1", "running", 0.85), "pve1")
	assertAlertPresent(t, m, alertID)

	m.mu.Lock()
	m.recentAlerts[trackingKey] = &Alert{ID: alertID, ResourceID: resourceID, CanonicalState: trackingKey, CanonicalSpecID: specID, StartTime: time.Now(), LastSeen: time.Now()}
	m.suppressedUntil[trackingKey] = time.Now().Add(time.Hour)
	m.alertRateLimit[trackingKey] = []time.Time{time.Now()}
	m.mu.Unlock()

	if !m.ClearAlert(alertID) {
		t.Fatalf("expected ClearAlert(%q) to succeed", alertID)
	}

	m.mu.RLock()
	_, recentExists := m.recentAlerts[trackingKey]
	_, suppressedExists := m.suppressedUntil[trackingKey]
	_, rateExists := m.alertRateLimit[trackingKey]
	m.mu.RUnlock()
	if recentExists || suppressedExists || rateExists {
		t.Fatalf("expected manual clear to remove canonical tracking entries, got recent=%t suppressed=%t rate=%t", recentExists, suppressedExists, rateExists)
	}
}

func TestAlertCharacterizationResolvedCallbackUsesCanonicalIDForCanonicalAliasClear(t *testing.T) {
	resourceID := BuildGuestKey("pve1", "node1", 101)
	alertID := canonicalMetricSpecID(resourceID, "cpu")
	canonicalState := canonicalMetricStateID(resourceID, "cpu")

	m := newCharacterizationManager(t, characterizationBaseConfig())
	resolved := make(chan string, 1)
	m.SetResolvedCallback(func(id string) {
		select {
		case resolved <- id:
		default:
		}
	})

	alert := &Alert{
		ID:              alertID,
		Type:            "cpu",
		ResourceID:      resourceID,
		ResourceName:    "app01",
		CanonicalSpecID: canonicalMetricSpecID(resourceID, "cpu"),
		CanonicalKind:   "metric_threshold",
		CanonicalState:  canonicalState,
		StartTime:       time.Now().Add(-time.Minute),
		LastSeen:        time.Now(),
	}

	m.mu.Lock()
	m.setActiveAlertNoLock(canonicalState, alert)
	m.mu.Unlock()

	m.clearAlert(canonicalState)

	select {
	case got := <-resolved:
		if got != canonicalState {
			t.Fatalf("resolved callback = %q, want %q", got, canonicalState)
		}
	case <-time.After(time.Second):
		t.Fatalf("expected resolved callback for %q", alertID)
	}
}