From 3c01803b5ec5ef56f066d7589d8a6835b92dd2e5 Mon Sep 17 00:00:00 2001 From: Shirly Radco Date: Thu, 12 Mar 2026 20:42:50 +0200 Subject: [PATCH 1/2] k8s: add orphan AlertRelabelConfig GC Detect and remove orphan AlertRelabelConfig resources that no longer have a matching PrometheusRule, preventing stale relabel configs from accumulating. Cover orphan deletion and keeper cases in e2e (live rule, GitOps, unannotated). Signed-off-by: Shirly Radco Co-authored-by: AI Assistant --- pkg/k8s/alert_relabel_config_gc.go | 57 ++++++ pkg/k8s/alert_relabel_config_gc_test.go | 188 +++++++++++++++++++ pkg/k8s/relabeled_rules.go | 19 +- test/e2e/orphan_arc_gc_test.go | 230 ++++++++++++++++++++++++ 4 files changed, 488 insertions(+), 6 deletions(-) create mode 100644 pkg/k8s/alert_relabel_config_gc.go create mode 100644 pkg/k8s/alert_relabel_config_gc_test.go create mode 100644 test/e2e/orphan_arc_gc_test.go diff --git a/pkg/k8s/alert_relabel_config_gc.go b/pkg/k8s/alert_relabel_config_gc.go new file mode 100644 index 000000000..f3fb3f864 --- /dev/null +++ b/pkg/k8s/alert_relabel_config_gc.go @@ -0,0 +1,57 @@ +package k8s + +import ( + "context" + + "github.com/openshift/monitoring-plugin/pkg/managementlabels" +) + +// gcOrphanedARCs deletes AlertRelabelConfigs whose associated alert rule no +// longer exists. This handles the case where an operator (or manual action) +// removes rules from a PrometheusRule or deletes the CR entirely — the ARCs +// that were created by the plugin for classification/drop/stamp become orphans. +// +// Only ARCs carrying the plugin's alertRuleId annotation are considered. +// GitOps-managed ARCs are never deleted automatically; a warning is logged +// so that operators can clean them up manually. +// +// liveRuleIDs must include every alerting-rule ID still present on a +// PrometheusRule, including platform rules dropped by relabel configs. +// IDs are recorded in collectAlerts before the drop continue, so a Drop +// ARC for a disabled rule is not treated as an orphan. +func (rrm *relabeledRulesManager) gcOrphanedARCs(ctx context.Context, liveRuleIDs map[string]struct{}) { + if rrm.alertRelabelConfigs == nil { + return + } + + arcs, err := rrm.alertRelabelConfigs.List(ctx, "") + if err != nil { + log.Errorf("orphan ARC GC: failed to list ARCs: %v", err) + return + } + + for i := range arcs { + arc := &arcs[i] + + ruleID, ok := arc.Annotations[managementlabels.ARCAnnotationAlertRuleIDKey] + if !ok || ruleID == "" { + continue + } + + if _, alive := liveRuleIDs[ruleID]; alive { + continue + } + + if IsManagedByGitOps(arc.Annotations, arc.Labels) { + log.Warnf("orphan ARC GC: ARC %s/%s (ruleId=%s) is orphaned but GitOps-managed — skipping deletion, manual cleanup required", arc.Namespace, arc.Name, ruleID) + continue + } + + if err := rrm.alertRelabelConfigs.Delete(ctx, arc.Namespace, arc.Name); err != nil { + log.Errorf("orphan ARC GC: failed to delete ARC %s/%s: %v", arc.Namespace, arc.Name, err) + continue + } + + log.Infof("orphan ARC GC: deleted orphaned ARC %s/%s (ruleId=%s)", arc.Namespace, arc.Name, ruleID) + } +} diff --git a/pkg/k8s/alert_relabel_config_gc_test.go b/pkg/k8s/alert_relabel_config_gc_test.go new file mode 100644 index 000000000..21acc246f --- /dev/null +++ b/pkg/k8s/alert_relabel_config_gc_test.go @@ -0,0 +1,188 @@ +package k8s + +import ( + "context" + "testing" + + osmv1 "github.com/openshift/api/monitoring/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/openshift/monitoring-plugin/pkg/managementlabels" +) + +type mockARCInterface struct { + arcs map[string]*osmv1.AlertRelabelConfig + deleted []string +} + +func (m *mockARCInterface) List(_ context.Context, _ string) ([]osmv1.AlertRelabelConfig, error) { + var result []osmv1.AlertRelabelConfig + for _, arc := range m.arcs { + result = append(result, *arc) + } + return result, nil +} + +func (m *mockARCInterface) Get(_ context.Context, ns, name string) (*osmv1.AlertRelabelConfig, bool, error) { + if arc, ok := m.arcs[ns+"/"+name]; ok { + return arc, true, nil + } + return nil, false, nil +} + +func (m *mockARCInterface) Create(_ context.Context, arc osmv1.AlertRelabelConfig) (*osmv1.AlertRelabelConfig, error) { + return &arc, nil +} + +func (m *mockARCInterface) Update(_ context.Context, _ osmv1.AlertRelabelConfig) error { return nil } + +func (m *mockARCInterface) Delete(_ context.Context, ns, name string) error { + m.deleted = append(m.deleted, ns+"/"+name) + delete(m.arcs, ns+"/"+name) + return nil +} + +func newARC(ns, name, ruleID string, annotations, labels map[string]string) *osmv1.AlertRelabelConfig { + if annotations == nil { + annotations = map[string]string{} + } + if ruleID != "" { + annotations[managementlabels.ARCAnnotationAlertRuleIDKey] = ruleID + } + return &osmv1.AlertRelabelConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + Annotations: annotations, + Labels: labels, + }, + } +} + +func TestGCOrphanedARCs_DeletesOrphan(t *testing.T) { + mock := &mockARCInterface{ + arcs: map[string]*osmv1.AlertRelabelConfig{ + "openshift-monitoring/arc-orphan": newARC("openshift-monitoring", "arc-orphan", "rule-gone", nil, nil), + }, + } + rrm := &relabeledRulesManager{alertRelabelConfigs: mock} + + rrm.gcOrphanedARCs(context.Background(), map[string]struct{}{}) + + if len(mock.deleted) != 1 || mock.deleted[0] != "openshift-monitoring/arc-orphan" { + t.Fatalf("expected orphan ARC to be deleted, got deleted=%v", mock.deleted) + } +} + +func TestGCOrphanedARCs_KeepsLiveRule(t *testing.T) { + mock := &mockARCInterface{ + arcs: map[string]*osmv1.AlertRelabelConfig{ + "openshift-monitoring/arc-live": newARC("openshift-monitoring", "arc-live", "rule-alive", nil, nil), + }, + } + rrm := &relabeledRulesManager{alertRelabelConfigs: mock} + + rrm.gcOrphanedARCs(context.Background(), map[string]struct{}{"rule-alive": {}}) + + if len(mock.deleted) != 0 { + t.Fatalf("expected no deletions, got deleted=%v", mock.deleted) + } +} + +func TestGCOrphanedARCs_SkipsGitOpsManaged(t *testing.T) { + mock := &mockARCInterface{ + arcs: map[string]*osmv1.AlertRelabelConfig{ + "openshift-monitoring/arc-gitops": newARC("openshift-monitoring", "arc-gitops", "rule-gone", + map[string]string{"argocd.argoproj.io/tracking-id": "some-id"}, nil), + }, + } + rrm := &relabeledRulesManager{alertRelabelConfigs: mock} + + rrm.gcOrphanedARCs(context.Background(), map[string]struct{}{}) + + if len(mock.deleted) != 0 { + t.Fatalf("expected GitOps-managed ARC to be preserved, got deleted=%v", mock.deleted) + } +} + +func TestGCOrphanedARCs_SkipsARCWithoutAnnotation(t *testing.T) { + mock := &mockARCInterface{ + arcs: map[string]*osmv1.AlertRelabelConfig{ + "openshift-monitoring/arc-manual": newARC("openshift-monitoring", "arc-manual", "", nil, nil), + }, + } + rrm := &relabeledRulesManager{alertRelabelConfigs: mock} + + rrm.gcOrphanedARCs(context.Background(), map[string]struct{}{}) + + if len(mock.deleted) != 0 { + t.Fatalf("expected ARC without annotation to be preserved, got deleted=%v", mock.deleted) + } +} + +func TestGCOrphanedARCs_MixedScenario(t *testing.T) { + mock := &mockARCInterface{ + arcs: map[string]*osmv1.AlertRelabelConfig{ + "openshift-monitoring/arc-live": newARC("openshift-monitoring", "arc-live", "rule-1", nil, nil), + "openshift-monitoring/arc-orphan1": newARC("openshift-monitoring", "arc-orphan1", "rule-deleted-1", nil, nil), + "openshift-monitoring/arc-orphan2": newARC("openshift-monitoring", "arc-orphan2", "rule-deleted-2", nil, nil), + "openshift-monitoring/arc-gitops": newARC("openshift-monitoring", "arc-gitops", "rule-deleted-3", + map[string]string{"argocd.argoproj.io/tracking-id": "t"}, nil), + "openshift-monitoring/arc-manual": newARC("openshift-monitoring", "arc-manual", "", nil, nil), + }, + } + rrm := &relabeledRulesManager{alertRelabelConfigs: mock} + + liveIDs := map[string]struct{}{"rule-1": {}} + rrm.gcOrphanedARCs(context.Background(), liveIDs) + + deletedSet := map[string]bool{} + for _, d := range mock.deleted { + deletedSet[d] = true + } + + if len(mock.deleted) != 2 { + t.Fatalf("expected 2 deletions, got %d: %v", len(mock.deleted), mock.deleted) + } + if !deletedSet["openshift-monitoring/arc-orphan1"] { + t.Error("expected arc-orphan1 to be deleted") + } + if !deletedSet["openshift-monitoring/arc-orphan2"] { + t.Error("expected arc-orphan2 to be deleted") + } + if deletedSet["openshift-monitoring/arc-live"] { + t.Error("arc-live should not have been deleted") + } + if deletedSet["openshift-monitoring/arc-gitops"] { + t.Error("arc-gitops should not have been deleted (GitOps-managed)") + } + if deletedSet["openshift-monitoring/arc-manual"] { + t.Error("arc-manual should not have been deleted (no annotation)") + } +} + +func TestGCOrphanedARCs_NilInterface(t *testing.T) { + rrm := &relabeledRulesManager{alertRelabelConfigs: nil} + // Should not panic + rrm.gcOrphanedARCs(context.Background(), map[string]struct{}{}) +} + +func TestGCOrphanedARCs_NilAnnotations(t *testing.T) { + mock := &mockARCInterface{ + arcs: map[string]*osmv1.AlertRelabelConfig{ + "openshift-monitoring/arc-nil": { + ObjectMeta: metav1.ObjectMeta{ + Name: "arc-nil", + Namespace: "openshift-monitoring", + }, + }, + }, + } + rrm := &relabeledRulesManager{alertRelabelConfigs: mock} + + rrm.gcOrphanedARCs(context.Background(), map[string]struct{}{}) + + if len(mock.deleted) != 0 { + t.Fatalf("expected ARC with nil annotations to be preserved, got deleted=%v", mock.deleted) + } +} diff --git a/pkg/k8s/relabeled_rules.go b/pkg/k8s/relabeled_rules.go index 02452c385..19f5acbd2 100644 --- a/pkg/k8s/relabeled_rules.go +++ b/pkg/k8s/relabeled_rules.go @@ -149,7 +149,7 @@ func newRelabeledRulesManager(ctx context.Context, namespaceManager NamespaceInt return nil, fmt.Errorf("failed to sync RelabeledRulesConfig informer") } - if err := rrm.sync(ctx); err != nil { + if err := rrm.sync(ctx, "initial-sync"); err != nil { return nil, fmt.Errorf("initial relabeled rules sync failed: %w", err) } @@ -180,7 +180,7 @@ func (rrm *relabeledRulesManager) processNextWorkItem(ctx context.Context) bool defer rrm.queue.Done(key) - if err := rrm.sync(ctx); err != nil { + if err := rrm.sync(ctx, key); err != nil { log.Errorf("error syncing relabeled rules: %v", err) rrm.queue.AddRateLimited(key) return true @@ -191,7 +191,7 @@ func (rrm *relabeledRulesManager) processNextWorkItem(ctx context.Context) bool return true } -func (rrm *relabeledRulesManager) sync(ctx context.Context) error { +func (rrm *relabeledRulesManager) sync(ctx context.Context, key string) error { relabelConfigs, err := rrm.loadRelabelConfigs() if err != nil { return fmt.Errorf("failed to load relabel configs: %w", err) @@ -201,13 +201,20 @@ func (rrm *relabeledRulesManager) sync(ctx context.Context) error { rrm.relabelConfigs = relabelConfigs rrm.mu.Unlock() - alerts := rrm.collectAlerts(ctx, relabelConfigs) + alerts, allRuleIDs := rrm.collectAlerts(ctx, relabelConfigs) rrm.mu.Lock() rrm.relabeledRules = alerts rrm.mu.Unlock() log.Infof("Synced %d relabeled rules in memory", len(alerts)) + + // GC orphaned ARCs only when triggered by PrometheusRule events or + // initial sync — secret-only changes cannot create orphans. + if key == "prometheus-rule-sync" || key == "initial-sync" { + rrm.gcOrphanedARCs(ctx, allRuleIDs) + } + return nil } @@ -256,7 +263,7 @@ func (rrm *relabeledRulesManager) loadRelabelConfigs() ([]*relabel.Config, error return configs, nil } -func (rrm *relabeledRulesManager) collectAlerts(ctx context.Context, relabelConfigs []*relabel.Config) map[string]monitoringv1.Rule { +func (rrm *relabeledRulesManager) collectAlerts(ctx context.Context, relabelConfigs []*relabel.Config) (map[string]monitoringv1.Rule, map[string]struct{}) { alerts := make(map[string]monitoringv1.Rule) seenIDs := make(map[string]struct{}) @@ -336,7 +343,7 @@ func (rrm *relabeledRulesManager) collectAlerts(ctx context.Context, relabelConf } log.Debugf("Collected %d alerts", len(alerts)) - return alerts + return alerts, seenIDs } // alertingRuleOwner returns the name of the AlertingRule CR that generated diff --git a/test/e2e/orphan_arc_gc_test.go b/test/e2e/orphan_arc_gc_test.go new file mode 100644 index 000000000..97fbde608 --- /dev/null +++ b/test/e2e/orphan_arc_gc_test.go @@ -0,0 +1,230 @@ +//go:build e2e + +package e2e + +import ( + "context" + "fmt" + "testing" + "time" + + osmv1 "github.com/openshift/api/monitoring/v1" + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + + "github.com/openshift/monitoring-plugin/pkg/k8s" + "github.com/openshift/monitoring-plugin/pkg/managementlabels" + "github.com/openshift/monitoring-plugin/test/e2e/framework" +) + +const ( + orphanARCGCPollInterval = time.Second + orphanARCGCPollTimeout = 2 * time.Minute +) + +// TestOrphanAlertRelabelConfigGC creates plugin-owned AlertRelabelConfigs +// and a PrometheusRule, then waits for a PrometheusRule-driven sync to +// delete the orphan while keeping live, GitOps-managed, and unannotated ARCs. +func TestOrphanAlertRelabelConfigGC(t *testing.T) { + f, err := framework.New() + if err != nil { + t.Fatalf("Failed to create framework: %v", err) + } + + ctx := context.Background() + + testNamespace, cleanup, err := f.CreateUserNamespace(ctx, "test-orphan-arc-gc") + if err != nil { + t.Fatalf("Failed to create test namespace: %v", err) + } + defer func() { + if err := cleanup(); err != nil { + t.Logf("cleanup failed: %v", err) + } + }() + + alertName := "E2EOrphanARCGCLive" + forDuration := monitoringv1.Duration("5m") + liveRule := monitoringv1.Rule{ + Alert: alertName, + Expr: intstr.FromString(`absent(nonexistent{e2e_test="orphan_arc_gc_live"})`), + For: &forDuration, + Labels: map[string]string{ + "severity": "none", + "e2e_test": "orphan_arc_gc", + }, + } + + promRule, err := createPrometheusRule(ctx, f, testNamespace, liveRule) + if err != nil { + t.Fatalf("Failed to create PrometheusRule: %v", err) + } + + var liveRuleID string + err = framework.Poll(orphanARCGCPollInterval, orphanARCGCPollTimeout, func() error { + rules, listErr := listRules(ctx, f) + if listErr != nil { + return fmt.Errorf("list rules: %w", listErr) + } + id, found := alertRuleIDByName(rules, alertName) + if !found { + return fmt.Errorf("alert %s not in GET /rules yet", alertName) + } + liveRuleID = id + return nil + }) + if err != nil { + t.Fatalf("Timeout waiting for live rule to appear: %v", err) + } + + idSuffix := fmt.Sprintf("%d", time.Now().UnixNano()) + orphanName := "e2e-ogc-orphan-" + idSuffix + liveName := "e2e-ogc-live-" + idSuffix + gitopsName := "e2e-ogc-gitops-" + idSuffix + manualName := "e2e-ogc-manual-" + idSuffix + orphanRuleID := "e2e-orphan-gc-missing-" + idSuffix + gitopsRuleID := "e2e-orphan-gc-gitops-" + idSuffix + + t.Cleanup(func() { + for _, name := range []string{orphanName, liveName, gitopsName, manualName} { + if delErr := deleteAlertRelabelConfig(ctx, f, name); delErr != nil { + t.Logf("cleanup ARC %s: %v", name, delErr) + } + } + }) + + if err := createAlertRelabelConfig(ctx, f, orphanName, map[string]string{ + managementlabels.ARCAnnotationAlertRuleIDKey: orphanRuleID, + }, nil); err != nil { + t.Fatalf("Failed to create orphan ARC: %v", err) + } + if err := createAlertRelabelConfig(ctx, f, liveName, map[string]string{ + managementlabels.ARCAnnotationAlertRuleIDKey: liveRuleID, + }, nil); err != nil { + t.Fatalf("Failed to create live ARC: %v", err) + } + if err := createAlertRelabelConfig(ctx, f, gitopsName, map[string]string{ + managementlabels.ARCAnnotationAlertRuleIDKey: gitopsRuleID, + "argocd.argoproj.io/tracking-id": "e2e-orphan-arc-gc", + }, nil); err != nil { + t.Fatalf("Failed to create GitOps ARC: %v", err) + } + if err := createAlertRelabelConfig(ctx, f, manualName, nil, nil); err != nil { + t.Fatalf("Failed to create unannotated ARC: %v", err) + } + + err = framework.Poll(orphanARCGCPollInterval, 20*time.Second, func() error { + current, getErr := f.Monitoringv1clientset.MonitoringV1().PrometheusRules(testNamespace).Get( + ctx, promRule.Name, metav1.GetOptions{}, + ) + if getErr != nil { + return getErr + } + if current.Annotations == nil { + current.Annotations = map[string]string{} + } + current.Annotations["e2e.monitoring.openshift.io/gc-sync"] = idSuffix + _, updateErr := f.Monitoringv1clientset.MonitoringV1().PrometheusRules(testNamespace).Update( + ctx, current, metav1.UpdateOptions{}, + ) + return updateErr + }) + if err != nil { + t.Fatalf("Failed to update PrometheusRule to trigger GC: %v", err) + } + + err = framework.Poll(orphanARCGCPollInterval, orphanARCGCPollTimeout, func() error { + exists, existsErr := alertRelabelConfigExists(ctx, f, orphanName) + if existsErr != nil { + return existsErr + } + if exists { + return fmt.Errorf("orphan ARC %s still present", orphanName) + } + return nil + }) + if err != nil { + t.Fatalf("Timeout waiting for orphan ARC GC: %v", err) + } + + for _, keeper := range []string{liveName, gitopsName, manualName} { + exists, existsErr := alertRelabelConfigExists(ctx, f, keeper) + if existsErr != nil { + t.Fatalf("Failed to get keeper ARC %s: %v", keeper, existsErr) + } + if !exists { + t.Errorf("keeper ARC %s was deleted", keeper) + } + } +} + +func alertRuleIDByName(rules []k8s.PrometheusRule, alertName string) (string, bool) { + for _, rule := range rules { + if rule.Name != alertName { + continue + } + id := rule.Labels[k8s.AlertRuleLabelId] + if id == "" { + return "", false + } + return id, true + } + return "", false +} + +func createAlertRelabelConfig(ctx context.Context, f *framework.Framework, name string, annotations, labels map[string]string) error { + arc := &osmv1.AlertRelabelConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: k8s.ClusterMonitoringNamespace, + Annotations: annotations, + Labels: labels, + }, + Spec: osmv1.AlertRelabelConfigSpec{ + Configs: []osmv1.RelabelConfig{ + { + SourceLabels: []osmv1.LabelName{"alertname"}, + Regex: "E2EOrphanARCGCNeverMatch", + TargetLabel: "e2e_orphan_gc", + Replacement: "test", + Action: "Replace", + }, + }, + }, + } + + return framework.Poll(time.Second, 20*time.Second, func() error { + _, err := f.Osmv1clientset.MonitoringV1().AlertRelabelConfigs(k8s.ClusterMonitoringNamespace).Create( + ctx, arc, metav1.CreateOptions{}, + ) + if err == nil || apierrors.IsAlreadyExists(err) { + return nil + } + return err + }) +} + +func deleteAlertRelabelConfig(ctx context.Context, f *framework.Framework, name string) error { + err := f.Osmv1clientset.MonitoringV1().AlertRelabelConfigs(k8s.ClusterMonitoringNamespace).Delete( + ctx, name, metav1.DeleteOptions{}, + ) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + return nil +} + +func alertRelabelConfigExists(ctx context.Context, f *framework.Framework, name string) (bool, error) { + _, err := f.Osmv1clientset.MonitoringV1().AlertRelabelConfigs(k8s.ClusterMonitoringNamespace).Get( + ctx, name, metav1.GetOptions{}, + ) + if apierrors.IsNotFound(err) { + return false, nil + } + if err != nil { + return false, err + } + return true, nil +} From 02eb5beee28db15c60bfabc157b177677c6d939a Mon Sep 17 00:00:00 2001 From: Shirly Radco Date: Thu, 12 Mar 2026 20:34:33 +0200 Subject: [PATCH 2/2] add alerts_effective_active_at metric Expose a Prometheus gauge metric whose value is the activeAt Unix timestamp for every effective alert (firing, pending, silenced). Labels include all alerts labels after relabeling plus enrichment labels and alertstate. Annotations are excluded since they are available from the alert rule definition. Always serve /metrics as valid Prometheus text so scrapes succeed when alert-management-api is off. Signed-off-by: Shirly Radco Co-authored-by: AI Assistant --- go.mod | 4 +- pkg/k8s/enrich_active_at_test.go | 106 +++++ pkg/k8s/prometheus_alerts.go | 72 ++- pkg/management/management.go | 9 + pkg/management/metrics/alerts_collector.go | 234 ++++++++++ .../metrics/alerts_collector_test.go | 389 ++++++++++++++++ pkg/management/metrics/leader_election.go | 87 ++++ pkg/management/types.go | 6 + pkg/server/server.go | 21 +- pkg/server/server_test.go | 10 + test/e2e/alerts_effective_metric_test.go | 414 ++++++++++++++++++ 11 files changed, 1344 insertions(+), 8 deletions(-) create mode 100644 pkg/k8s/enrich_active_at_test.go create mode 100644 pkg/management/metrics/alerts_collector.go create mode 100644 pkg/management/metrics/alerts_collector_test.go create mode 100644 pkg/management/metrics/leader_election.go create mode 100644 test/e2e/alerts_effective_metric_test.go diff --git a/go.mod b/go.mod index bcc7afc8a..e57ab90e6 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,8 @@ require ( github.com/openshift/library-go v0.0.0-20240905123346-5bdbfe35a6f5 github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.87.0 github.com/prometheus-operator/prometheus-operator/pkg/client v0.87.0 + github.com/prometheus/client_golang v1.23.2 + github.com/prometheus/client_model v0.6.2 github.com/prometheus/common v0.67.4 github.com/prometheus/prometheus v0.308.0 github.com/sirupsen/logrus v1.9.3 @@ -57,8 +59,6 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_golang v1.23.2 // indirect - github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/procfs v0.16.1 // indirect github.com/spf13/pflag v1.0.6 // indirect github.com/x448/float16 v0.8.4 // indirect diff --git a/pkg/k8s/enrich_active_at_test.go b/pkg/k8s/enrich_active_at_test.go new file mode 100644 index 000000000..95e205591 --- /dev/null +++ b/pkg/k8s/enrich_active_at_test.go @@ -0,0 +1,106 @@ +package k8s + +import ( + "testing" + "time" +) + +func TestEnrichActiveAt_ReplacesAlertmanagerTimestamp(t *testing.T) { + amTime := time.Date(2026, 3, 10, 12, 0, 0, 0, time.UTC) + promTime := time.Date(2026, 3, 9, 8, 0, 0, 0, time.UTC) + + amAlerts := []PrometheusAlert{{ + Labels: map[string]string{"alertname": "HighCPU", "severity": "critical", AlertSourceLabel: "platform", AlertBackendLabel: "am"}, + ActiveAt: amTime, + }} + promAlerts := []PrometheusAlert{{ + Labels: map[string]string{"alertname": "HighCPU", "severity": "critical", AlertSourceLabel: "platform", AlertBackendLabel: "prom"}, + ActiveAt: promTime, + }} + + enrichActiveAt(amAlerts, promAlerts) + + if !amAlerts[0].ActiveAt.Equal(promTime) { + t.Errorf("expected ActiveAt=%v, got %v", promTime, amAlerts[0].ActiveAt) + } +} + +func TestEnrichActiveAt_NoMatchKeepsOriginal(t *testing.T) { + amTime := time.Date(2026, 3, 10, 12, 0, 0, 0, time.UTC) + + amAlerts := []PrometheusAlert{{ + Labels: map[string]string{"alertname": "HighCPU", "severity": "critical"}, + ActiveAt: amTime, + }} + promAlerts := []PrometheusAlert{{ + Labels: map[string]string{"alertname": "DiskFull", "severity": "warning"}, + ActiveAt: time.Date(2026, 3, 9, 8, 0, 0, 0, time.UTC), + }} + + enrichActiveAt(amAlerts, promAlerts) + + if !amAlerts[0].ActiveAt.Equal(amTime) { + t.Errorf("expected ActiveAt to stay %v, got %v", amTime, amAlerts[0].ActiveAt) + } +} + +func TestEnrichActiveAt_EmptyPromAlerts(t *testing.T) { + amTime := time.Date(2026, 3, 10, 12, 0, 0, 0, time.UTC) + + amAlerts := []PrometheusAlert{{ + Labels: map[string]string{"alertname": "HighCPU"}, + ActiveAt: amTime, + }} + + enrichActiveAt(amAlerts, nil) + + if !amAlerts[0].ActiveAt.Equal(amTime) { + t.Errorf("expected ActiveAt to stay %v, got %v", amTime, amAlerts[0].ActiveAt) + } +} + +func TestEnrichActiveAt_SkipsZeroPromActiveAt(t *testing.T) { + amTime := time.Date(2026, 3, 10, 12, 0, 0, 0, time.UTC) + + amAlerts := []PrometheusAlert{{ + Labels: map[string]string{"alertname": "HighCPU"}, + ActiveAt: amTime, + }} + promAlerts := []PrometheusAlert{{ + Labels: map[string]string{"alertname": "HighCPU"}, + }} + + enrichActiveAt(amAlerts, promAlerts) + + if !amAlerts[0].ActiveAt.Equal(amTime) { + t.Errorf("expected ActiveAt to stay %v when prom has zero time, got %v", amTime, amAlerts[0].ActiveAt) + } +} + +func TestAlertFingerprint_IgnoresMetadataLabels(t *testing.T) { + fp1 := alertFingerprint(map[string]string{ + "alertname": "HighCPU", + "severity": "critical", + AlertSourceLabel: "platform", + AlertBackendLabel: "am", + }) + fp2 := alertFingerprint(map[string]string{ + "alertname": "HighCPU", + "severity": "critical", + AlertSourceLabel: "platform", + AlertBackendLabel: "prom", + }) + + if fp1 != fp2 { + t.Errorf("fingerprints should match when only metadata labels differ:\n fp1=%q\n fp2=%q", fp1, fp2) + } +} + +func TestAlertFingerprint_DifferentLabelsProduceDifferentKeys(t *testing.T) { + fp1 := alertFingerprint(map[string]string{"alertname": "HighCPU", "severity": "critical"}) + fp2 := alertFingerprint(map[string]string{"alertname": "HighCPU", "severity": "warning"}) + + if fp1 == fp2 { + t.Error("fingerprints should differ when label values differ") + } +} diff --git a/pkg/k8s/prometheus_alerts.go b/pkg/k8s/prometheus_alerts.go index 60690b457..53542ff6c 100644 --- a/pkg/k8s/prometheus_alerts.go +++ b/pkg/k8s/prometheus_alerts.go @@ -10,6 +10,7 @@ import ( "net/http" "net/url" "os" + "sort" "strings" "sync" "time" @@ -289,11 +290,21 @@ func (pa *prometheusAlerts) routeHealth(ctx context.Context, namespace string, r return health } +// getAlertsForSource fetches alerts from both Alertmanager and Prometheus in +// parallel and merges the results. The fallback strategy is: +// - Both succeed: AM (firing+silenced) + Prom pending, with AM timestamps +// enriched from Prometheus activeAt. +// - AM only: AM alerts returned as-is (no Prom data to enrich from). +// - Prom only: all Prom alerts returned (AM was unreachable). +// - Both fail: error propagated from Prometheus. func (pa *prometheusAlerts) getAlertsForSource(ctx context.Context, namespace string, promRouteName string, amRouteName string, source string) ([]PrometheusAlert, error) { amAlerts, amErr := pa.getAlertmanagerAlerts(ctx, namespace, amRouteName, source) promAlerts, promErr := pa.getAlertsViaProxy(ctx, namespace, promRouteName, source) if amErr == nil { + if promErr == nil { + enrichActiveAt(amAlerts, promAlerts) + } pending := filterAlertsByState(promAlerts, "pending") return append(amAlerts, pending...), nil } @@ -348,15 +359,17 @@ func (pa *prometheusAlerts) getUserWorkloadAlertsViaAlertmanager(ctx context.Con } } - pending, err := pa.getAlertsViaProxy(ctx, UserWorkloadMonitoringNamespace, UserWorkloadRouteName, AlertSourceUser) + promAlerts, err := pa.getAlertsViaProxy(ctx, UserWorkloadMonitoringNamespace, UserWorkloadRouteName, AlertSourceUser) if err != nil { - pending, err = pa.getPrometheusAlertsViaService(ctx, UserWorkloadMonitoringNamespace, UserWorkloadPrometheusServiceName, UserWorkloadPrometheusPort, AlertSourceUser) + promAlerts, err = pa.getPrometheusAlertsViaService(ctx, UserWorkloadMonitoringNamespace, UserWorkloadPrometheusServiceName, UserWorkloadPrometheusPort, AlertSourceUser) if err != nil { return alerts, nil } } - return append(alerts, filterAlertsByState(pending, "pending")...), nil + // Enrich before filtering: AM alerts need activeAt from all Prom states. + enrichActiveAt(alerts, promAlerts) + return append(alerts, filterAlertsByState(promAlerts, "pending")...), nil } func (pa *prometheusAlerts) getPrometheusAlertsViaService(ctx context.Context, namespace string, serviceName string, port int32, source string) ([]PrometheusAlert, error) { @@ -782,6 +795,59 @@ func filterAlertsByState(alerts []PrometheusAlert, state string) []PrometheusAle return out } +// enrichActiveAt replaces ActiveAt in Alertmanager-sourced alerts with the +// authoritative value from Prometheus. Alertmanager only exposes startsAt +// (when it received the alert), while Prometheus tracks the true activeAt +// (when the alert condition first became true). +func enrichActiveAt(amAlerts, promAlerts []PrometheusAlert) { + if len(promAlerts) == 0 { + return + } + + lookup := make(map[string]time.Time, len(promAlerts)) + for _, alert := range promAlerts { + fp := alertFingerprint(alert.Labels) + if !alert.ActiveAt.IsZero() { + lookup[fp] = alert.ActiveAt + } + } + + for i := range amAlerts { + fp := alertFingerprint(amAlerts[i].Labels) + if activeAt, ok := lookup[fp]; ok { + amAlerts[i].ActiveAt = activeAt + } + } +} + +// alertFingerprint builds a stable identity key from an alert's labels, +// excluding metadata labels injected by this plugin (source, backend). +// This matches the same alert *instance* across Alertmanager and Prometheus +// (which may differ only in injected metadata). It is distinct from the +// alert rule ID (GetAlertingRuleId) which identifies the *rule definition* +// and is computed from the rule spec (name, expr, duration, static labels). +func alertFingerprint(labels map[string]string) string { + keys := make([]string, 0, len(labels)) + for k := range labels { + if k == AlertSourceLabel || k == AlertBackendLabel { + continue + } + keys = append(keys, k) + } + sort.Strings(keys) + + var b strings.Builder + for i, k := range keys { + if i > 0 { + b.WriteByte('\xff') + } + b.WriteString(k) + b.WriteByte('\xfe') + b.WriteString(labels[k]) + } + return b.String() +} + func mapAlertmanagerState(state string) string { if state == "active" { return "firing" diff --git a/pkg/management/management.go b/pkg/management/management.go index 652ac14de..124a98785 100644 --- a/pkg/management/management.go +++ b/pkg/management/management.go @@ -1,9 +1,14 @@ package management import ( + "context" + "net/http" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" "github.com/openshift/monitoring-plugin/pkg/k8s" + "github.com/openshift/monitoring-plugin/pkg/management/metrics" ) type client struct { @@ -20,3 +25,7 @@ type client struct { func (c *client) isPlatformManagedPrometheusRule(nn types.NamespacedName) bool { return c.k8sClient.Namespace().IsClusterMonitoringNamespace(nn.Namespace) } + +func (c *client) MetricsHandler(ctx context.Context, kubeConfig *rest.Config) (http.Handler, error) { + return metrics.NewHandler(ctx, c, kubeConfig) +} diff --git a/pkg/management/metrics/alerts_collector.go b/pkg/management/metrics/alerts_collector.go new file mode 100644 index 000000000..081d8e77b --- /dev/null +++ b/pkg/management/metrics/alerts_collector.go @@ -0,0 +1,234 @@ +package metrics + +import ( + "context" + "fmt" + "net/http" + "sort" + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/sirupsen/logrus" + "k8s.io/client-go/rest" + + "github.com/openshift/monitoring-plugin/pkg/k8s" +) + +var metricsLog = logrus.WithField("module", "metrics") + +const ( + MetricName = "alerts_effective_active_at_timestamp_seconds" + metricHelp = "The activeAt timestamp of effective (post-ARC) alerts. " + + "Value is the Unix timestamp when the alert became active." + + DefaultSyncInterval = 30 * time.Second + + labelAlertState = "alertstate" +) + +// AlertsFetcher retrieves enriched alerts for the metric. The management.Client +// satisfies this interface — it applies ARC relabeling and computes +// classification (AlertComponent / AlertLayer) on every alert. +type AlertsFetcher interface { + EnrichAlerts(ctx context.Context, req k8s.GetAlertsRequest) ([]k8s.PrometheusAlert, []string, error) +} + +// alertMetric holds a single alert's pre-built metric data. +// The prometheus.Desc is created once during sync, not on every scrape. +type alertMetric struct { + desc *prometheus.Desc + labelValues []string + activeAtSec float64 +} + +// AlertsCollector implements prometheus.Collector. It periodically fetches +// alerts via the management client's EnrichAlerts (which applies ARC relabeling +// and computes classification) and exposes them as the +// alerts_effective_active_at_timestamp_seconds gauge. +// +// Only the leader pod (determined via Lease-based leader election) runs the +// sync loop and exposes metrics. Follower pods return nothing on Collect, +// ensuring each alert appears exactly once in Prometheus. +// +// Each alert produces one time series whose value is the alert's activeAt +// Unix timestamp. Labels are the alert's enriched labels (post-ARC, source, +// backend, component, layer) plus "alertstate". Thanos-sourced alerts are +// filtered out to avoid duplicates. Annotations are excluded because they +// are available from the alert rule definition. +type AlertsCollector struct { + fetcher AlertsFetcher + syncInterval time.Duration + isLeader func() bool + + mu sync.RWMutex + metrics []alertMetric + + sentinelDesc *prometheus.Desc +} + +// NewHandler creates a metrics HTTP handler that exposes the alerts effective +// metric. It sets up Lease-based leader election internally so that only one +// replica produces metrics, then wires the collector, registry and promhttp +// handler. Callers receive a ready-to-use http.Handler. +func NewHandler(ctx context.Context, fetcher AlertsFetcher, kubeConfig *rest.Config) (http.Handler, error) { + isLeader, err := startLeaderElection(ctx, kubeConfig, k8s.ClusterMonitoringNamespace) + if err != nil { + return nil, fmt.Errorf("start metrics leader election: %w", err) + } + + collector := NewAlertsCollector(ctx, fetcher, DefaultSyncInterval, isLeader) + registry := prometheus.NewRegistry() + registry.MustRegister(collector) + return handlerFor(registry), nil +} + +// NewEmptyHandler returns a Prometheus /metrics handler with no series. +// Used when alert-management-api is disabled so platform scrapes still +// succeed (up=1) instead of falling through to the static file server. +func NewEmptyHandler() http.Handler { + return handlerFor(prometheus.NewRegistry()) +} + +func handlerFor(registry *prometheus.Registry) http.Handler { + return promhttp.HandlerFor(registry, promhttp.HandlerOpts{}) +} + +// NewAlertsCollector creates a collector that periodically syncs alerts and +// exposes them as Prometheus metrics. The isLeader callback controls whether +// this replica actively syncs and exposes metrics (follower pods return nothing). +func NewAlertsCollector(ctx context.Context, fetcher AlertsFetcher, syncInterval time.Duration, isLeader func() bool) *AlertsCollector { + c := &AlertsCollector{ + fetcher: fetcher, + syncInterval: syncInterval, + isLeader: isLeader, + sentinelDesc: prometheus.NewDesc(MetricName, metricHelp, nil, nil), + } + go c.syncLoop(ctx) + return c +} + +// Describe sends a sentinel descriptor to satisfy the Collector contract. +func (c *AlertsCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- c.sentinelDesc +} + +// Collect emits the current set of alert metrics using pre-built Descs. +// Returns nothing if this replica is not the leader. +func (c *AlertsCollector) Collect(ch chan<- prometheus.Metric) { + if !c.isLeader() { + return + } + + c.mu.RLock() + defer c.mu.RUnlock() + + for i := range c.metrics { + m := &c.metrics[i] + metric, err := prometheus.NewConstMetric(m.desc, prometheus.GaugeValue, m.activeAtSec, m.labelValues...) + if err != nil { + metricsLog.WithError(err).Warn("failed to create metric") + continue + } + ch <- metric + } +} + +func (c *AlertsCollector) syncLoop(ctx context.Context) { + c.sync(ctx) + + ticker := time.NewTicker(c.syncInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + c.sync(ctx) + } + } +} + +func (c *AlertsCollector) sync(ctx context.Context) { + if !c.isLeader() { + return + } + + alerts, _, err := c.fetcher.EnrichAlerts(ctx, k8s.GetAlertsRequest{}) + if err != nil { + metricsLog.WithError(err).Warn("failed to fetch alerts for effective metric") + return + } + + built := make([]alertMetric, 0, len(alerts)) + for i := range alerts { + alert := &alerts[i] + + // Drop Thanos-sourced alerts: they duplicate what Alertmanager and + // Prometheus already provide and would inflate the metric cardinality. + if alert.Labels[k8s.AlertBackendLabel] == k8s.AlertBackendThanos { + continue + } + + enrichClassificationLabels(alert) + + m := buildAlertMetric(alert) + if m != nil { + built = append(built, *m) + } + } + + c.mu.Lock() + c.metrics = built + c.mu.Unlock() + + metricsLog.Debugf("synced %d alerts for effective metric", len(built)) +} + +// enrichClassificationLabels copies the management-computed AlertComponent and +// AlertLayer into the alert's Labels map so they appear on the metric. Labels +// already set (e.g. via ARC) take precedence. +func enrichClassificationLabels(alert *k8s.PrometheusAlert) { + if alert.AlertComponent != "" { + if _, exists := alert.Labels[k8s.AlertRuleClassificationComponentKey]; !exists { + alert.Labels[k8s.AlertRuleClassificationComponentKey] = alert.AlertComponent + } + } + if alert.AlertLayer != "" { + if _, exists := alert.Labels[k8s.AlertRuleClassificationLayerKey]; !exists { + alert.Labels[k8s.AlertRuleClassificationLayerKey] = alert.AlertLayer + } + } +} + +// buildAlertMetric converts a PrometheusAlert into an alertMetric with a +// pre-built prometheus.Desc. Uses the alert's labels plus the alertstate label. +func buildAlertMetric(alert *k8s.PrometheusAlert) *alertMetric { + if alert.ActiveAt.IsZero() { + return nil + } + + labelNames := make([]string, 0, len(alert.Labels)+1) + for k := range alert.Labels { + labelNames = append(labelNames, k) + } + sort.Strings(labelNames) + labelNames = append(labelNames, labelAlertState) + + labelValues := make([]string, 0, len(labelNames)) + for _, name := range labelNames { + if name == labelAlertState { + labelValues = append(labelValues, alert.State) + } else { + labelValues = append(labelValues, alert.Labels[name]) + } + } + + return &alertMetric{ + desc: prometheus.NewDesc(MetricName, metricHelp, labelNames, nil), + labelValues: labelValues, + activeAtSec: float64(alert.ActiveAt.Unix()), + } +} diff --git a/pkg/management/metrics/alerts_collector_test.go b/pkg/management/metrics/alerts_collector_test.go new file mode 100644 index 000000000..8ac846393 --- /dev/null +++ b/pkg/management/metrics/alerts_collector_test.go @@ -0,0 +1,389 @@ +package metrics_test + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + + "github.com/openshift/monitoring-plugin/pkg/k8s" + "github.com/openshift/monitoring-plugin/pkg/management/metrics" +) + +type mockAlertsFetcher struct { + alerts []k8s.PrometheusAlert + err error +} + +func (m *mockAlertsFetcher) EnrichAlerts(_ context.Context, _ k8s.GetAlertsRequest) ([]k8s.PrometheusAlert, []string, error) { + return m.alerts, nil, m.err +} + +func collectMetrics(t *testing.T, collector prometheus.Collector) []*dto.MetricFamily { + t.Helper() + reg := prometheus.NewRegistry() + reg.MustRegister(collector) + families, err := reg.Gather() + if err != nil { + t.Fatalf("gather metrics: %v", err) + } + return families +} + +func findFamily(families []*dto.MetricFamily, name string) *dto.MetricFamily { + for _, f := range families { + if f.GetName() == name { + return f + } + } + return nil +} + +func labelValue(m *dto.Metric, name string) string { + for _, lp := range m.GetLabel() { + if lp.GetName() == name { + return lp.GetValue() + } + } + return "" +} + +func newCollector(t *testing.T, mock *mockAlertsFetcher) (prometheus.Collector, context.CancelFunc) { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + collector := metrics.NewAlertsCollector(ctx, mock, 1*time.Hour, func() bool { return true }) + time.Sleep(100 * time.Millisecond) + t.Cleanup(cancel) + return collector, cancel +} + +func TestAlertsCollector_FiringAndSilenced(t *testing.T) { + activeAt := time.Date(2026, 3, 5, 10, 0, 0, 0, time.UTC) + mock := &mockAlertsFetcher{ + alerts: []k8s.PrometheusAlert{ + { + Labels: map[string]string{"alertname": "HighCPU", "severity": "critical", "namespace": "production"}, + State: "firing", + ActiveAt: activeAt, + }, + { + Labels: map[string]string{"alertname": "DiskFull", "severity": "warning", "namespace": "storage"}, + State: "silenced", + ActiveAt: activeAt.Add(-1 * time.Hour), + }, + }, + } + collector, _ := newCollector(t, mock) + families := collectMetrics(t, collector) + family := findFamily(families, metrics.MetricName) + if family == nil { + t.Fatal("expected metric family, got nil") + } + if len(family.GetMetric()) != 2 { + t.Fatalf("expected 2 metrics, got %d", len(family.GetMetric())) + } + + var firing, silenced *dto.Metric + for _, m := range family.GetMetric() { + switch labelValue(m, "alertname") { + case "HighCPU": + firing = m + case "DiskFull": + silenced = m + } + } + + if firing == nil { + t.Fatal("expected HighCPU metric") + } + if labelValue(firing, "alertstate") != "firing" { + t.Errorf("alertstate: want firing, got %q", labelValue(firing, "alertstate")) + } + if labelValue(firing, "severity") != "critical" { + t.Errorf("severity: want critical, got %q", labelValue(firing, "severity")) + } + if labelValue(firing, "namespace") != "production" { + t.Errorf("namespace: want production, got %q", labelValue(firing, "namespace")) + } + if firing.GetGauge().GetValue() != float64(activeAt.Unix()) { + t.Errorf("gauge value: want %v, got %v", float64(activeAt.Unix()), firing.GetGauge().GetValue()) + } + + if silenced == nil { + t.Fatal("expected DiskFull metric") + } + if labelValue(silenced, "alertstate") != "silenced" { + t.Errorf("alertstate: want silenced, got %q", labelValue(silenced, "alertstate")) + } + if silenced.GetGauge().GetValue() != float64(activeAt.Add(-1*time.Hour).Unix()) { + t.Errorf("silenced gauge value mismatch") + } +} + +func TestAlertsCollector_NoAnnotationLabels(t *testing.T) { + mock := &mockAlertsFetcher{ + alerts: []k8s.PrometheusAlert{ + {Labels: map[string]string{"alertname": "TestAlert"}, State: "firing", ActiveAt: time.Now()}, + }, + } + collector, _ := newCollector(t, mock) + families := collectMetrics(t, collector) + family := findFamily(families, metrics.MetricName) + if family == nil { + t.Fatal("expected metric family") + } + if len(family.GetMetric()) != 1 { + t.Fatalf("expected 1 metric, got %d", len(family.GetMetric())) + } + for _, lp := range family.GetMetric()[0].GetLabel() { + switch lp.GetName() { + case "summary", "description", "runbook_url": + t.Errorf("unexpected annotation label: %s", lp.GetName()) + } + } +} + +func TestAlertsCollector_SkipsZeroActiveAt(t *testing.T) { + mock := &mockAlertsFetcher{ + alerts: []k8s.PrometheusAlert{ + {Labels: map[string]string{"alertname": "NoActiveAt"}, State: "firing", ActiveAt: time.Time{}}, + }, + } + collector, _ := newCollector(t, mock) + families := collectMetrics(t, collector) + family := findFamily(families, metrics.MetricName) + if family != nil && len(family.GetMetric()) != 0 { + t.Errorf("expected no metrics for zero ActiveAt, got %d", len(family.GetMetric())) + } +} + +func TestAlertsCollector_EmptyAlerts(t *testing.T) { + mock := &mockAlertsFetcher{alerts: []k8s.PrometheusAlert{}} + collector, _ := newCollector(t, mock) + families := collectMetrics(t, collector) + family := findFamily(families, metrics.MetricName) + if family != nil && len(family.GetMetric()) != 0 { + t.Errorf("expected no metrics for empty alerts, got %d", len(family.GetMetric())) + } +} + +func TestAlertsCollector_FetcherErrorProducesNoMetrics(t *testing.T) { + mock := &mockAlertsFetcher{err: errors.New("connection refused")} + collector, _ := newCollector(t, mock) + families := collectMetrics(t, collector) + family := findFamily(families, metrics.MetricName) + if family != nil && len(family.GetMetric()) != 0 { + t.Errorf("expected no metrics on initial failure, got %d", len(family.GetMetric())) + } +} + +func TestAlertsCollector_ClassificationLabels(t *testing.T) { + mock := &mockAlertsFetcher{ + alerts: []k8s.PrometheusAlert{ + { + Labels: map[string]string{ + "alertname": "KubePodCrashLooping", + "severity": "warning", + "namespace": "kube-system", + k8s.AlertRuleLabelId: "abc123", + k8s.AlertRuleClassificationComponentKey: "kube-controller-manager", + k8s.AlertRuleClassificationLayerKey: "cluster", + }, + State: "firing", + ActiveAt: time.Now(), + }, + }, + } + collector, _ := newCollector(t, mock) + families := collectMetrics(t, collector) + family := findFamily(families, metrics.MetricName) + if family == nil || len(family.GetMetric()) != 1 { + t.Fatalf("expected 1 metric, got family=%v", family) + } + m := family.GetMetric()[0] + checks := map[string]string{ + k8s.AlertRuleLabelId: "abc123", + k8s.AlertRuleClassificationComponentKey: "kube-controller-manager", + k8s.AlertRuleClassificationLayerKey: "cluster", + "alertstate": "firing", + } + for k, want := range checks { + if got := labelValue(m, k); got != want { + t.Errorf("label[%s]: want %q, got %q", k, want, got) + } + } +} + +func TestAlertsCollector_IncludesPendingAlerts(t *testing.T) { + mock := &mockAlertsFetcher{ + alerts: []k8s.PrometheusAlert{ + {Labels: map[string]string{"alertname": "Firing"}, State: "firing", ActiveAt: time.Now()}, + {Labels: map[string]string{"alertname": "Silenced"}, State: "silenced", ActiveAt: time.Now()}, + {Labels: map[string]string{"alertname": "Pending"}, State: "pending", ActiveAt: time.Now()}, + }, + } + collector, _ := newCollector(t, mock) + families := collectMetrics(t, collector) + family := findFamily(families, metrics.MetricName) + if family == nil || len(family.GetMetric()) != 3 { + t.Fatalf("expected 3 metrics, got %v", family) + } + states := map[string]bool{} + for _, m := range family.GetMetric() { + states[labelValue(m, "alertstate")] = true + } + for _, s := range []string{"firing", "silenced", "pending"} { + if !states[s] { + t.Errorf("expected state %q in metrics", s) + } + } +} + +func TestAlertsCollector_SourceAndBackendLabels(t *testing.T) { + mock := &mockAlertsFetcher{ + alerts: []k8s.PrometheusAlert{ + { + Labels: map[string]string{ + "alertname": "HighCPU", + "severity": "critical", + k8s.AlertSourceLabel: k8s.AlertSourcePlatform, + k8s.AlertBackendLabel: k8s.AlertBackendAM, + }, + State: "firing", + ActiveAt: time.Now(), + }, + }, + } + collector, _ := newCollector(t, mock) + families := collectMetrics(t, collector) + family := findFamily(families, metrics.MetricName) + if family == nil || len(family.GetMetric()) != 1 { + t.Fatalf("expected 1 metric") + } + m := family.GetMetric()[0] + if got := labelValue(m, k8s.AlertSourceLabel); got != k8s.AlertSourcePlatform { + t.Errorf("source: want %q, got %q", k8s.AlertSourcePlatform, got) + } + if got := labelValue(m, k8s.AlertBackendLabel); got != k8s.AlertBackendAM { + t.Errorf("backend: want %q, got %q", k8s.AlertBackendAM, got) + } +} + +func TestAlertsCollector_FiltersThanosBackend(t *testing.T) { + now := time.Now() + mock := &mockAlertsFetcher{ + alerts: []k8s.PrometheusAlert{ + {Labels: map[string]string{"alertname": "HighCPU", k8s.AlertBackendLabel: k8s.AlertBackendAM, k8s.AlertSourceLabel: k8s.AlertSourcePlatform}, State: "firing", ActiveAt: now}, + {Labels: map[string]string{"alertname": "HighCPU", k8s.AlertBackendLabel: k8s.AlertBackendThanos, k8s.AlertSourceLabel: k8s.AlertSourceUser}, State: "firing", ActiveAt: now}, + {Labels: map[string]string{"alertname": "PendingAlert", k8s.AlertBackendLabel: k8s.AlertBackendProm, k8s.AlertSourceLabel: k8s.AlertSourcePlatform}, State: "pending", ActiveAt: now}, + }, + } + collector, _ := newCollector(t, mock) + families := collectMetrics(t, collector) + family := findFamily(families, metrics.MetricName) + if family == nil || len(family.GetMetric()) != 2 { + t.Fatalf("expected 2 metrics (thanos filtered), got %v", family) + } + for _, m := range family.GetMetric() { + if labelValue(m, k8s.AlertBackendLabel) == k8s.AlertBackendThanos { + t.Error("thanos duplicate should be filtered out") + } + } +} + +func TestAlertsCollector_InjectsClassificationFromFields(t *testing.T) { + mock := &mockAlertsFetcher{ + alerts: []k8s.PrometheusAlert{ + { + Labels: map[string]string{"alertname": "TestAlert", k8s.AlertBackendLabel: k8s.AlertBackendAM}, + State: "firing", + ActiveAt: time.Now(), + AlertComponent: "networking", + AlertLayer: "cluster", + }, + }, + } + collector, _ := newCollector(t, mock) + families := collectMetrics(t, collector) + family := findFamily(families, metrics.MetricName) + if family == nil || len(family.GetMetric()) != 1 { + t.Fatalf("expected 1 metric") + } + m := family.GetMetric()[0] + if got := labelValue(m, k8s.AlertRuleClassificationComponentKey); got != "networking" { + t.Errorf("component: want networking, got %q", got) + } + if got := labelValue(m, k8s.AlertRuleClassificationLayerKey); got != "cluster" { + t.Errorf("layer: want cluster, got %q", got) + } +} + +func TestAlertsCollector_DoesNotOverwriteARCLabels(t *testing.T) { + mock := &mockAlertsFetcher{ + alerts: []k8s.PrometheusAlert{ + { + Labels: map[string]string{ + "alertname": "TestAlert", + k8s.AlertBackendLabel: k8s.AlertBackendAM, + k8s.AlertRuleClassificationComponentKey: "arc-component", + k8s.AlertRuleClassificationLayerKey: "namespace", + }, + State: "firing", + ActiveAt: time.Now(), + AlertComponent: "default-component", + AlertLayer: "cluster", + }, + }, + } + collector, _ := newCollector(t, mock) + families := collectMetrics(t, collector) + family := findFamily(families, metrics.MetricName) + if family == nil || len(family.GetMetric()) != 1 { + t.Fatalf("expected 1 metric") + } + m := family.GetMetric()[0] + if got := labelValue(m, k8s.AlertRuleClassificationComponentKey); got != "arc-component" { + t.Errorf("component: want arc-component, got %q", got) + } + if got := labelValue(m, k8s.AlertRuleClassificationLayerKey); got != "namespace" { + t.Errorf("layer: want namespace, got %q", got) + } +} + +func TestAlertsCollector_FollowerExposesNoMetrics(t *testing.T) { + mock := &mockAlertsFetcher{ + alerts: []k8s.PrometheusAlert{ + {Labels: map[string]string{"alertname": "HighCPU"}, State: "firing", ActiveAt: time.Now()}, + }, + } + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + collector := metrics.NewAlertsCollector(ctx, mock, 1*time.Hour, func() bool { return false }) + families := collectMetrics(t, collector) + family := findFamily(families, metrics.MetricName) + if family != nil && len(family.GetMetric()) != 0 { + t.Errorf("follower should expose no metrics, got %d", len(family.GetMetric())) + } +} + +func TestNewEmptyHandler(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/metrics", nil) + rec := httptest.NewRecorder() + metrics.NewEmptyHandler().ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("expected status %d, got %d", http.StatusOK, rec.Code) + } + if !strings.Contains(rec.Header().Get("Content-Type"), "text/plain") { + t.Fatalf("expected Content-Type text/plain, got %q", rec.Header().Get("Content-Type")) + } + if strings.Contains(rec.Body.String(), metrics.MetricName) { + t.Fatalf("empty handler must not expose %s, got %q", metrics.MetricName, rec.Body.String()) + } +} diff --git a/pkg/management/metrics/leader_election.go b/pkg/management/metrics/leader_election.go new file mode 100644 index 000000000..2a71f9231 --- /dev/null +++ b/pkg/management/metrics/leader_election.go @@ -0,0 +1,87 @@ +package metrics + +import ( + "context" + "fmt" + "os" + "sync" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + coordinationv1client "k8s.io/client-go/kubernetes/typed/coordination/v1" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/leaderelection" + "k8s.io/client-go/tools/leaderelection/resourcelock" +) + +const ( + leaseName = "monitoring-plugin-metrics" + leaseDuration = 15 * time.Second + leaseRenew = 10 * time.Second + leaseRetry = 2 * time.Second +) + +// startLeaderElection sets up Lease-based leader election for the alerts +// effective metric. Returns a thread-safe isLeader callback. +func startLeaderElection(ctx context.Context, kubeConfig *rest.Config, namespace string) (func() bool, error) { + coordClient, err := coordinationv1client.NewForConfig(kubeConfig) + if err != nil { + return nil, fmt.Errorf("create coordination client: %w", err) + } + + identity, err := os.Hostname() + if err != nil { + return nil, fmt.Errorf("get hostname: %w", err) + } + + lock := &resourcelock.LeaseLock{ + LeaseMeta: metav1.ObjectMeta{ + Name: leaseName, + Namespace: namespace, + }, + Client: coordClient, + LockConfig: resourcelock.ResourceLockConfig{ + Identity: identity, + }, + } + + var mu sync.Mutex + isLeading := false + + isLeader := func() bool { + mu.Lock() + defer mu.Unlock() + return isLeading + } + + le, err := leaderelection.NewLeaderElector(leaderelection.LeaderElectionConfig{ + Lock: lock, + LeaseDuration: leaseDuration, + RenewDeadline: leaseRenew, + RetryPeriod: leaseRetry, + ReleaseOnCancel: true, + Callbacks: leaderelection.LeaderCallbacks{ + OnStartedLeading: func(_ context.Context) { + mu.Lock() + isLeading = true + mu.Unlock() + metricsLog.Info("became leader for alert management metrics") + }, + OnStoppedLeading: func() { + mu.Lock() + isLeading = false + mu.Unlock() + metricsLog.Info("lost leadership for alert management metrics") + }, + OnNewLeader: func(identity string) { + metricsLog.Infof("new leader for alert management metrics: %s", identity) + }, + }, + }) + if err != nil { + return nil, fmt.Errorf("create leader elector: %w", err) + } + + go le.Run(ctx) + return isLeader, nil +} diff --git a/pkg/management/types.go b/pkg/management/types.go index fd158a7db..2a314f076 100644 --- a/pkg/management/types.go +++ b/pkg/management/types.go @@ -2,8 +2,10 @@ package management import ( "context" + "net/http" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + "k8s.io/client-go/rest" "github.com/openshift/monitoring-plugin/pkg/k8s" ) @@ -59,6 +61,10 @@ type Client interface { // GetAlertingHealth retrieves the alerting stack health status GetAlertingHealth(ctx context.Context) (k8s.AlertingHealth, error) + + // MetricsHandler returns an HTTP handler that exposes alert management metrics. + // It handles leader election internally using the provided kubeConfig. + MetricsHandler(ctx context.Context, kubeConfig *rest.Config) (http.Handler, error) } // PrometheusRuleOptions specifies options for selecting PrometheusRule resources and groups diff --git a/pkg/server/server.go b/pkg/server/server.go index 5c5c49800..a6d95afa7 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -25,6 +25,7 @@ import ( "github.com/openshift/monitoring-plugin/internal/managementrouter" "github.com/openshift/monitoring-plugin/pkg/k8s" "github.com/openshift/monitoring-plugin/pkg/management" + "github.com/openshift/monitoring-plugin/pkg/management/metrics" "github.com/openshift/monitoring-plugin/pkg/monitoring" ) @@ -184,7 +185,10 @@ func createHTTPServer(ctx context.Context, cfg *Config) (*http.Server, error) { log.Info("alert management API enabled") } - router, pluginConfig := setupRoutes(cfg, managementClient) + router, pluginConfig, err := setupRoutes(ctx, cfg, managementClient, k8sconfig) + if err != nil { + return nil, fmt.Errorf("failed to set up routes: %w", err) + } router.Use(corsHeaderMiddleware()) tlsConfig := &tls.Config{} @@ -275,7 +279,7 @@ func createHTTPServer(ctx context.Context, cfg *Config) (*http.Server, error) { return httpServer, nil } -func setupRoutes(cfg *Config, managementClient management.Client) (*mux.Router, *PluginConfig) { +func setupRoutes(ctx context.Context, cfg *Config, managementClient management.Client, k8sconfig *rest.Config) (*mux.Router, *PluginConfig, error) { configHandlerFunc, pluginConfig := configHandler(cfg) router := mux.NewRouter() @@ -287,14 +291,25 @@ func setupRoutes(cfg *Config, managementClient management.Client) (*mux.Router, router.Path("/features").HandlerFunc(featuresHandler(cfg)) router.Path("/config").HandlerFunc(configHandlerFunc) + var metricsHandler http.Handler if managementClient != nil { managementRouter := managementrouter.New(managementClient) router.PathPrefix("/api/v1/alerting").Handler(managementRouter) + + var err error + metricsHandler, err = managementClient.MetricsHandler(ctx, k8sconfig) + if err != nil { + return nil, nil, fmt.Errorf("failed to start alert management metrics: %w", err) + } + log.Info("alert management metrics started") + } else { + metricsHandler = metrics.NewEmptyHandler() } + router.Path("/metrics").Handler(metricsHandler) router.PathPrefix("/").Handler(filesHandler(http.Dir(cfg.StaticPath))) - return router, pluginConfig + return router, pluginConfig, nil } func setupProxyRoutes(cfg *Config, k8sclient *dynamic.DynamicClient, kind monitoring.KindType) *mux.Router { diff --git a/pkg/server/server_test.go b/pkg/server/server_test.go index b2a415dc7..c40c92b7f 100644 --- a/pkg/server/server_test.go +++ b/pkg/server/server_test.go @@ -21,6 +21,8 @@ import ( "time" "github.com/stretchr/testify/require" + + "github.com/openshift/monitoring-plugin/pkg/management/metrics" ) type httpClientConfig struct { @@ -152,6 +154,14 @@ func TestServerRunning(t *testing.T) { t.Fatalf("Failed: could not fetch features endpoint: %v", err) } + metricsBody, err := getRequestResults(t, httpClient, serverURL+"/metrics") + if err != nil { + t.Fatalf("Failed: could not fetch /metrics with alert-management-api disabled: %v", err) + } + if strings.Contains(metricsBody, metrics.MetricName) { + t.Fatalf("expected no %s without alert-management-api, got %q", metrics.MetricName, metricsBody) + } + // sanity check - make sure we cannot get to a bogus context path if _, err = getRequestResults(t, httpClient, serverURL+"/badroot"); err == nil { t.Fatalf("Failed: Should have failed going to /badroot") diff --git a/test/e2e/alerts_effective_metric_test.go b/test/e2e/alerts_effective_metric_test.go new file mode 100644 index 000000000..f68185dd4 --- /dev/null +++ b/test/e2e/alerts_effective_metric_test.go @@ -0,0 +1,414 @@ +//go:build e2e + +package e2e + +import ( + "context" + "fmt" + "io" + "net/http" + "strings" + "testing" + "time" + + osmv1 "github.com/openshift/api/monitoring/v1" + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + + "github.com/openshift/monitoring-plugin/pkg/k8s" + "github.com/openshift/monitoring-plugin/pkg/management/metrics" + "github.com/openshift/monitoring-plugin/test/e2e/framework" +) + +const ( + metricPollInterval = 5 * time.Second + metricPollTimeout = 3 * time.Minute +) + +func fetchMetrics(ctx context.Context, f *framework.Framework) (metricsBody string, err error) { + req, err := f.AuthorizedRequest(ctx, http.MethodGet, f.PluginURL+"/metrics", nil) + if err != nil { + return "", err + } + + resp, err := f.HTTPClient().Do(req) + if err != nil { + return "", err + } + defer func() { + if closeErr := resp.Body.Close(); closeErr != nil && err == nil { + err = fmt.Errorf("closing response body: %w", closeErr) + } + }() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("unexpected status code: %d", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + return string(body), nil +} + +func parseMetricLines(body string) []string { + var lines []string + for _, line := range strings.Split(body, "\n") { + if strings.HasPrefix(line, metrics.MetricName+"{") { + lines = append(lines, line) + } + } + return lines +} + +func extractLabel(metricLine, labelName string) string { + key := labelName + `="` + idx := strings.Index(metricLine, key) + if idx < 0 { + return "" + } + start := idx + len(key) + end := strings.Index(metricLine[start:], `"`) + if end < 0 { + return "" + } + return metricLine[start : start+end] +} + +func parseGaugeValue(line string) (float64, error) { + idx := strings.LastIndex(line, " ") + if idx < 0 { + return 0, fmt.Errorf("metric line has no value: %s", line) + } + var ts float64 + if _, err := fmt.Sscanf(line[idx+1:], "%g", &ts); err != nil { + return 0, fmt.Errorf("unparseable value %q: %w", line[idx+1:], err) + } + return ts, nil +} + +func waitForMetricLines(t *testing.T, f *framework.Framework, match func(string) bool) []string { + t.Helper() + ctx := context.Background() + var matched []string + err := framework.Poll(metricPollInterval, metricPollTimeout, func() error { + body, err := fetchMetrics(ctx, f) + if err != nil { + return err + } + var found []string + for _, line := range parseMetricLines(body) { + if match == nil || match(line) { + found = append(found, line) + } + } + if len(found) == 0 { + return fmt.Errorf("no matching %s series yet", metrics.MetricName) + } + matched = found + return nil + }) + if err != nil { + t.Fatalf("Timeout waiting for metric series: %v", err) + } + return matched +} + +func alwaysFiringRule(alertName, exprLabel string, labels map[string]string) monitoringv1.Rule { + forDuration := monitoringv1.Duration("1s") + ruleLabels := map[string]string{ + "severity": "none", + } + for k, v := range labels { + ruleLabels[k] = v + } + return monitoringv1.Rule{ + Alert: alertName, + Expr: intstr.FromString(fmt.Sprintf(`absent(nonexistent{test_label=%q})`, exprLabel)), + For: &forDuration, + Labels: ruleLabels, + Annotations: map[string]string{ + "summary": "e2e effective metric test", + "description": "should not appear as a metric label", + "runbook_url": "https://example.invalid/runbook", + }, + } +} + +func assertRequiredMetricLabels(t *testing.T, line string) { + t.Helper() + requiredLabels := []string{ + "alertname", + "alertstate", + k8s.AlertSourceLabel, + k8s.AlertBackendLabel, + } + for _, label := range requiredLabels { + if extractLabel(line, label) == "" { + t.Errorf("missing required label %q: %s", label, line) + } + } + + state := extractLabel(line, "alertstate") + switch state { + case "firing", "pending", "silenced": + default: + t.Errorf("unexpected alertstate=%q: %s", state, line) + } + + ts, err := parseGaugeValue(line) + if err != nil { + t.Errorf("%v", err) + return + } + if ts < 9.46e+08 { + t.Errorf("suspiciously low timestamp value: %g (before year 2000): %s", ts, line) + } +} + +func assertNoAnnotationLabels(t *testing.T, line string) { + t.Helper() + for _, annLabel := range []string{"summary", "description", "runbook_url"} { + if extractLabel(line, annLabel) != "" { + t.Errorf("contains annotation label %q (annotations should be excluded): %s", + annLabel, line) + } + } +} + +func assertClassificationLabels(t *testing.T, line string) { + t.Helper() + if extractLabel(line, k8s.AlertRuleClassificationComponentKey) == "" { + t.Errorf("missing %s label: %s", k8s.AlertRuleClassificationComponentKey, line) + } + if extractLabel(line, k8s.AlertRuleClassificationLayerKey) == "" { + t.Errorf("missing %s label: %s", k8s.AlertRuleClassificationLayerKey, line) + } +} + +// TestMetricEndpointExposesEffectiveMetric verifies that /metrics exposes +// alerts_effective_active_at_timestamp_seconds as a gauge. +func TestMetricEndpointExposesEffectiveMetric(t *testing.T) { + f, err := framework.New() + if err != nil { + t.Fatalf("Failed to create framework: %v", err) + } + + ctx := context.Background() + var metricBody string + err = framework.Poll(metricPollInterval, metricPollTimeout, func() error { + body, err := fetchMetrics(ctx, f) + if err != nil { + return err + } + if !strings.Contains(body, metrics.MetricName) { + return fmt.Errorf("metric %s not found yet (leader election may be in progress)", metrics.MetricName) + } + metricBody = body + return nil + }) + if err != nil { + t.Fatalf("Timeout waiting for metric to appear: %v", err) + } + + if !strings.Contains(metricBody, "# HELP "+metrics.MetricName) { + t.Error("Missing HELP line for metric") + } + if !strings.Contains(metricBody, "# TYPE "+metrics.MetricName+" gauge") { + t.Error("Missing or incorrect TYPE line for metric (expected gauge)") + } + + lines := parseMetricLines(metricBody) + if len(lines) == 0 { + t.Fatal("Expected at least one metric series, got none") + } + t.Logf("Found %d metric series for %s", len(lines), metrics.MetricName) +} + +// TestMetricSeriesHaveRequiredLabels verifies every series has alertname, +// alertstate, source, backend, and a valid timestamp. +func TestMetricSeriesHaveRequiredLabels(t *testing.T) { + f, err := framework.New() + if err != nil { + t.Fatalf("Failed to create framework: %v", err) + } + + lines := waitForMetricLines(t, f, nil) + for _, line := range lines { + assertRequiredMetricLabels(t, line) + } + t.Logf("All %d series have required labels and valid values", len(lines)) +} + +// TestMetricIncludesClassificationLabels verifies component and layer labels. +func TestMetricIncludesClassificationLabels(t *testing.T) { + f, err := framework.New() + if err != nil { + t.Fatalf("Failed to create framework: %v", err) + } + + lines := waitForMetricLines(t, f, nil) + for _, line := range lines { + assertClassificationLabels(t, line) + } + t.Logf("All %d series have classification labels (component + layer)", len(lines)) +} + +// TestMetricExcludesAnnotations verifies summary/description/runbook_url are +// not metric labels. +func TestMetricExcludesAnnotations(t *testing.T) { + f, err := framework.New() + if err != nil { + t.Fatalf("Failed to create framework: %v", err) + } + + lines := waitForMetricLines(t, f, nil) + for _, line := range lines { + assertNoAnnotationLabels(t, line) + } + t.Logf("Verified %d series - none contain annotation labels", len(lines)) +} + +// TestMetricIncludesCreatedFiringAlert creates a unique always-firing alert +// and checks it appears on the effective metric with expected labels and a +// timestamp that matches GET /alerts ActiveAt. +func TestMetricIncludesCreatedFiringAlert(t *testing.T) { + f, err := framework.New() + if err != nil { + t.Fatalf("Failed to create framework: %v", err) + } + + ctx := context.Background() + testNamespace, cleanup, err := f.CreateUserNamespace(ctx, "test-effective-metric") + if err != nil { + t.Fatalf("Failed to create test namespace: %v", err) + } + defer func() { + if err := cleanup(); err != nil { + t.Logf("cleanup failed: %v", err) + } + }() + + alertName := "E2EEffectiveMetricFiring" + rule := alwaysFiringRule(alertName, "e2e_effective_metric_firing", map[string]string{ + "team": "e2e", + }) + if _, err := createPrometheusRule(ctx, f, testNamespace, rule); err != nil { + t.Fatalf("Failed to create PrometheusRule: %v", err) + } + + lines := waitForMetricLines(t, f, func(line string) bool { + return extractLabel(line, "alertname") == alertName + }) + line := lines[0] + assertRequiredMetricLabels(t, line) + assertClassificationLabels(t, line) + assertNoAnnotationLabels(t, line) + + if got := extractLabel(line, "team"); got != "e2e" { + t.Errorf("expected team=e2e, got %q", got) + } + state := extractLabel(line, "alertstate") + if state != "firing" && state != "pending" { + t.Errorf("expected alertstate firing or pending, got %q", state) + } + + metricTS, err := parseGaugeValue(line) + if err != nil { + t.Fatalf("%v", err) + } + + // Collector EnrichAlerts uses the cluster-wide path; match that here so + // tenancy lag on a brand-new namespace cannot flake the timestamp check. + alerts, status, err := getAlertsWithToken(f, ctx, f.BearerToken, "") + if err != nil { + t.Fatalf("GET /alerts failed: %v", err) + } + if status != http.StatusOK { + t.Fatalf("GET /alerts status %d", status) + } + var alertActiveAt time.Time + for _, alert := range alerts { + if alert.Labels["alertname"] == alertName { + alertActiveAt = alert.ActiveAt + break + } + } + if alertActiveAt.IsZero() { + t.Fatalf("GET /alerts did not return %s", alertName) + } + if got := float64(alertActiveAt.Unix()); got != metricTS { + t.Errorf("metric timestamp %g does not match GET /alerts ActiveAt %g", metricTS, got) + } +} + +// TestMetricReflectsAlertRelabelConfig creates a firing platform alert and an +// ARC that rewrites team, then checks the metric exposes the post-relabel +// value. +func TestMetricReflectsAlertRelabelConfig(t *testing.T) { + f, err := framework.New() + if err != nil { + t.Fatalf("Failed to create framework: %v", err) + } + + ctx := context.Background() + testNamespace, cleanup, err := f.CreatePlatformNamespace(ctx, "test-effective-metric-relabel") + if err != nil { + t.Fatalf("Failed to create test namespace: %v", err) + } + defer func() { + if err := cleanup(); err != nil { + t.Logf("cleanup failed: %v", err) + } + }() + + alertName := "E2EEffectiveMetricRelabel" + rule := alwaysFiringRule(alertName, "e2e_effective_metric_relabel", map[string]string{ + "team": "web", + }) + if _, err := createPrometheusRule(ctx, f, testNamespace, rule); err != nil { + t.Fatalf("Failed to create PrometheusRule: %v", err) + } + + relabelConfigName := "e2e-effective-metric-team" + arc := &osmv1.AlertRelabelConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: relabelConfigName, + Namespace: k8s.ClusterMonitoringNamespace, + }, + Spec: osmv1.AlertRelabelConfigSpec{ + Configs: []osmv1.RelabelConfig{ + { + SourceLabels: []osmv1.LabelName{"alertname"}, + Regex: alertName, + TargetLabel: "team", + Replacement: "ops", + Action: "Replace", + }, + }, + }, + } + _, err = f.Osmv1clientset.MonitoringV1().AlertRelabelConfigs(k8s.ClusterMonitoringNamespace).Create( + ctx, arc, metav1.CreateOptions{}, + ) + if err != nil { + t.Fatalf("Failed to create AlertRelabelConfig: %v", err) + } + defer func() { + err := f.Osmv1clientset.MonitoringV1().AlertRelabelConfigs(k8s.ClusterMonitoringNamespace).Delete( + ctx, relabelConfigName, metav1.DeleteOptions{}, + ) + if err != nil { + t.Logf("Failed to delete AlertRelabelConfig: %v", err) + } + }() + + lines := waitForMetricLines(t, f, func(line string) bool { + return extractLabel(line, "alertname") == alertName && extractLabel(line, "team") == "ops" + }) + assertRequiredMetricLabels(t, lines[0]) + assertNoAnnotationLabels(t, lines[0]) + t.Logf("Relabeling verified on metric: %s", lines[0]) +}