mirror of
https://github.com/hanzoai/o11y.git
synced 2026-10-15 12:58:33 +00:00
zeekay
b9f9cdf918
Re-sync the entire SigNoz-derived Go tree to a clean fork of latest SigNoz main (upstream 3e6339019), eliminating the version-skew that broke `go build ./...` (mixed-era packages, ~14 imported-but-absent packages, partial signoz->o11y / clickhouse->datastore rebrands, self-import cycle, duplicate package decls). - pkg/ + cmd/ wholesale replaced with latest SigNoz; imports rewritten github.com/SigNoz/signoz/ -> github.com/hanzoai/o11y/. - Adopt SigNoz's CURRENT structure (resourcefilter->telemetryresourcefilter, integrationtypes->cloudintegrationtypes, roletypes/grammar/times folded in upstream); old paths dropped. - Fork deps repathed to hanzoai + bumped: signoz-otel-collector v0.144.6, clickhouse-go-mock v0.14.1, govaluate v0.1.0; expr-lang/expr => hanzoai/expr v1.17.8 (replace). go.mod reconciled from latest SigNoz; go.sum re-tidied. - ee/ + cmd/enterprise stripped — o11y stays MIT-only; community edition imports no ee packages. - Preserved o11y additions: module path github.com/hanzoai/o11y, mount.go (cloud.Register/zip mount adapter, HIP-0106), NOTICE + LICENSE (SigNoz §4c attribution retained; upstream pkg files are headerless per SigNoz convention). - Reverted o11y cosmetic renames (o11y*/datastore*) and experimental originals (authz/iamauthz, billing, zapreceiver, zapmetricreceiver) to SigNoz canonical to kill the skew; snapshots retained in git history for re-layering. go build ./... = exit 0. go vet clean except benign "unreachable code" in generated ANTLR parser files (inherited from upstream).
299 lines
10 KiB
Go
299 lines
10 KiB
Go
package alertmanagerserver
|
|
|
|
import (
|
|
"context"
|
|
"log/slog"
|
|
"net/http"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/hanzoai/o11y/pkg/types/alertmanagertypes/alertmanagertypestest"
|
|
"github.com/prometheus/alertmanager/dispatch"
|
|
|
|
"github.com/hanzoai/o11y/pkg/alertmanager/nfmanager"
|
|
"github.com/hanzoai/o11y/pkg/alertmanager/nfmanager/nfroutingstore/nfroutingstoretest"
|
|
"github.com/hanzoai/o11y/pkg/alertmanager/nfmanager/rulebasednotification"
|
|
"github.com/hanzoai/o11y/pkg/instrumentation/instrumentationtest"
|
|
"github.com/hanzoai/o11y/pkg/types"
|
|
"github.com/hanzoai/o11y/pkg/types/alertmanagertypes"
|
|
"github.com/hanzoai/o11y/pkg/valuer"
|
|
|
|
"github.com/go-openapi/strfmt"
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
"github.com/prometheus/common/model"
|
|
|
|
"github.com/stretchr/testify/mock"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestEndToEndAlertManagerFlow(t *testing.T) {
|
|
ctx := context.Background()
|
|
providerSettings := instrumentationtest.New().ToProviderSettings()
|
|
|
|
store := nfroutingstoretest.NewMockSQLRouteStore()
|
|
store.MatchExpectationsInOrder(false)
|
|
notificationManager, err := rulebasednotification.New(ctx, providerSettings, nfmanager.Config{}, store)
|
|
require.NoError(t, err)
|
|
orgID := "test-org"
|
|
|
|
routes := []*alertmanagertypes.RoutePolicy{
|
|
{
|
|
Identifiable: types.Identifiable{
|
|
ID: valuer.GenerateUUID(),
|
|
},
|
|
Expression: `ruleId = "high-cpu-usage" AND severity = "critical"`,
|
|
ExpressionKind: alertmanagertypes.RuleBasedExpression,
|
|
Name: "high-cpu-usage",
|
|
Description: "High CPU critical alerts to webhook",
|
|
Enabled: true,
|
|
OrgID: orgID,
|
|
Channels: []string{"webhook"},
|
|
},
|
|
{
|
|
Identifiable: types.Identifiable{
|
|
ID: valuer.GenerateUUID(),
|
|
},
|
|
Expression: `ruleId = "high-cpu-usage" AND severity = "warning"`,
|
|
ExpressionKind: alertmanagertypes.RuleBasedExpression,
|
|
Name: "high-cpu-usage",
|
|
Description: "High CPU warning alerts to webhook",
|
|
Enabled: true,
|
|
OrgID: orgID,
|
|
Channels: []string{"webhook"},
|
|
},
|
|
}
|
|
|
|
store.ExpectCreateBatch(routes)
|
|
err = notificationManager.CreateRoutePolicies(ctx, orgID, routes)
|
|
require.NoError(t, err)
|
|
|
|
for range routes {
|
|
ruleID := "high-cpu-usage"
|
|
store.ExpectGetAllByName(orgID, ruleID, routes)
|
|
store.ExpectGetAllByName(orgID, ruleID, routes)
|
|
}
|
|
|
|
notifConfig := alertmanagertypes.NotificationConfig{
|
|
NotificationGroup: map[model.LabelName]struct{}{
|
|
model.LabelName("cluster"): {},
|
|
model.LabelName("instance"): {},
|
|
},
|
|
Renotify: alertmanagertypes.ReNotificationConfig{
|
|
RenotifyInterval: 5 * time.Minute,
|
|
},
|
|
UsePolicy: false,
|
|
}
|
|
|
|
err = notificationManager.SetNotificationConfig(orgID, "high-cpu-usage", ¬ifConfig)
|
|
require.NoError(t, err)
|
|
|
|
activeSchedule := &alertmanagertypes.Schedule{
|
|
Timezone: "UTC",
|
|
StartTime: time.Now().Add(-time.Hour),
|
|
EndTime: time.Now().Add(time.Hour),
|
|
}
|
|
// mwRuleIDAndScope: only critical high-cpu-usage alerts.
|
|
mwRuleIDAndScope := valuer.GenerateUUID()
|
|
// mwRuleIDOnly: all high-cpu-usage alerts regardless of severity.
|
|
mwRuleIDOnly := valuer.GenerateUUID()
|
|
// mwScopeOnly: all critical alerts regardless of rule ID.
|
|
mwScopeOnly := valuer.GenerateUUID()
|
|
|
|
maintenanceStore := alertmanagertypestest.NewMockMaintenanceStore(t)
|
|
maintenanceStore.On("ListPlannedMaintenance", mock.Anything, orgID).Return(
|
|
[]*alertmanagertypes.PlannedMaintenance{
|
|
{ID: mwRuleIDAndScope, Schedule: activeSchedule, RuleIDs: []string{"high-cpu-usage"}, Scope: `severity = "critical"`},
|
|
{ID: mwRuleIDOnly, Schedule: activeSchedule, RuleIDs: []string{"high-cpu-usage"}},
|
|
{ID: mwScopeOnly, Schedule: activeSchedule, Scope: `severity = "critical"`},
|
|
}, nil,
|
|
)
|
|
|
|
srvCfg := NewConfig()
|
|
stateStore := alertmanagertypestest.NewStateStore()
|
|
registry := prometheus.NewRegistry()
|
|
logger := slog.New(slog.DiscardHandler)
|
|
server, err := New(context.Background(), logger, registry, srvCfg, orgID, stateStore, notificationManager, maintenanceStore)
|
|
require.NoError(t, err)
|
|
amConfig, err := alertmanagertypes.NewDefaultConfig(srvCfg.Global, srvCfg.Route, orgID)
|
|
require.NoError(t, err)
|
|
err = server.SetConfig(ctx, amConfig)
|
|
require.NoError(t, err)
|
|
|
|
// Create test alerts
|
|
now := time.Now()
|
|
testAlerts := []*alertmanagertypes.PostableAlert{
|
|
{
|
|
Alert: alertmanagertypes.AlertModel{
|
|
Labels: map[string]string{
|
|
"ruleId": "high-cpu-usage",
|
|
"severity": "critical",
|
|
"cluster": "prod-cluster",
|
|
"instance": "server-01",
|
|
"alertname": "HighCPUUsage",
|
|
},
|
|
},
|
|
Annotations: map[string]string{
|
|
"summary": "High CPU usage detected",
|
|
"description": "CPU usage is above 90% for 5 minutes",
|
|
},
|
|
StartsAt: strfmt.DateTime(now.Add(-5 * time.Minute)),
|
|
EndsAt: strfmt.DateTime(time.Time{}), // Active alert
|
|
},
|
|
{
|
|
Alert: alertmanagertypes.AlertModel{
|
|
Labels: map[string]string{
|
|
"ruleId": "high-cpu-usage",
|
|
"severity": "warning",
|
|
"cluster": "prod-cluster",
|
|
"instance": "server-02",
|
|
"alertname": "HighCPUUsage",
|
|
},
|
|
},
|
|
Annotations: map[string]string{
|
|
"summary": "Moderate CPU usage detected",
|
|
"description": "CPU usage is above 70% for 10 minutes",
|
|
},
|
|
StartsAt: strfmt.DateTime(now.Add(-10 * time.Minute)),
|
|
EndsAt: strfmt.DateTime(time.Time{}), // Active alert
|
|
},
|
|
{
|
|
Alert: alertmanagertypes.AlertModel{
|
|
Labels: map[string]string{
|
|
"ruleId": "high-cpu-usage",
|
|
"severity": "critical",
|
|
"cluster": "prod-cluster",
|
|
"instance": "server-03",
|
|
"alertname": "HighCPUUsage",
|
|
},
|
|
},
|
|
Annotations: map[string]string{
|
|
"summary": "High CPU usage detected on server-03",
|
|
"description": "CPU usage is above 95% for 3 minutes",
|
|
},
|
|
StartsAt: strfmt.DateTime(now.Add(-3 * time.Minute)),
|
|
EndsAt: strfmt.DateTime(time.Time{}), // Active alert
|
|
},
|
|
{
|
|
Alert: alertmanagertypes.AlertModel{
|
|
Labels: map[string]string{
|
|
"ruleId": "other-rule",
|
|
"alertname": "OtherAlert",
|
|
},
|
|
},
|
|
StartsAt: strfmt.DateTime(now.Add(-time.Minute)),
|
|
EndsAt: strfmt.DateTime(time.Time{}), // Active alert
|
|
},
|
|
}
|
|
|
|
err = server.PutAlerts(ctx, testAlerts)
|
|
require.NoError(t, err)
|
|
|
|
time.Sleep(2 * time.Second)
|
|
|
|
t.Run("verify_alerts_processed", func(t *testing.T) {
|
|
dummyRequest, err := http.NewRequest(http.MethodGet, "/alerts", nil)
|
|
require.NoError(t, err)
|
|
|
|
params, err := alertmanagertypes.NewGettableAlertsParams(dummyRequest)
|
|
require.NoError(t, err)
|
|
alerts, err := server.GetAlerts(context.Background(), params)
|
|
require.NoError(t, err)
|
|
require.Len(t, alerts, 4, "Expected 4 active alerts")
|
|
|
|
for _, alert := range alerts {
|
|
if alert.Labels["ruleId"] != "high-cpu-usage" {
|
|
continue
|
|
}
|
|
require.NotEmpty(t, alert.Labels["severity"])
|
|
require.Contains(t, []string{"critical", "warning"}, alert.Labels["severity"])
|
|
require.Equal(t, "prod-cluster", alert.Labels["cluster"])
|
|
require.NotEmpty(t, alert.Labels["instance"])
|
|
}
|
|
|
|
criticalAlerts := 0
|
|
warningAlerts := 0
|
|
for _, alert := range alerts {
|
|
switch alert.Labels["severity"] {
|
|
case "critical":
|
|
criticalAlerts++
|
|
case "warning":
|
|
warningAlerts++
|
|
}
|
|
}
|
|
require.Equal(t, 2, criticalAlerts, "Expected 2 critical alerts")
|
|
require.Equal(t, 1, warningAlerts, "Expected 1 warning alert")
|
|
})
|
|
|
|
t.Run("verify_notification_routing", func(t *testing.T) {
|
|
|
|
notifConfig, err := notificationManager.GetNotificationConfig(orgID, "high-cpu-usage")
|
|
require.NoError(t, err)
|
|
require.NotNil(t, notifConfig)
|
|
require.Equal(t, 5*time.Minute, notifConfig.Renotify.RenotifyInterval)
|
|
require.Contains(t, notifConfig.NotificationGroup, model.LabelName("ruleId"))
|
|
require.Contains(t, notifConfig.NotificationGroup, model.LabelName("cluster"))
|
|
require.Contains(t, notifConfig.NotificationGroup, model.LabelName("instance"))
|
|
})
|
|
|
|
t.Run("verify_alert_groups_and_stages", func(t *testing.T) {
|
|
time.Sleep(2 * time.Second)
|
|
|
|
alertGroups, _ := server.dispatcher.Groups(
|
|
func(route *dispatch.Route) bool { return true }, // Accept all routes
|
|
func(alert *alertmanagertypes.Alert, now time.Time) bool { return true }, // Accept all alerts
|
|
)
|
|
require.Len(t, alertGroups, 3)
|
|
|
|
require.NotEmpty(t, alertGroups, "Should have alert groups created by dispatcher")
|
|
|
|
totalAlerts := 0
|
|
for _, group := range alertGroups {
|
|
totalAlerts += len(group.Alerts)
|
|
}
|
|
require.Equal(t, 3, totalAlerts, "Should have 3 alerts total across all groups")
|
|
require.Equal(t, "{__receiver__=\"webhook\"}:{cluster=\"prod-cluster\", instance=\"server-01\", ruleId=\"high-cpu-usage\"}", alertGroups[0].GroupKey)
|
|
require.Equal(t, "{__receiver__=\"webhook\"}:{cluster=\"prod-cluster\", instance=\"server-02\", ruleId=\"high-cpu-usage\"}", alertGroups[1].GroupKey)
|
|
require.Equal(t, "{__receiver__=\"webhook\"}:{cluster=\"prod-cluster\", instance=\"server-03\", ruleId=\"high-cpu-usage\"}", alertGroups[2].GroupKey)
|
|
})
|
|
|
|
req, err := http.NewRequest(http.MethodGet, "/alerts", nil)
|
|
require.NoError(t, err)
|
|
params, err := alertmanagertypes.NewGettableAlertsParams(req)
|
|
require.NoError(t, err)
|
|
alerts, err := server.GetAlerts(ctx, params)
|
|
require.NoError(t, err)
|
|
|
|
t.Run("verify_muting_ruleid_and_scope", func(t *testing.T) {
|
|
// Window with ruleID + scope mutes only alerts matching both.
|
|
for _, alert := range alerts {
|
|
if alert.Labels["ruleId"] == "high-cpu-usage" && alert.Labels["severity"] == "critical" {
|
|
require.Contains(t, alert.Status.MutedBy, mwRuleIDAndScope.String())
|
|
} else {
|
|
require.NotContains(t, alert.Status.MutedBy, mwRuleIDAndScope.String())
|
|
}
|
|
}
|
|
})
|
|
|
|
t.Run("verify_muting_ruleid_only", func(t *testing.T) {
|
|
// Window with ruleID but no scope mutes all severities for that rule.
|
|
for _, alert := range alerts {
|
|
if alert.Labels["ruleId"] == "high-cpu-usage" {
|
|
require.Contains(t, alert.Status.MutedBy, mwRuleIDOnly.String())
|
|
} else {
|
|
require.NotContains(t, alert.Status.MutedBy, mwRuleIDOnly.String())
|
|
}
|
|
}
|
|
})
|
|
|
|
t.Run("verify_muting_scope_only", func(t *testing.T) {
|
|
// Window with scope but no ruleIDs mutes all critical alerts regardless of rule.
|
|
for _, alert := range alerts {
|
|
if alert.Labels["severity"] == "critical" {
|
|
require.Contains(t, alert.Status.MutedBy, mwScopeOnly.String())
|
|
} else {
|
|
require.NotContains(t, alert.Status.MutedBy, mwScopeOnly.String())
|
|
}
|
|
}
|
|
})
|
|
}
|