// Copyright The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package dispatch

import (
	"context"
	"fmt"
	"log/slog"
	"reflect"
	"runtime"
	"sort"
	"sync"
	"testing"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/testutil"
	"github.com/prometheus/common/model"
	"github.com/prometheus/common/promslog"
	"github.com/stretchr/testify/require"

	"github.com/prometheus/alertmanager/alert"
	"github.com/prometheus/alertmanager/config"
	"github.com/prometheus/alertmanager/eventrecorder"
	"github.com/prometheus/alertmanager/featurecontrol"
	"github.com/prometheus/alertmanager/marker"
	"github.com/prometheus/alertmanager/notify"
	"github.com/prometheus/alertmanager/provider/mem"
)

const testMaintenanceInterval = 30 * time.Second

func TestAggrGroup(t *testing.T) {
	lset := model.LabelSet{
		"a": "v1",
		"b": "v2",
	}
	opts := &RouteOpts{
		Receiver: "n1",
		GroupBy: map[model.LabelName]struct{}{
			"a": {},
			"b": {},
		},
		GroupWait:      1 * time.Second,
		GroupInterval:  300 * time.Millisecond,
		RepeatInterval: 1 * time.Hour,
	}
	route := &Route{
		RouteOpts: *opts,
	}

	var (
		a1 = &alert.Alert{
			Alert: model.Alert{
				Labels: model.LabelSet{
					"a": "v1",
					"b": "v2",
					"c": "v3",
				},
				StartsAt: time.Now().Add(time.Minute),
				EndsAt:   time.Now().Add(time.Hour),
			},
			UpdatedAt: time.Now(),
		}
		a2 = &alert.Alert{
			Alert: model.Alert{
				Labels: model.LabelSet{
					"a": "v1",
					"b": "v2",
					"c": "v4",
				},
				StartsAt: time.Now().Add(-time.Hour),
				EndsAt:   time.Now().Add(2 * time.Hour),
			},
			UpdatedAt: time.Now(),
		}
		a3 = &alert.Alert{
			Alert: model.Alert{
				Labels: model.LabelSet{
					"a": "v1",
					"b": "v2",
					"c": "v5",
				},
				StartsAt: time.Now().Add(time.Minute),
				EndsAt:   time.Now().Add(5 * time.Minute),
			},
			UpdatedAt: time.Now(),
		}
	)

	type batch struct {
		alerts alert.AlertSlice
		now    time.Time // timer fire time, propagated via notify.WithNow
	}
	batchCh := make(chan batch)

	ntfy := func(ctx context.Context, alerts ...*alert.Alert) bool {
		// Validate that the context is properly populated.
		now, ok := notify.Now(ctx)
		if !ok {
			t.Errorf("now missing")
		}
		if _, ok := notify.GroupKey(ctx); !ok {
			t.Errorf("group key missing")
		}
		if lbls, ok := notify.GroupLabels(ctx); !ok || !reflect.DeepEqual(lbls, lset) {
			t.Errorf("wrong group labels: %q", lbls)
		}
		if rcv, ok := notify.ReceiverName(ctx); !ok || rcv != opts.Receiver {
			t.Errorf("wrong receiver: %q", rcv)
		}
		if ri, ok := notify.RepeatInterval(ctx); !ok || ri != opts.RepeatInterval {
			t.Errorf("wrong repeat interval: %q", ri)
		}

		batchCh <- batch{alerts: alerts, now: now}
		return true
	}

	removeEndsAt := func(as alert.AlertSlice) alert.AlertSlice {
		for i, a := range as {
			ac := *a
			ac.EndsAt = time.Time{}
			as[i] = &ac
		}
		return as
	}

	// receiveBatch waits for the next flush, asserts it didn't fire earlier than
	// minWait after `since`, and returns the timer-reported fire time. We use
	// the timer's own `now` (propagated via notify.WithNow) instead of a
	// time.Now() snapshot inside ntfy — the latter accumulates scheduler jitter
	// between the timer firing and ntfy actually running, which made the
	// assertion flake under load.
	receiveBatch := func(t *testing.T, since time.Time, minWait time.Duration, want alert.AlertSlice) time.Time {
		t.Helper()
		select {
		case <-time.After(2 * minWait):
			t.Fatalf("expected new batch after %v but received none", minWait)
		case b := <-batchCh:
			if got := b.now.Sub(since); got < minWait {
				t.Fatalf("received batch too early after %v (want >= %v)", got, minWait)
			}
			sort.Sort(b.alerts)
			if !reflect.DeepEqual(b.alerts, want) {
				t.Fatalf("expected alerts %v but got %v", want, b.alerts)
			}
			return b.now
		}
		return time.Time{}
	}

	// Test regular situation where we wait for group_wait to send out alerts.
	createdAt := time.Now()
	ag := newAggrGroup(context.Background(), lset, route, nil, eventrecorder.NopRecorder(), promslog.NewNopLogger())
	go ag.run(ntfy)

	ctx := context.Background()
	ag.insert(ctx, a1)

	last := receiveBatch(t, createdAt, opts.GroupWait, removeEndsAt(alert.AlertSlice{a1}))

	for range 3 {
		// New alert should come in after group interval.
		ag.insert(ctx, a3)
		last = receiveBatch(t, last, opts.GroupInterval, removeEndsAt(alert.AlertSlice{a1, a3}))
	}

	ag.stop()

	// Finally, set all alerts to be resolved. After successful notify the aggregation group
	// should empty itself.
	createdAt = time.Now()
	ag = newAggrGroup(context.Background(), lset, route, nil, eventrecorder.NopRecorder(), promslog.NewNopLogger())
	go ag.run(ntfy)

	ag.insert(ctx, a1)
	ag.insert(ctx, a2)

	last = receiveBatch(t, createdAt, opts.GroupWait, removeEndsAt(alert.AlertSlice{a1, a2}))

	for range 3 {
		// New alert should come in after group interval.
		ag.insert(ctx, a3)
		last = receiveBatch(t, last, opts.GroupInterval, removeEndsAt(alert.AlertSlice{a1, a2, a3}))
	}

	// Resolve an alert, and it should be removed after the next batch was sent.
	a1r := *a1
	a1r.EndsAt = time.Now()
	ag.insert(ctx, &a1r)
	last = receiveBatch(t, last, opts.GroupInterval, append(alert.AlertSlice{&a1r}, removeEndsAt(alert.AlertSlice{a2, a3})...))

	// Resolve all remaining alerts, they should be removed after the next batch was sent.
	// Do not add a1r as it should have been deleted following the previous batch.
	a2r, a3r := *a2, *a3
	resolved := alert.AlertSlice{&a2r, &a3r}
	for _, a := range resolved {
		a.EndsAt = time.Now()
		ag.insert(ctx, a)
	}
	receiveBatch(t, last, opts.GroupInterval, resolved)
	// ntfy unblocks before flush() finishes deleting resolved alerts, so poll.
	require.Eventually(t, ag.empty, time.Second, 10*time.Millisecond,
		"Expected aggregation group to be empty after resolving alerts: %v", ag)

	ag.stop()
}

func TestGroupLabels(t *testing.T) {
	a := &alert.Alert{
		Alert: model.Alert{
			Labels: model.LabelSet{
				"a": "v1",
				"b": "v2",
				"c": "v3",
			},
		},
	}

	route := &Route{
		RouteOpts: RouteOpts{
			GroupBy: map[model.LabelName]struct{}{
				"a": {},
				"b": {},
			},
			GroupByAll: false,
		},
	}

	expLs := model.LabelSet{
		"a": "v1",
		"b": "v2",
	}

	ls := getGroupLabels(a, route)

	if !reflect.DeepEqual(ls, expLs) {
		t.Fatalf("expected labels are %v, but got %v", expLs, ls)
	}
}

func TestGroupByAllLabels(t *testing.T) {
	a := &alert.Alert{
		Alert: model.Alert{
			Labels: model.LabelSet{
				"a": "v1",
				"b": "v2",
				"c": "v3",
			},
		},
	}

	route := &Route{
		RouteOpts: RouteOpts{
			GroupBy:    map[model.LabelName]struct{}{},
			GroupByAll: true,
		},
	}

	expLs := model.LabelSet{
		"a": "v1",
		"b": "v2",
		"c": "v3",
	}

	ls := getGroupLabels(a, route)

	if !reflect.DeepEqual(ls, expLs) {
		t.Fatalf("expected labels are %v, but got %v", expLs, ls)
	}
}

func TestGroups(t *testing.T) {
	confData := `receivers:
- name: 'kafka'
- name: 'prod'
- name: 'testing'

route:
  group_by: ['alertname']
  group_wait: 10ms
  group_interval: 10ms
  receiver: 'prod'
  routes:
  - match:
      env: 'testing'
    receiver: 'testing'
    group_by: ['alertname', 'service']
  - match:
      env: 'prod'
    receiver: 'prod'
    group_by: ['alertname', 'service', 'cluster']
    continue: true
  - match:
      kafka: 'yes'
    receiver: 'kafka'
    group_by: ['alertname', 'service', 'cluster']`
	conf, err := config.Load(confData)
	if err != nil {
		t.Fatal(err)
	}

	logger := promslog.NewNopLogger()
	route := NewRoute(conf.Route, nil)
	reg := prometheus.NewRegistry()
	marker := marker.NewGroupMarker()
	alerts, err := mem.NewAlerts(context.Background(), time.Hour, 0, nil, logger, eventrecorder.NopRecorder(), reg, nil)
	if err != nil {
		t.Fatal(err)
	}
	defer alerts.Close()

	timeout := func(d time.Duration) time.Duration { return time.Duration(0) }
	recorder := &recordStage{alerts: make(map[string]map[model.Fingerprint]*alert.Alert)}
	dispatcher := NewDispatcher(alerts, route, recorder, marker, timeout, testMaintenanceInterval, nil, logger, eventrecorder.NopRecorder(), NewDispatcherMetrics(false, reg, nil))
	go dispatcher.Run(time.Now())
	defer dispatcher.Stop()

	// Create alerts. the dispatcher will automatically create the groups.
	inputAlerts := []*alert.Alert{
		// Matches the parent route.
		newAlert(model.LabelSet{"alertname": "OtherAlert", "cluster": "cc", "service": "dd"}),
		// Matches the first sub-route.
		newAlert(model.LabelSet{"env": "testing", "alertname": "TestingAlert", "service": "api", "instance": "inst1"}),
		// Matches the second sub-route.
		newAlert(model.LabelSet{"env": "prod", "alertname": "HighErrorRate", "cluster": "aa", "service": "api", "instance": "inst1"}),
		newAlert(model.LabelSet{"env": "prod", "alertname": "HighErrorRate", "cluster": "aa", "service": "api", "instance": "inst2"}),
		// Matches the second sub-route.
		newAlert(model.LabelSet{"env": "prod", "alertname": "HighErrorRate", "cluster": "bb", "service": "api", "instance": "inst1"}),
		// Matches the second and third sub-route.
		newAlert(model.LabelSet{"env": "prod", "alertname": "HighLatency", "cluster": "bb", "service": "db", "kafka": "yes", "instance": "inst3"}),
	}
	alerts.Put(context.Background(), inputAlerts...)

	// Let alerts get processed.
	for i := 0; len(recorder.Alerts()) != 7 && i < 10; i++ {
		time.Sleep(200 * time.Millisecond)
	}
	require.Len(t, recorder.Alerts(), 7)

	alertGroups, receivers, _ := dispatcher.Groups(context.Background(),
		func(*Route) bool {
			return true
		}, func(*alert.Alert, time.Time) bool {
			return true
		},
	)

	// Each group should have pre-computed AlertStatuses. Verify and then
	// nil them out so the struct comparison below works.
	for _, ag := range alertGroups {
		require.NotNil(t, ag.AlertStatuses)
		ag.AlertStatuses = nil
	}

	require.Equal(t, AlertGroups{
		&AlertGroup{
			Alerts: []*alert.Alert{inputAlerts[0]},
			Labels: model.LabelSet{
				"alertname": "OtherAlert",
			},
			Receiver: "prod",
			GroupKey: "{}:{alertname=\"OtherAlert\"}",
			RouteID:  "{}",
		},
		&AlertGroup{
			Alerts: []*alert.Alert{inputAlerts[1]},
			Labels: model.LabelSet{
				"alertname": "TestingAlert",
				"service":   "api",
			},
			Receiver: "testing",
			GroupKey: "{}/{env=\"testing\"}:{alertname=\"TestingAlert\", service=\"api\"}",
			RouteID:  "{}/{env=\"testing\"}/0",
		},
		&AlertGroup{
			Alerts: []*alert.Alert{inputAlerts[2], inputAlerts[3]},
			Labels: model.LabelSet{
				"alertname": "HighErrorRate",
				"service":   "api",
				"cluster":   "aa",
			},
			Receiver: "prod",
			GroupKey: "{}/{env=\"prod\"}:{alertname=\"HighErrorRate\", cluster=\"aa\", service=\"api\"}",
			RouteID:  "{}/{env=\"prod\"}/1",
		},
		&AlertGroup{
			Alerts: []*alert.Alert{inputAlerts[4]},
			Labels: model.LabelSet{
				"alertname": "HighErrorRate",
				"service":   "api",
				"cluster":   "bb",
			},
			Receiver: "prod",
			GroupKey: "{}/{env=\"prod\"}:{alertname=\"HighErrorRate\", cluster=\"bb\", service=\"api\"}",
			RouteID:  "{}/{env=\"prod\"}/1",
		},
		&AlertGroup{
			Alerts: []*alert.Alert{inputAlerts[5]},
			Labels: model.LabelSet{
				"alertname": "HighLatency",
				"service":   "db",
				"cluster":   "bb",
			},
			Receiver: "kafka",
			GroupKey: "{}/{kafka=\"yes\"}:{alertname=\"HighLatency\", cluster=\"bb\", service=\"db\"}",
			RouteID:  "{}/{kafka=\"yes\"}/2",
		},
		&AlertGroup{
			Alerts: []*alert.Alert{inputAlerts[5]},
			Labels: model.LabelSet{
				"alertname": "HighLatency",
				"service":   "db",
				"cluster":   "bb",
			},
			Receiver: "prod",
			GroupKey: "{}/{env=\"prod\"}:{alertname=\"HighLatency\", cluster=\"bb\", service=\"db\"}",
			RouteID:  "{}/{env=\"prod\"}/1",
		},
	}, alertGroups)
	require.Equal(t, map[model.Fingerprint][]string{
		inputAlerts[0].Fingerprint(): {"prod"},
		inputAlerts[1].Fingerprint(): {"testing"},
		inputAlerts[2].Fingerprint(): {"prod"},
		inputAlerts[3].Fingerprint(): {"prod"},
		inputAlerts[4].Fingerprint(): {"prod"},
		inputAlerts[5].Fingerprint(): {"kafka", "prod"},
	}, receivers)
}

func TestGroupsWithLimits(t *testing.T) {
	confData := `receivers:
- name: 'kafka'
- name: 'prod'
- name: 'testing'

route:
  group_by: ['alertname']
  group_wait: 10ms
  group_interval: 10ms
  receiver: 'prod'
  routes:
  - match:
      env: 'testing'
    receiver: 'testing'
    group_by: ['alertname', 'service']
  - match:
      env: 'prod'
    receiver: 'prod'
    group_by: ['alertname', 'service', 'cluster']
    continue: true
  - match:
      kafka: 'yes'
    receiver: 'kafka'
    group_by: ['alertname', 'service', 'cluster']`
	conf, err := config.Load(confData)
	if err != nil {
		t.Fatal(err)
	}

	logger := promslog.NewNopLogger()
	route := NewRoute(conf.Route, nil)
	reg := prometheus.NewRegistry()
	marker := marker.NewGroupMarker()
	alerts, err := mem.NewAlerts(context.Background(), time.Hour, 0, nil, logger, eventrecorder.NopRecorder(), reg, nil)
	if err != nil {
		t.Fatal(err)
	}
	defer alerts.Close()

	timeout := func(d time.Duration) time.Duration { return time.Duration(0) }
	recorder := &recordStage{alerts: make(map[string]map[model.Fingerprint]*alert.Alert)}
	lim := limits{groups: 6}
	m := NewDispatcherMetrics(true, reg, nil)
	dispatcher := NewDispatcher(alerts, route, recorder, marker, timeout, testMaintenanceInterval, lim, logger, eventrecorder.NopRecorder(), m)
	go dispatcher.Run(time.Now())
	defer dispatcher.Stop()

	// Create alerts. the dispatcher will automatically create the groups.
	inputAlerts := []*alert.Alert{
		// Matches the parent route.
		newAlert(model.LabelSet{"alertname": "OtherAlert", "cluster": "cc", "service": "dd"}),
		// Matches the first sub-route.
		newAlert(model.LabelSet{"env": "testing", "alertname": "TestingAlert", "service": "api", "instance": "inst1"}),
		// Matches the second sub-route.
		newAlert(model.LabelSet{"env": "prod", "alertname": "HighErrorRate", "cluster": "aa", "service": "api", "instance": "inst1"}),
		newAlert(model.LabelSet{"env": "prod", "alertname": "HighErrorRate", "cluster": "aa", "service": "api", "instance": "inst2"}),
		// Matches the second sub-route.
		newAlert(model.LabelSet{"env": "prod", "alertname": "HighErrorRate", "cluster": "bb", "service": "api", "instance": "inst1"}),
		// Matches the second and third sub-route.
		newAlert(model.LabelSet{"env": "prod", "alertname": "HighLatency", "cluster": "bb", "service": "db", "kafka": "yes", "instance": "inst3"}),
	}
	err = alerts.Put(context.Background(), inputAlerts...)
	if err != nil {
		t.Fatal(err)
	}

	// Let alerts get processed.
	for i := 0; len(recorder.Alerts()) != 7 && i < 10; i++ {
		time.Sleep(200 * time.Millisecond)
	}
	require.Len(t, recorder.Alerts(), 7)

	routeFilter := func(*Route) bool { return true }
	alertFilter := func(*alert.Alert, time.Time) bool { return true }

	alertGroups, _, _ := dispatcher.Groups(context.Background(), routeFilter, alertFilter)
	require.Len(t, alertGroups, 6)

	require.Equal(t, 0.0, testutil.ToFloat64(m.aggrGroupLimitReached))

	// Try to store new alert. This time, we will hit limit for number of groups.
	err = alerts.Put(context.Background(), newAlert(model.LabelSet{"env": "prod", "alertname": "NewAlert", "cluster": "new-cluster", "service": "db"}))
	if err != nil {
		t.Fatal(err)
	}

	// Let alert get processed.
	for i := 0; testutil.ToFloat64(m.aggrGroupLimitReached) == 0 && i < 10; i++ {
		time.Sleep(200 * time.Millisecond)
	}
	require.Equal(t, 1.0, testutil.ToFloat64(m.aggrGroupLimitReached))

	// Verify there are still only 6 groups.
	alertGroups, _, _ = dispatcher.Groups(context.Background(), routeFilter, alertFilter)
	require.Len(t, alertGroups, 6)
}

type recordStage struct {
	mtx    sync.RWMutex
	alerts map[string]map[model.Fingerprint]*alert.Alert
}

func (r *recordStage) Alerts() []*alert.Alert {
	r.mtx.RLock()
	defer r.mtx.RUnlock()
	alerts := make([]*alert.Alert, 0)
	for k := range r.alerts {
		for _, a := range r.alerts[k] {
			alerts = append(alerts, a)
		}
	}
	return alerts
}

func (r *recordStage) Exec(ctx context.Context, l *slog.Logger, alerts ...*alert.Alert) (context.Context, []*alert.Alert, error) {
	r.mtx.Lock()
	defer r.mtx.Unlock()
	gk, ok := notify.GroupKey(ctx)
	if !ok {
		panic("GroupKey not present!")
	}
	if _, ok := r.alerts[gk]; !ok {
		r.alerts[gk] = make(map[model.Fingerprint]*alert.Alert)
	}
	for _, a := range alerts {
		r.alerts[gk][a.Fingerprint()] = a
	}
	return ctx, nil, nil
}

var (
	// Set the start time in the past to trigger a flush immediately.
	t0 = time.Now().Add(-time.Minute)
	// Set the end time in the future to avoid deleting the alert.
	t1 = t0.Add(2 * time.Minute)
)

func newAlert(labels model.LabelSet) *alert.Alert {
	return &alert.Alert{
		Alert: model.Alert{
			Labels:       labels,
			Annotations:  model.LabelSet{"foo": "bar"},
			StartsAt:     t0,
			EndsAt:       t1,
			GeneratorURL: "http://example.com/prometheus",
		},
		UpdatedAt: t0,
		Timeout:   false,
	}
}

func TestDispatcherRace(t *testing.T) {
	logger := promslog.NewNopLogger()
	reg := prometheus.NewRegistry()
	marker := marker.NewGroupMarker()
	alerts, err := mem.NewAlerts(context.Background(), time.Hour, 0, nil, logger, eventrecorder.NopRecorder(), reg, nil)
	if err != nil {
		t.Fatal(err)
	}
	defer alerts.Close()

	timeout := func(d time.Duration) time.Duration { return time.Duration(0) }
	route := &Route{}
	dispatcher := NewDispatcher(alerts, route, nil, marker, timeout, testMaintenanceInterval, nil, logger, eventrecorder.NopRecorder(), NewDispatcherMetrics(false, reg, nil))
	go dispatcher.Run(time.Now())
	dispatcher.Stop()
}

func TestDispatcherRaceOnFirstAlertNotDeliveredWhenGroupWaitIsZero(t *testing.T) {
	const numAlerts = 5000

	logger := promslog.NewNopLogger()
	reg := prometheus.NewRegistry()
	marker := marker.NewGroupMarker()
	alerts, err := mem.NewAlerts(context.Background(), time.Hour, 0, nil, logger, eventrecorder.NopRecorder(), reg, nil)
	if err != nil {
		t.Fatal(err)
	}
	defer alerts.Close()

	route := &Route{
		RouteOpts: RouteOpts{
			Receiver:       "default",
			GroupBy:        map[model.LabelName]struct{}{"alertname": {}},
			GroupWait:      0,
			GroupInterval:  1 * time.Hour, // Should never hit in this test.
			RepeatInterval: 1 * time.Hour, // Should never hit in this test.
		},
	}

	timeout := func(d time.Duration) time.Duration { return d }
	recorder := &recordStage{alerts: make(map[string]map[model.Fingerprint]*alert.Alert)}
	dispatcher := NewDispatcher(alerts, route, recorder, marker, timeout, testMaintenanceInterval, nil, logger, eventrecorder.NopRecorder(), NewDispatcherMetrics(false, reg, nil))
	go dispatcher.Run(time.Now())
	defer dispatcher.Stop()

	// Push all alerts.
	for i := range numAlerts {
		alert := newAlert(model.LabelSet{"alertname": model.LabelValue(fmt.Sprintf("Alert_%d", i))})
		require.NoError(t, alerts.Put(context.Background(), alert))
	}

	// Wait until the alerts have been notified or the waiting timeout expires.
	for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); {
		if len(recorder.Alerts()) >= numAlerts {
			break
		}

		// Throttle.
		time.Sleep(10 * time.Millisecond)
	}

	// We expect all alerts to be notified immediately, since they all belong to different groups.
	require.Len(t, recorder.Alerts(), numAlerts)
}

type limits struct {
	groups int
}

func (l limits) MaxNumberOfAggregationGroups() int {
	return l.groups
}

func TestDispatcher_DoMaintenance(t *testing.T) {
	r := prometheus.NewRegistry()
	marker := marker.NewGroupMarker()

	alerts, err := mem.NewAlerts(context.Background(), time.Minute, 0, nil, promslog.NewNopLogger(), eventrecorder.NopRecorder(), r, nil)
	if err != nil {
		t.Fatal(err)
	}

	route := &Route{
		RouteOpts: RouteOpts{
			GroupBy:       map[model.LabelName]struct{}{"alertname": {}},
			GroupWait:     0,
			GroupInterval: 5 * time.Minute, // Should never hit in this test.
		},
		Idx: 0,
	}
	timeout := func(d time.Duration) time.Duration { return d }
	recorder := &recordStage{alerts: make(map[string]map[model.Fingerprint]*alert.Alert)}

	ctx := context.Background()
	dispatcher := NewDispatcher(alerts, route, recorder, marker, timeout, testMaintenanceInterval, nil, promslog.NewNopLogger(), eventrecorder.NopRecorder(), NewDispatcherMetrics(false, r, nil))
	// Manually create the routeAggrGroups structure since we are not calling Run().
	dispatcher.routeGroupsSlice = make([]routeAggrGroups, route.Idx+1)
	dispatcher.routeGroupsSlice[route.Idx] = routeAggrGroups{
		route: route,
	}

	// Insert an aggregation group with one resolved alert.
	labels := model.LabelSet{"alertname": "1"}
	aggrGroup1 := newAggrGroup(ctx, labels, route, timeout, eventrecorder.NopRecorder(), promslog.NewNopLogger())
	dispatcher.routeGroupsSlice[route.Idx].groups.Store(aggrGroup1.fingerprint(), aggrGroup1)

	// Add a resolved alert
	resolvedAlert := &alert.Alert{
		Alert: model.Alert{
			Labels:   labels,
			StartsAt: time.Now().Add(-2 * time.Hour),
			EndsAt:   time.Now().Add(-1 * time.Hour), // Already resolved
		},
		UpdatedAt: time.Now().Add(-1 * time.Hour),
	}
	aggrGroup1.alerts.Set(resolvedAlert)

	// Flush will detect the resolved alert and delete it via DeleteIfNotModified
	// This is the actual production code path
	notified := false
	aggrGroup1.flush(func(alerts ...*alert.Alert) bool {
		require.Len(t, alerts, 1)
		require.Equal(t, labels, alerts[0].Labels)
		notified = true
		return true // Simulate successful notification
	})
	require.True(t, notified, "flush should have called notify function")

	// Must run otherwise doMaintenance blocks on aggrGroup1.stop().
	go aggrGroup1.run(func(context.Context, ...*alert.Alert) bool { return true })

	// Insert a marker for the aggregation group's group key.
	marker.SetMuted(route.ID(), aggrGroup1.GroupKey(), []string{"weekends"})
	mutedBy, isMuted := marker.Muted(route.ID(), aggrGroup1.GroupKey())
	require.True(t, isMuted)
	require.Equal(t, []string{"weekends"}, mutedBy)

	// Run the maintenance and the marker should be removed.
	dispatcher.doMaintenance()
	mutedBy, isMuted = marker.Muted(route.ID(), aggrGroup1.GroupKey())
	require.False(t, isMuted)
	require.Empty(t, mutedBy)
}

// TestGroupAlert_RecoversWhenCASFails reproduces a bug where groupAlert would
// retry the same CompareAndSwap with a stale `el` reference after the slot at
// fp had been mutated by a competing goroutine (or removed by maintenance).
// Without the fix, CAS losers spin until the 100-retry give-up and lose their
// alert. With the fix, they fall back to LoadOrStore and insert into whichever
// live group now occupies the slot.
//
// Each round pre-stores a destroyed aggrGroup at fp and fires many concurrent
// groupAlert calls. We keep running rounds until we observe the contended CAS
// branch firing at least once (cap at maxRounds to avoid hangs on a pathologic
// scheduler) — a single round can be unlucky and serialize, taking the early
// Load+insert path on every goroutine.
func TestGroupAlert_RecoversWhenCASFails(t *testing.T) {
	const (
		alertsPerRound = 200
		maxRounds      = 50
	)

	logger := promslog.NewNopLogger()
	reg := prometheus.NewRegistry()
	alerts, err := mem.NewAlerts(context.Background(), time.Hour, 0, nil, logger, eventrecorder.NopRecorder(), reg, nil)
	require.NoError(t, err)
	defer alerts.Close()

	route := &Route{
		RouteOpts: RouteOpts{
			Receiver:       "test",
			GroupBy:        map[model.LabelName]struct{}{"alertname": {}},
			GroupWait:      time.Hour, // never flush during this test
			GroupInterval:  time.Hour,
			RepeatInterval: time.Hour,
		},
		Idx: 0,
	}
	timeout := func(d time.Duration) time.Duration { return d }
	recorder := &recordStage{alerts: make(map[string]map[model.Fingerprint]*alert.Alert)}
	metrics := NewDispatcherMetrics(false, reg, featurecontrol.NoopFlags{})
	dispatcher := NewDispatcher(alerts, route, recorder, marker.NewGroupMarker(), timeout, testMaintenanceInterval, nil, logger, eventrecorder.NopRecorder(), metrics)
	// Don't call Run — put the dispatcher manually in  the DispatcherStateWaitingToStart
	// state so groupAlert's final switch falls through the default branch and the
	// aggregation group's run goroutine is never started.
	dispatcher.routeGroupsSlice = []routeAggrGroups{{route: route}}
	dispatcher.state.Store(DispatcherStateWaitingToStart) // silences the warn that would happen in unknown mode.
	rounds := 0
	for rounds < maxRounds && testutil.ToFloat64(metrics.aggrGroupCreationRetries) == 0 {
		groupLabels := model.LabelSet{"alertname": model.LabelValue(fmt.Sprintf("shared-%d", rounds))}
		destroyedAg := newAggrGroup(context.Background(), groupLabels, route, timeout, eventrecorder.NopRecorder(), logger)
		// Mark the store destroyed: empty store + destroyIfEmpty=true.
		require.NoError(t, destroyedAg.alerts.DeleteIfNotModified(alert.AlertSlice{}, true))
		require.True(t, destroyedAg.destroyed())
		fp := destroyedAg.fingerprint()
		dispatcher.routeGroupsSlice[0].groups.Store(fp, destroyedAg)

		var ready, done sync.WaitGroup
		ready.Add(alertsPerRound)
		done.Add(alertsPerRound)
		start := make(chan struct{})
		for i := range alertsPerRound {
			go func() {
				defer done.Done()
				alert := newAlert(model.LabelSet{
					"alertname": groupLabels["alertname"],
					"instance":  model.LabelValue(fmt.Sprintf("inst-%d", i)),
				})
				ready.Done()
				<-start
				dispatcher.groupAlert(context.Background(), alert, route)
			}()
		}
		ready.Wait()
		close(start)
		done.Wait()

		el, ok := dispatcher.routeGroupsSlice[0].groups.Load(fp)
		require.True(t, ok, "round %d: a live group must occupy the fp after the race", rounds)
		finalAg := el.(*aggrGroup)
		require.False(t, finalAg.destroyed(), "round %d: the final group must not be destroyed", rounds)
		require.NotSame(t, destroyedAg, finalAg, "round %d: destroyed group must have been replaced", rounds)
		require.Len(t, finalAg.alerts.List(), alertsPerRound, "round %d: all alerts must land in the final group", rounds)
		rounds++
	}

	// Give-ups must stay 0: losers must recover via LoadOrStore, not spin to
	// the 100-retry limit.
	require.Zero(t, testutil.ToFloat64(metrics.aggrGroupCreationGivenUp), "no alert should be dropped to the give-up branch")

	// With GOMAXPROCS=1 the contention can't be fully exercised.
	// Skip the check that retries > 0.
	if runtime.GOMAXPROCS(0) == 1 {
		return
	}

	// Retries > 0 proves the test actually exercised the contended CAS-recovery
	// branch (rather than every goroutine taking the early Load+insert path).
	require.Positive(t, testutil.ToFloat64(metrics.aggrGroupCreationRetries), "contended CAS path was not exercised in %d rounds — scheduler is unusually serial", rounds)
}

// TestGroupAlert_DisplacedAggrGroupGoroutineExits is a regression test for a
// goroutine leak: when groupAlert CAS-replaces a destroyed aggrGroup in the
// map, the displaced group's run goroutine must be torn down. Otherwise it
// stays parked in its select forever (doMaintenance can no longer find it
// because it's been removed from the map), accumulating one stuck goroutine
// per replacement for the lifetime of the process.
func TestGroupAlert_DisplacedAggrGroupGoroutineExits(t *testing.T) {
	logger := promslog.NewNopLogger()
	reg := prometheus.NewRegistry()
	alerts, err := mem.NewAlerts(context.Background(), time.Hour, 0, nil, logger, eventrecorder.NopRecorder(), reg, nil)
	require.NoError(t, err)
	defer alerts.Close()

	route := &Route{
		RouteOpts: RouteOpts{
			Receiver:       "test",
			GroupBy:        map[model.LabelName]struct{}{"alertname": {}},
			GroupWait:      time.Hour, // never auto-flush during the test
			GroupInterval:  time.Hour,
			RepeatInterval: time.Hour,
		},
		Idx: 0,
	}
	timeout := func(d time.Duration) time.Duration { return d }
	recorder := &recordStage{alerts: make(map[string]map[model.Fingerprint]*alert.Alert)}
	dispatcher := NewDispatcher(alerts, route, recorder, marker.NewGroupMarker(), timeout, testMaintenanceInterval, nil, logger, eventrecorder.NopRecorder(), NewDispatcherMetrics(false, reg, featurecontrol.NoopFlags{}))
	dispatcher.routeGroupsSlice = []routeAggrGroups{{route: route}}
	// WaitingToStart so groupAlert won't auto-start the new ag — keeps the
	// test focused on the displaced group.
	dispatcher.state.Store(DispatcherStateWaitingToStart)

	groupLabels := model.LabelSet{"alertname": "displaced"}
	displaced := newAggrGroup(context.Background(), groupLabels, route, timeout, eventrecorder.NopRecorder(), logger)
	// Mark destroyed so groupAlert can't insert into it and is forced down
	// the CAS-replace path.
	require.NoError(t, displaced.alerts.DeleteIfNotModified(alert.AlertSlice{}, true))
	require.True(t, displaced.destroyed())
	dispatcher.routeGroupsSlice[0].groups.Store(displaced.fingerprint(), displaced)

	// Start the run goroutine on the displaced group — this is the orphan
	// candidate. Without the fix it would never exit.
	go displaced.run(func(context.Context, ...*alert.Alert) bool { return true })

	// Trigger the CAS replacement.
	dispatcher.groupAlert(context.Background(), newAlert(groupLabels), route)

	// The displaced group should have been swapped out for a fresh one.
	el, ok := dispatcher.routeGroupsSlice[0].groups.Load(displaced.fingerprint())
	require.True(t, ok)
	require.NotSame(t, displaced, el.(*aggrGroup), "destroyed group must have been replaced")

	// And its run goroutine must have exited.
	select {
	case <-displaced.done:
	case <-time.After(2 * time.Second):
		t.Fatal("displaced aggrGroup.run goroutine did not exit after CAS replacement")
	}
}

func TestDispatcher_DeleteResolvedAlertsFromMarker(t *testing.T) {
	t.Run("successful flush deletes markers for resolved alerts", func(t *testing.T) {
		ctx := context.Background()
		labels := model.LabelSet{"alertname": "TestAlert"}
		route := &Route{
			RouteOpts: RouteOpts{
				Receiver:       "test",
				GroupBy:        map[model.LabelName]struct{}{"alertname": {}},
				GroupWait:      0,
				GroupInterval:  time.Minute,
				RepeatInterval: time.Hour,
			},
		}
		timeout := func(d time.Duration) time.Duration { return d }
		logger := promslog.NewNopLogger()

		// Create an aggregation group
		ag := newAggrGroup(ctx, labels, route, timeout, eventrecorder.NopRecorder(), logger)

		// Create test alerts: one active and one resolved
		now := time.Now()
		activeAlert := &alert.Alert{
			Alert: model.Alert{
				Labels: model.LabelSet{
					"alertname": "TestAlert",
					"instance":  "1",
				},
				StartsAt: now.Add(-time.Hour),
				EndsAt:   now.Add(time.Hour), // Active alert
			},
			UpdatedAt: now,
		}
		resolvedAlert := &alert.Alert{
			Alert: model.Alert{
				Labels: model.LabelSet{
					"alertname": "TestAlert",
					"instance":  "2",
				},
				StartsAt: now.Add(-time.Hour),
				EndsAt:   now.Add(-time.Minute), // Resolved alert
			},
			UpdatedAt: now,
		}

		// Insert alerts into the aggregation group
		ag.insert(ctx, activeAlert)
		ag.insert(ctx, resolvedAlert)

		// Set markers for both alerts via the per-group marker
		ag.marker.SetSilenced(activeAlert.Fingerprint(), nil)
		ag.marker.SetSilenced(resolvedAlert.Fingerprint(), nil)

		// Verify markers exist before flush
		require.Equal(t, alert.AlertStateActive, ag.marker.Status(activeAlert.Fingerprint()).State)
		require.Equal(t, alert.AlertStateActive, ag.marker.Status(resolvedAlert.Fingerprint()).State)

		// Create a notify function that succeeds
		notifyFunc := func(alerts ...*alert.Alert) bool {
			return true
		}

		// Flush the alerts
		ag.flush(notifyFunc)

		// Verify that the resolved alert's marker was deleted (status becomes unprocessed)
		require.Equal(t, alert.AlertStateActive, ag.marker.Status(activeAlert.Fingerprint()).State, "active alert marker should still exist")
		require.Equal(t, alert.AlertStateUnprocessed, ag.marker.Status(resolvedAlert.Fingerprint()).State, "resolved alert marker should be deleted")
	})

	t.Run("failed flush does not delete markers", func(t *testing.T) {
		ctx := context.Background()
		labels := model.LabelSet{"alertname": "TestAlert"}
		route := &Route{
			RouteOpts: RouteOpts{
				Receiver:       "test",
				GroupBy:        map[model.LabelName]struct{}{"alertname": {}},
				GroupWait:      0,
				GroupInterval:  time.Minute,
				RepeatInterval: time.Hour,
			},
		}
		timeout := func(d time.Duration) time.Duration { return d }
		logger := promslog.NewNopLogger()

		// Create an aggregation group
		ag := newAggrGroup(ctx, labels, route, timeout, eventrecorder.NopRecorder(), logger)

		// Create a resolved alert
		now := time.Now()
		resolvedAlert := &alert.Alert{
			Alert: model.Alert{
				Labels: model.LabelSet{
					"alertname": "TestAlert",
					"instance":  "1",
				},
				StartsAt: now.Add(-time.Hour),
				EndsAt:   now.Add(-time.Minute), // Resolved alert
			},
			UpdatedAt: now,
		}

		// Insert alert into the aggregation group
		ag.insert(ctx, resolvedAlert)

		// Set marker for the alert via the per-group marker
		ag.marker.SetSilenced(resolvedAlert.Fingerprint(), nil)

		// Verify marker exists before flush
		require.Equal(t, alert.AlertStateActive, ag.marker.Status(resolvedAlert.Fingerprint()).State)

		// Create a notify function that fails
		notifyFunc := func(alerts ...*alert.Alert) bool {
			return false
		}

		// Flush the alerts (notify will fail)
		ag.flush(notifyFunc)

		// Verify that the marker was NOT deleted due to failed notification
		require.Equal(t, alert.AlertStateActive, ag.marker.Status(resolvedAlert.Fingerprint()).State, "marker should not be deleted when notify fails")
	})

	t.Run("markers not deleted when alert is modified during flush", func(t *testing.T) {
		ctx := context.Background()
		labels := model.LabelSet{"alertname": "TestAlert"}
		route := &Route{
			RouteOpts: RouteOpts{
				Receiver:       "test",
				GroupBy:        map[model.LabelName]struct{}{"alertname": {}},
				GroupWait:      0,
				GroupInterval:  time.Minute,
				RepeatInterval: time.Hour,
			},
		}
		timeout := func(d time.Duration) time.Duration { return d }
		logger := promslog.NewNopLogger()

		// Create an aggregation group
		ag := newAggrGroup(ctx, labels, route, timeout, eventrecorder.NopRecorder(), logger)

		// Create a resolved alert
		now := time.Now()
		resolvedAlert := &alert.Alert{
			Alert: model.Alert{
				Labels: model.LabelSet{
					"alertname": "TestAlert",
					"instance":  "1",
				},
				StartsAt: now.Add(-time.Hour),
				EndsAt:   now.Add(-time.Minute), // Resolved alert
			},
			UpdatedAt: now,
		}

		// Insert alert into the aggregation group
		ag.insert(ctx, resolvedAlert)

		// Set marker for the alert via the per-group marker
		ag.marker.SetSilenced(resolvedAlert.Fingerprint(), nil)

		// Verify marker exists before flush
		require.Equal(t, alert.AlertStateActive, ag.marker.Status(resolvedAlert.Fingerprint()).State)

		// Create a notify function that modifies the alert before returning
		notifyFunc := func(alerts ...*alert.Alert) bool {
			// Simulate the alert being modified (e.g., firing again) during flush
			modifiedAlert := &alert.Alert{
				Alert: model.Alert{
					Labels: model.LabelSet{
						"alertname": "TestAlert",
						"instance":  "1",
					},
					StartsAt: now.Add(-time.Hour),
					EndsAt:   now.Add(time.Hour), // Active again
				},
				UpdatedAt: now.Add(time.Second), // More recent update
			}
			// Update the alert in the store
			ag.alerts.Set(modifiedAlert)
			return true
		}

		// Flush the alerts
		ag.flush(notifyFunc)

		// Verify that the marker was NOT deleted because the alert was modified
		// during the flush (DeleteIfNotModified should have failed)
		require.Equal(t, alert.AlertStateActive, ag.marker.Status(resolvedAlert.Fingerprint()).State, "marker should not be deleted when alert is modified during flush")
	})
}

func TestDispatchOnStartup(t *testing.T) {
	logger := promslog.NewNopLogger()
	reg := prometheus.NewRegistry()
	marker := marker.NewGroupMarker()
	alerts, err := mem.NewAlerts(context.Background(), time.Hour, 0, nil, logger, eventrecorder.NopRecorder(), reg, nil)
	if err != nil {
		t.Fatal(err)
	}
	defer alerts.Close()

	// Set up a route with GroupBy to separate alerts into different aggregation groups.
	route := &Route{
		RouteOpts: RouteOpts{
			Receiver:       "default",
			GroupBy:        map[model.LabelName]struct{}{"instance": {}},
			GroupWait:      1 * time.Second,
			GroupInterval:  3 * time.Minute,
			RepeatInterval: 1 * time.Hour,
		},
	}

	recorder := &recordStage{alerts: make(map[string]map[model.Fingerprint]*alert.Alert)}
	timeout := func(d time.Duration) time.Duration { return d }

	// Set start time to 3 seconds in the future
	now := time.Now()
	startDelay := 2 * time.Second
	startTime := time.Now().Add(startDelay)
	dispatcher := NewDispatcher(alerts, route, recorder, marker, timeout, testMaintenanceInterval, nil, logger, eventrecorder.NopRecorder(), NewDispatcherMetrics(false, reg, nil))
	go dispatcher.Run(startTime)
	defer dispatcher.Stop()

	// Create 2 similar alerts with start times in the past
	alert1 := &alert.Alert{
		Alert: model.Alert{
			Labels:       model.LabelSet{"alertname": "TestAlert1", "instance": "1"},
			Annotations:  model.LabelSet{"foo": "bar"},
			StartsAt:     now.Add(-1 * time.Hour),
			EndsAt:       now.Add(time.Hour),
			GeneratorURL: "http://example.com/prometheus",
		},
		UpdatedAt: now,
		Timeout:   false,
	}

	alert2 := &alert.Alert{
		Alert: model.Alert{
			Labels:       model.LabelSet{"alertname": "TestAlert2", "instance": "2"},
			Annotations:  model.LabelSet{"foo": "bar"},
			StartsAt:     now.Add(-1 * time.Hour),
			EndsAt:       now.Add(time.Hour),
			GeneratorURL: "http://example.com/prometheus",
		},
		UpdatedAt: now,
		Timeout:   false,
	}

	// Send alert1
	require.NoError(t, alerts.Put(context.Background(), alert1))

	var recordedAlerts []*alert.Alert
	// Expect a recorded alert after startTime + GroupWait which is in future
	require.Eventually(t, func() bool {
		recordedAlerts = recorder.Alerts()
		return len(recordedAlerts) == 1
	}, startDelay+route.RouteOpts.GroupWait, 500*time.Millisecond)

	require.Equal(t, alert1.Fingerprint(), recordedAlerts[0].Fingerprint(), "expected alert1 to be dispatched after GroupWait")

	// Send alert2
	require.NoError(t, alerts.Put(context.Background(), alert2))

	// Expect a recorded alert after GroupInterval
	require.Eventually(t, func() bool {
		recordedAlerts = recorder.Alerts()
		return len(recordedAlerts) == 2
	}, route.RouteOpts.GroupInterval, 100*time.Millisecond)

	// Sort alerts by fingerprint for deterministic ordering
	sort.Slice(recordedAlerts, func(i, j int) bool {
		return recordedAlerts[i].Fingerprint() < recordedAlerts[j].Fingerprint()
	})
	require.Equal(t, alert2.Fingerprint(), recordedAlerts[1].Fingerprint(), "expected alert2 to be dispatched after GroupInterval")

	// Verify both alerts are present
	fingerprints := make(map[model.Fingerprint]bool)
	for _, a := range recordedAlerts {
		fingerprints[a.Fingerprint()] = true
	}
	require.True(t, fingerprints[alert1.Fingerprint()], "expected alert1 to be present")
	require.True(t, fingerprints[alert2.Fingerprint()], "expected alert2 to be present")
}

func TestGetGroupLabels(t *testing.T) {
	alert := &alert.Alert{
		Alert: model.Alert{
			Labels: model.LabelSet{
				"alertname": "TestAlert",
				"job":       "prometheus",
				"instance":  "localhost:9090",
				"severity":  "critical",
			},
		},
	}

	t.Run("specific labels", func(t *testing.T) {
		route := &Route{
			RouteOpts: RouteOpts{
				GroupBy: map[model.LabelName]struct{}{
					"alertname": {},
					"job":       {},
				},
			},
		}
		labels := getGroupLabels(alert, route)
		require.Len(t, labels, 2)
		require.Equal(t, model.LabelValue("TestAlert"), labels["alertname"])
		require.Equal(t, model.LabelValue("prometheus"), labels["job"])
	})

	t.Run("group by all", func(t *testing.T) {
		route := &Route{
			RouteOpts: RouteOpts{
				GroupByAll: true,
			},
		}
		labels := getGroupLabels(alert, route)
		require.Len(t, labels, 4)
		require.Equal(t, alert.Labels, labels)
	})
}

func BenchmarkGetGroupLabels(b *testing.B) {
	now := time.Now()

	// Alert with many labels (typical production alert)
	alert := &alert.Alert{
		Alert: model.Alert{
			Labels: model.LabelSet{
				"alertname":  "TestAlert",
				"severity":   "critical",
				"job":        "prometheus",
				"instance":   "localhost:9090",
				"namespace":  "monitoring",
				"cluster":    "prod-us-east-1",
				"datacenter": "dc1",
				"env":        "production",
				"team":       "platform",
				"service":    "alertmanager",
			},
			StartsAt: now.Add(-time.Hour),
			EndsAt:   now.Add(time.Hour),
		},
	}

	b.Run("specific_labels", func(b *testing.B) {
		route := &Route{
			RouteOpts: RouteOpts{
				GroupBy: map[model.LabelName]struct{}{
					"alertname": {},
					"job":       {},
					"severity":  {},
				},
			},
		}

		b.ResetTimer()
		b.ReportAllocs()

		for i := 0; i < b.N; i++ {
			_ = getGroupLabels(alert, route)
		}
	})

	b.Run("group_by_all", func(b *testing.B) {
		route := &Route{
			RouteOpts: RouteOpts{
				GroupByAll: true,
			},
		}

		b.ResetTimer()
		b.ReportAllocs()

		for i := 0; i < b.N; i++ {
			_ = getGroupLabels(alert, route)
		}
	})
}
