alertmanager/silence/silence_test.go

// Copyright 2016 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package silence

import (
	"bytes"
	"fmt"
	"os"
	"runtime"
	"sort"
	"sync"
	"testing"
	"time"

	"github.com/benbjohnson/clock"
	"github.com/go-kit/log"
	"github.com/matttproud/golang_protobuf_extensions/pbutil"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/testutil"
	"github.com/prometheus/common/model"
	"github.com/stretchr/testify/require"
	"go.uber.org/atomic"

	pb "github.com/prometheus/alertmanager/silence/silencepb"
	"github.com/prometheus/alertmanager/types"
)

func checkErr(t *testing.T, expected string, got error) {
	t.Helper()

	if expected == "" {
		require.NoError(t, got)
		return
	}

	if got == nil {
		t.Errorf("expected error containing %q but got none", expected)
		return
	}

	require.Contains(t, got.Error(), expected)
}

func TestOptionsValidate(t *testing.T) {
	cases := []struct {
		options *Options
		err     string
	}{
		{
			options: &Options{
				SnapshotReader: &bytes.Buffer{},
			},
		},
		{
			options: &Options{
				SnapshotFile: "test.bkp",
			},
		},
		{
			options: &Options{
				SnapshotFile:   "test bkp",
				SnapshotReader: &bytes.Buffer{},
			},
			err: "only one of SnapshotFile and SnapshotReader must be set",
		},
	}

	for _, c := range cases {
		checkErr(t, c.err, c.options.validate())
	}
}

func TestSilencesGC(t *testing.T) {
	s, err := New(Options{})
	require.NoError(t, err)

	s.clock = clock.NewMock()
	now := s.nowUTC()

	newSilence := func(exp time.Time) *pb.MeshSilence {
		return &pb.MeshSilence{ExpiresAt: exp}
	}
	s.st = state{
		"1": newSilence(now),
		"2": newSilence(now.Add(-time.Second)),
		"3": newSilence(now.Add(time.Second)),
	}
	want := state{
		"3": newSilence(now.Add(time.Second)),
	}

	n, err := s.GC()
	require.NoError(t, err)
	require.Equal(t, 2, n)
	require.Equal(t, want, s.st)
}

func TestSilencesSnapshot(t *testing.T) {
	// Check whether storing and loading the snapshot is symmetric.
	now := clock.NewMock().Now().UTC()

	cases := []struct {
		entries []*pb.MeshSilence
	}{
		{
			entries: []*pb.MeshSilence{
				{
					Silence: &pb.Silence{
						Id: "3be80475-e219-4ee7-b6fc-4b65114e362f",
						Matchers: []*pb.Matcher{
							{Name: "label1", Pattern: "val1", Type: pb.Matcher_EQUAL},
							{Name: "label2", Pattern: "val.+", Type: pb.Matcher_REGEXP},
						},
						StartsAt:  now,
						EndsAt:    now,
						UpdatedAt: now,
					},
					ExpiresAt: now,
				},
				{
					Silence: &pb.Silence{
						Id: "3dfb2528-59ce-41eb-b465-f875a4e744a4",
						Matchers: []*pb.Matcher{
							{Name: "label1", Pattern: "val1", Type: pb.Matcher_NOT_EQUAL},
							{Name: "label2", Pattern: "val.+", Type: pb.Matcher_NOT_REGEXP},
						},
						StartsAt:  now,
						EndsAt:    now,
						UpdatedAt: now,
					},
					ExpiresAt: now,
				},
				{
					Silence: &pb.Silence{
						Id: "4b1e760d-182c-4980-b873-c1a6827c9817",
						Matchers: []*pb.Matcher{
							{Name: "label1", Pattern: "val1", Type: pb.Matcher_EQUAL},
						},
						StartsAt:  now.Add(time.Hour),
						EndsAt:    now.Add(2 * time.Hour),
						UpdatedAt: now,
					},
					ExpiresAt: now.Add(24 * time.Hour),
				},
			},
		},
	}

	for _, c := range cases {
		f, err := os.CreateTemp("", "snapshot")
		require.NoError(t, err, "creating temp file failed")

		s1 := &Silences{st: state{}, metrics: newMetrics(nil, nil)}
		// Setup internal state manually.
		for _, e := range c.entries {
			s1.st[e.Silence.Id] = e
		}
		_, err = s1.Snapshot(f)
		require.NoError(t, err, "creating snapshot failed")

		require.NoError(t, f.Close(), "closing snapshot file failed")

		f, err = os.Open(f.Name())
		require.NoError(t, err, "opening snapshot file failed")

		// Check again against new nlog instance.
		s2 := &Silences{mc: matcherCache{}, st: state{}}
		err = s2.loadSnapshot(f)
		require.NoError(t, err, "error loading snapshot")
		require.Equal(t, s1.st, s2.st, "state after loading snapshot did not match snapshotted state")

		require.NoError(t, f.Close(), "closing snapshot file failed")
	}
}

// This tests a regression introduced by https://github.com/prometheus/alertmanager/pull/2689.
func TestSilences_Maintenance_DefaultMaintenanceFuncDoesntCrash(t *testing.T) {
	f, err := os.CreateTemp("", "snapshot")
	require.NoError(t, err, "creating temp file failed")
	clock := clock.NewMock()
	s := &Silences{st: state{}, logger: log.NewNopLogger(), clock: clock, metrics: newMetrics(nil, nil)}
	stopc := make(chan struct{})

	done := make(chan struct{})
	go func() {
		s.Maintenance(100*time.Millisecond, f.Name(), stopc, nil)
		close(done)
	}()
	runtime.Gosched()

	clock.Add(100 * time.Millisecond)
	close(stopc)

	<-done
}

func TestSilences_Maintenance_SupportsCustomCallback(t *testing.T) {
	f, err := os.CreateTemp("", "snapshot")
	require.NoError(t, err, "creating temp file failed")
	clock := clock.NewMock()
	reg := prometheus.NewRegistry()
	s := &Silences{st: state{}, logger: log.NewNopLogger(), clock: clock}
	s.metrics = newMetrics(reg, s)
	stopc := make(chan struct{})

	var calls atomic.Int32
	var wg sync.WaitGroup

	wg.Add(1)
	go func() {
		defer wg.Done()
		s.Maintenance(10*time.Second, f.Name(), stopc, func() (int64, error) {
			calls.Add(1)
			return 0, nil
		})
	}()
	gosched()

	// Before the first tick, no maintenance executed.
	clock.Add(9 * time.Second)
	require.EqualValues(t, 0, calls.Load())

	// Tick once.
	clock.Add(1 * time.Second)
	require.EqualValues(t, 1, calls.Load())

	// Stop the maintenance loop. We should get exactly one more execution of the maintenance func.
	close(stopc)
	wg.Wait()

	require.EqualValues(t, 2, calls.Load())

	// Check the maintenance metrics.
	require.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(`
# HELP alertmanager_silences_maintenance_errors_total How many maintenances were executed for silences that failed.
# TYPE alertmanager_silences_maintenance_errors_total counter
alertmanager_silences_maintenance_errors_total 0
# HELP alertmanager_silences_maintenance_total How many maintenances were executed for silences.
# TYPE alertmanager_silences_maintenance_total counter
alertmanager_silences_maintenance_total 2
`), "alertmanager_silences_maintenance_total", "alertmanager_silences_maintenance_errors_total"))
}

func TestSilencesSetSilence(t *testing.T) {
	s, err := New(Options{
		Retention: time.Minute,
	})
	require.NoError(t, err)

	clock := clock.NewMock()
	s.clock = clock

	nowpb := s.nowUTC()

	sil := &pb.Silence{
		Id:       "some_id",
		Matchers: []*pb.Matcher{{Name: "abc", Pattern: "def"}},
		StartsAt: nowpb,
		EndsAt:   nowpb,
	}

	want := state{
		"some_id": &pb.MeshSilence{
			Silence:   sil,
			ExpiresAt: nowpb.Add(time.Minute),
		},
	}

	done := make(chan struct{})
	s.broadcast = func(b []byte) {
		var e pb.MeshSilence
		r := bytes.NewReader(b)
		_, err := pbutil.ReadDelimited(r, &e)
		require.NoError(t, err)

		require.Equal(t, want["some_id"], &e)
		close(done)
	}

	// setSilence() is always called with s.mtx locked() in the application code
	func() {
		s.mtx.Lock()
		defer s.mtx.Unlock()
		require.NoError(t, s.setSilence(sil, nowpb))
	}()

	// Ensure broadcast was called.
	if _, isOpen := <-done; isOpen {
		t.Fatal("broadcast was not called")
	}

	require.Equal(t, want, s.st, "Unexpected silence state")
}

func TestSilenceSet(t *testing.T) {
	s, err := New(Options{
		Retention: time.Hour,
	})
	require.NoError(t, err)

	clock := clock.NewMock()
	s.clock = clock
	start1 := s.nowUTC()

	// Insert silence with fixed start time.
	sil1 := &pb.Silence{
		Matchers: []*pb.Matcher{{Name: "a", Pattern: "b"}},
		StartsAt: start1.Add(2 * time.Minute),
		EndsAt:   start1.Add(5 * time.Minute),
	}
	id1, err := s.Set(sil1)
	require.NoError(t, err)
	require.NotEqual(t, id1, "")

	want := state{
		id1: &pb.MeshSilence{
			Silence: &pb.Silence{
				Id:        id1,
				Matchers:  []*pb.Matcher{{Name: "a", Pattern: "b"}},
				StartsAt:  start1.Add(2 * time.Minute),
				EndsAt:    start1.Add(5 * time.Minute),
				UpdatedAt: start1,
			},
			ExpiresAt: start1.Add(5*time.Minute + s.retention),
		},
	}
	require.Equal(t, want, s.st, "unexpected state after silence creation")

	// Insert silence with unset start time. Must be set to now.
	clock.Add(time.Minute)
	start2 := s.nowUTC()

	sil2 := &pb.Silence{
		Matchers: []*pb.Matcher{{Name: "a", Pattern: "b"}},
		EndsAt:   start2.Add(1 * time.Minute),
	}
	id2, err := s.Set(sil2)
	require.NoError(t, err)
	require.NotEqual(t, id2, "")

	want = state{
		id1: want[id1],
		id2: &pb.MeshSilence{
			Silence: &pb.Silence{
				Id:        id2,
				Matchers:  []*pb.Matcher{{Name: "a", Pattern: "b"}},
				StartsAt:  start2,
				EndsAt:    start2.Add(1 * time.Minute),
				UpdatedAt: start2,
			},
			ExpiresAt: start2.Add(1*time.Minute + s.retention),
		},
	}
	require.Equal(t, want, s.st, "unexpected state after silence creation")

	// Overwrite silence 2 with new end time.
	clock.Add(time.Minute)
	start3 := s.nowUTC()

	sil3 := cloneSilence(sil2)
	sil3.EndsAt = start3.Add(100 * time.Minute)

	id3, err := s.Set(sil3)
	require.NoError(t, err)
	require.Equal(t, id2, id3)

	want = state{
		id1: want[id1],
		id2: &pb.MeshSilence{
			Silence: &pb.Silence{
				Id:        id2,
				Matchers:  []*pb.Matcher{{Name: "a", Pattern: "b"}},
				StartsAt:  start2,
				EndsAt:    start3.Add(100 * time.Minute),
				UpdatedAt: start3,
			},
			ExpiresAt: start3.Add(100*time.Minute + s.retention),
		},
	}
	require.Equal(t, want, s.st, "unexpected state after silence creation")

	// Update this silence again with new matcher. This expires it and creates a new one.
	clock.Add(time.Minute)
	start4 := s.nowUTC()

	sil4 := cloneSilence(sil3)
	sil4.Matchers = []*pb.Matcher{{Name: "a", Pattern: "c"}}

	id4, err := s.Set(sil4)
	require.NoError(t, err)
	// This new silence gets a new id.
	require.NotEqual(t, id2, id4)

	want = state{
		id1: want[id1],
		id2: &pb.MeshSilence{
			Silence: &pb.Silence{
				Id:        id2,
				Matchers:  []*pb.Matcher{{Name: "a", Pattern: "b"}},
				StartsAt:  start2,
				EndsAt:    start4, // Expired
				UpdatedAt: start4,
			},
			ExpiresAt: start4.Add(s.retention),
		},
		id4: &pb.MeshSilence{
			Silence: &pb.Silence{
				Id:        id4,
				Matchers:  []*pb.Matcher{{Name: "a", Pattern: "c"}},
				StartsAt:  start4,
				EndsAt:    start3.Add(100 * time.Minute),
				UpdatedAt: start4,
			},
			ExpiresAt: start3.Add(100*time.Minute + s.retention),
		},
	}
	require.Equal(t, want, s.st, "unexpected state after silence creation")

	// Re-create the silence that just expired.
	clock.Add(time.Minute)
	start5 := s.nowUTC()

	sil5 := cloneSilence(sil3)
	sil5.StartsAt = start1
	sil5.EndsAt = start1.Add(5 * time.Minute)

	id5, err := s.Set(sil5)
	require.NoError(t, err)
	require.NotEqual(t, id2, id4)

	want = state{
		id1: want[id1],
		id2: want[id2],
		id4: want[id4],
		id5: &pb.MeshSilence{
			Silence: &pb.Silence{
				Id:        id5,
				Matchers:  []*pb.Matcher{{Name: "a", Pattern: "b"}},
				StartsAt:  start5, // New silences have their start time set to "now" when created.
				EndsAt:    start1.Add(5 * time.Minute),
				UpdatedAt: start5,
			},
			ExpiresAt: start1.Add(5*time.Minute + s.retention),
		},
	}
	require.Equal(t, want, s.st, "unexpected state after silence creation")
}

func TestSetActiveSilence(t *testing.T) {
	s, err := New(Options{
		Retention: time.Hour,
	})
	require.NoError(t, err)

	clock := clock.NewMock()
	s.clock = clock
	now := clock.Now()

	startsAt := now.Add(-1 * time.Minute)
	endsAt := now.Add(5 * time.Minute)
	// Insert silence with fixed start time.
	sil1 := &pb.Silence{
		Matchers: []*pb.Matcher{{Name: "a", Pattern: "b"}},
		StartsAt: startsAt,
		EndsAt:   endsAt,
	}
	id1, _ := s.Set(sil1)

	// Update silence with 2 extra nanoseconds so the "seconds" part should not change

	newStartsAt := now.Add(2 * time.Nanosecond)
	newEndsAt := endsAt.Add(2 * time.Minute)

	sil2 := cloneSilence(sil1)
	sil2.Id = id1
	sil2.StartsAt = newStartsAt
	sil2.EndsAt = newEndsAt

	clock.Add(time.Minute)
	now = s.nowUTC()
	id2, err := s.Set(sil2)
	require.NoError(t, err)
	require.Equal(t, id1, id2)

	want := state{
		id2: &pb.MeshSilence{
			Silence: &pb.Silence{
				Id:        id1,
				Matchers:  []*pb.Matcher{{Name: "a", Pattern: "b"}},
				StartsAt:  newStartsAt,
				EndsAt:    newEndsAt,
				UpdatedAt: now,
			},
			ExpiresAt: newEndsAt.Add(s.retention),
		},
	}
	require.Equal(t, want, s.st, "unexpected state after silence creation")
}

func TestSilencesSetFail(t *testing.T) {
	s, err := New(Options{})
	require.NoError(t, err)

	clock := clock.NewMock()
	s.clock = clock

	cases := []struct {
		s   *pb.Silence
		err string
	}{
		{
			s:   &pb.Silence{Id: "some_id"},
			err: ErrNotFound.Error(),
		}, {
			s:   &pb.Silence{}, // Silence without matcher.
			err: "silence invalid",
		},
	}
	for _, c := range cases {
		_, err := s.Set(c.s)
		checkErr(t, c.err, err)
	}
}

func TestQState(t *testing.T) {
	now := time.Now().UTC()

	cases := []struct {
		sil    *pb.Silence
		states []types.SilenceState
		keep   bool
	}{
		{
			sil: &pb.Silence{
				StartsAt: now.Add(time.Minute),
				EndsAt:   now.Add(time.Hour),
			},
			states: []types.SilenceState{types.SilenceStateActive, types.SilenceStateExpired},
			keep:   false,
		},
		{
			sil: &pb.Silence{
				StartsAt: now.Add(time.Minute),
				EndsAt:   now.Add(time.Hour),
			},
			states: []types.SilenceState{types.SilenceStatePending},
			keep:   true,
		},
		{
			sil: &pb.Silence{
				StartsAt: now.Add(time.Minute),
				EndsAt:   now.Add(time.Hour),
			},
			states: []types.SilenceState{types.SilenceStateExpired, types.SilenceStatePending},
			keep:   true,
		},
	}
	for i, c := range cases {
		q := &query{}
		QState(c.states...)(q)
		f := q.filters[0]

		keep, err := f(c.sil, nil, now)
		require.NoError(t, err)
		require.Equal(t, c.keep, keep, "unexpected filter result for case %d", i)
	}
}

func TestQMatches(t *testing.T) {
	qp := QMatches(model.LabelSet{
		"job":      "test",
		"instance": "web-1",
		"path":     "/user/profile",
		"method":   "GET",
	})

	q := &query{}
	qp(q)
	f := q.filters[0]

	cases := []struct {
		sil  *pb.Silence
		drop bool
	}{
		{
			sil: &pb.Silence{
				Matchers: []*pb.Matcher{
					{Name: "job", Pattern: "test", Type: pb.Matcher_EQUAL},
				},
			},
			drop: true,
		},
		{
			sil: &pb.Silence{
				Matchers: []*pb.Matcher{
					{Name: "job", Pattern: "test", Type: pb.Matcher_NOT_EQUAL},
				},
			},
			drop: false,
		},
		{
			sil: &pb.Silence{
				Matchers: []*pb.Matcher{
					{Name: "job", Pattern: "test", Type: pb.Matcher_EQUAL},
					{Name: "method", Pattern: "POST", Type: pb.Matcher_EQUAL},
				},
			},
			drop: false,
		},
		{
			sil: &pb.Silence{
				Matchers: []*pb.Matcher{
					{Name: "job", Pattern: "test", Type: pb.Matcher_EQUAL},
					{Name: "method", Pattern: "POST", Type: pb.Matcher_NOT_EQUAL},
				},
			},
			drop: true,
		},
		{
			sil: &pb.Silence{
				Matchers: []*pb.Matcher{
					{Name: "path", Pattern: "/user/.+", Type: pb.Matcher_REGEXP},
				},
			},
			drop: true,
		},
		{
			sil: &pb.Silence{
				Matchers: []*pb.Matcher{
					{Name: "path", Pattern: "/user/.+", Type: pb.Matcher_NOT_REGEXP},
				},
			},
			drop: false,
		},
		{
			sil: &pb.Silence{
				Matchers: []*pb.Matcher{
					{Name: "path", Pattern: "/user/.+", Type: pb.Matcher_REGEXP},
					{Name: "path", Pattern: "/nothing/.+", Type: pb.Matcher_REGEXP},
				},
			},
			drop: false,
		},
	}
	for _, c := range cases {
		drop, err := f(c.sil, &Silences{mc: matcherCache{}, st: state{}}, time.Time{})
		require.NoError(t, err)
		require.Equal(t, c.drop, drop, "unexpected filter result")
	}
}

func TestSilencesQuery(t *testing.T) {
	s, err := New(Options{})
	require.NoError(t, err)

	s.st = state{
		"1": &pb.MeshSilence{Silence: &pb.Silence{Id: "1"}},
		"2": &pb.MeshSilence{Silence: &pb.Silence{Id: "2"}},
		"3": &pb.MeshSilence{Silence: &pb.Silence{Id: "3"}},
		"4": &pb.MeshSilence{Silence: &pb.Silence{Id: "4"}},
		"5": &pb.MeshSilence{Silence: &pb.Silence{Id: "5"}},
	}
	cases := []struct {
		q   *query
		exp []*pb.Silence
	}{
		{
			// Default query of retrieving all silences.
			q: &query{},
			exp: []*pb.Silence{
				{Id: "1"},
				{Id: "2"},
				{Id: "3"},
				{Id: "4"},
				{Id: "5"},
			},
		},
		{
			// Retrieve by IDs.
			q: &query{
				ids: []string{"2", "5"},
			},
			exp: []*pb.Silence{
				{Id: "2"},
				{Id: "5"},
			},
		},
		{
			// Retrieve all and filter
			q: &query{
				filters: []silenceFilter{
					func(sil *pb.Silence, _ *Silences, _ time.Time) (bool, error) {
						return sil.Id == "1" || sil.Id == "2", nil
					},
				},
			},
			exp: []*pb.Silence{
				{Id: "1"},
				{Id: "2"},
			},
		},
		{
			// Retrieve by IDs and filter
			q: &query{
				ids: []string{"2", "5"},
				filters: []silenceFilter{
					func(sil *pb.Silence, _ *Silences, _ time.Time) (bool, error) {
						return sil.Id == "1" || sil.Id == "2", nil
					},
				},
			},
			exp: []*pb.Silence{
				{Id: "2"},
			},
		},
	}

	for _, c := range cases {
		// Run default query of retrieving all silences.
		res, _, err := s.query(c.q, time.Time{})
		require.NoError(t, err, "unexpected error on querying")

		// Currently there are no sorting guarantees in the querying API.
		sort.Sort(silencesByID(c.exp))
		sort.Sort(silencesByID(res))
		require.Equal(t, c.exp, res, "unexpected silences in result")
	}
}

type silencesByID []*pb.Silence

func (s silencesByID) Len() int           { return len(s) }
func (s silencesByID) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
func (s silencesByID) Less(i, j int) bool { return s[i].Id < s[j].Id }

func TestSilenceCanUpdate(t *testing.T) {
	now := time.Now().UTC()

	cases := []struct {
		a, b *pb.Silence
		ok   bool
	}{
		// Bad arguments.
		{
			a: &pb.Silence{},
			b: &pb.Silence{
				StartsAt: now,
				EndsAt:   now.Add(-time.Minute),
			},
			ok: false,
		},
		// Expired silence.
		{
			a: &pb.Silence{
				StartsAt: now.Add(-time.Hour),
				EndsAt:   now.Add(-time.Second),
			},
			b: &pb.Silence{
				StartsAt: now,
				EndsAt:   now,
			},
			ok: false,
		},
		// Pending silences.
		{
			a: &pb.Silence{
				StartsAt:  now.Add(time.Hour),
				EndsAt:    now.Add(2 * time.Hour),
				UpdatedAt: now.Add(-time.Hour),
			},
			b: &pb.Silence{
				StartsAt: now.Add(-time.Minute),
				EndsAt:   now.Add(time.Hour),
			},
			ok: false,
		},
		{
			a: &pb.Silence{
				StartsAt:  now.Add(time.Hour),
				EndsAt:    now.Add(2 * time.Hour),
				UpdatedAt: now.Add(-time.Hour),
			},
			b: &pb.Silence{
				StartsAt: now.Add(time.Minute),
				EndsAt:   now.Add(time.Minute),
			},
			ok: true,
		},
		{
			a: &pb.Silence{
				StartsAt:  now.Add(time.Hour),
				EndsAt:    now.Add(2 * time.Hour),
				UpdatedAt: now.Add(-time.Hour),
			},
			b: &pb.Silence{
				StartsAt: now, // set to exactly start now.
				EndsAt:   now.Add(2 * time.Hour),
			},
			ok: true,
		},
		// Active silences.
		{
			a: &pb.Silence{
				StartsAt:  now.Add(-time.Hour),
				EndsAt:    now.Add(2 * time.Hour),
				UpdatedAt: now.Add(-time.Hour),
			},
			b: &pb.Silence{
				StartsAt: now.Add(-time.Minute),
				EndsAt:   now.Add(2 * time.Hour),
			},
			ok: false,
		},
		{
			a: &pb.Silence{
				StartsAt:  now.Add(-time.Hour),
				EndsAt:    now.Add(2 * time.Hour),
				UpdatedAt: now.Add(-time.Hour),
			},
			b: &pb.Silence{
				StartsAt: now.Add(-time.Hour),
				EndsAt:   now.Add(-time.Second),
			},
			ok: false,
		},
		{
			a: &pb.Silence{
				StartsAt:  now.Add(-time.Hour),
				EndsAt:    now.Add(2 * time.Hour),
				UpdatedAt: now.Add(-time.Hour),
			},
			b: &pb.Silence{
				StartsAt: now.Add(-time.Hour),
				EndsAt:   now,
			},
			ok: true,
		},
		{
			a: &pb.Silence{
				StartsAt:  now.Add(-time.Hour),
				EndsAt:    now.Add(2 * time.Hour),
				UpdatedAt: now.Add(-time.Hour),
			},
			b: &pb.Silence{
				StartsAt: now.Add(-time.Hour),
				EndsAt:   now.Add(3 * time.Hour),
			},
			ok: true,
		},
	}
	for _, c := range cases {
		ok := canUpdate(c.a, c.b, now)
		if ok && !c.ok {
			t.Errorf("expected not-updateable but was: %v, %v", c.a, c.b)
		}
		if ok && !c.ok {
			t.Errorf("expected updateable but was not: %v, %v", c.a, c.b)
		}
	}
}

func TestSilenceExpire(t *testing.T) {
	s, err := New(Options{Retention: time.Hour})
	require.NoError(t, err)

	clock := clock.NewMock()
	s.clock = clock
	now := s.nowUTC()

	m := &pb.Matcher{Type: pb.Matcher_EQUAL, Name: "a", Pattern: "b"}

	s.st = state{
		"pending": &pb.MeshSilence{Silence: &pb.Silence{
			Id:        "pending",
			Matchers:  []*pb.Matcher{m},
			StartsAt:  now.Add(time.Minute),
			EndsAt:    now.Add(time.Hour),
			UpdatedAt: now.Add(-time.Hour),
		}},
		"active": &pb.MeshSilence{Silence: &pb.Silence{
			Id:        "active",
			Matchers:  []*pb.Matcher{m},
			StartsAt:  now.Add(-time.Minute),
			EndsAt:    now.Add(time.Hour),
			UpdatedAt: now.Add(-time.Hour),
		}},
		"expired": &pb.MeshSilence{Silence: &pb.Silence{
			Id:        "expired",
			Matchers:  []*pb.Matcher{m},
			StartsAt:  now.Add(-time.Hour),
			EndsAt:    now.Add(-time.Minute),
			UpdatedAt: now.Add(-time.Hour),
		}},
	}

	count, err := s.CountState(types.SilenceStatePending)
	require.NoError(t, err)
	require.Equal(t, 1, count)

	count, err = s.CountState(types.SilenceStateExpired)
	require.NoError(t, err)
	require.Equal(t, 1, count)

	require.NoError(t, s.Expire("pending"))
	require.NoError(t, s.Expire("active"))

	require.NoError(t, s.Expire("expired"))

	sil, err := s.QueryOne(QIDs("pending"))
	require.NoError(t, err)
	require.Equal(t, &pb.Silence{
		Id:        "pending",
		Matchers:  []*pb.Matcher{m},
		StartsAt:  now,
		EndsAt:    now,
		UpdatedAt: now,
	}, sil)

	// Let time pass...
	clock.Add(time.Second)

	count, err = s.CountState(types.SilenceStatePending)
	require.NoError(t, err)
	require.Equal(t, 0, count)

	count, err = s.CountState(types.SilenceStateExpired)
	require.NoError(t, err)
	require.Equal(t, 3, count)

	// Expiring a pending Silence should make the API return the
	// SilenceStateExpired Silence state.
	silenceState := types.CalcSilenceState(sil.StartsAt, sil.EndsAt)
	require.Equal(t, silenceState, types.SilenceStateExpired)

	sil, err = s.QueryOne(QIDs("active"))
	require.NoError(t, err)
	require.Equal(t, &pb.Silence{
		Id:        "active",
		Matchers:  []*pb.Matcher{m},
		StartsAt:  now.Add(-time.Minute),
		EndsAt:    now,
		UpdatedAt: now,
	}, sil)

	sil, err = s.QueryOne(QIDs("expired"))
	require.NoError(t, err)
	require.Equal(t, &pb.Silence{
		Id:        "expired",
		Matchers:  []*pb.Matcher{m},
		StartsAt:  now.Add(-time.Hour),
		EndsAt:    now.Add(-time.Minute),
		UpdatedAt: now.Add(-time.Hour),
	}, sil)
}

// TestSilenceExpireWithZeroRetention covers the problem that, with zero
// retention time, a silence explicitly set to expired will also immediately
// expire from the silence storage.
func TestSilenceExpireWithZeroRetention(t *testing.T) {
	s, err := New(Options{Retention: 0})
	require.NoError(t, err)

	clock := clock.NewMock()
	s.clock = clock
	now := s.nowUTC()

	m := &pb.Matcher{Type: pb.Matcher_EQUAL, Name: "a", Pattern: "b"}

	s.st = state{
		"pending": &pb.MeshSilence{Silence: &pb.Silence{
			Id:        "pending",
			Matchers:  []*pb.Matcher{m},
			StartsAt:  now.Add(time.Minute),
			EndsAt:    now.Add(time.Hour),
			UpdatedAt: now.Add(-time.Hour),
		}},
		"active": &pb.MeshSilence{Silence: &pb.Silence{
			Id:        "active",
			Matchers:  []*pb.Matcher{m},
			StartsAt:  now.Add(-time.Minute),
			EndsAt:    now.Add(time.Hour),
			UpdatedAt: now.Add(-time.Hour),
		}},
		"expired": &pb.MeshSilence{Silence: &pb.Silence{
			Id:        "expired",
			Matchers:  []*pb.Matcher{m},
			StartsAt:  now.Add(-time.Hour),
			EndsAt:    now.Add(-time.Minute),
			UpdatedAt: now.Add(-time.Hour),
		}},
	}

	count, err := s.CountState(types.SilenceStatePending)
	require.NoError(t, err)
	require.Equal(t, 1, count)

	count, err = s.CountState(types.SilenceStateActive)
	require.NoError(t, err)
	require.Equal(t, 1, count)

	count, err = s.CountState(types.SilenceStateExpired)
	require.NoError(t, err)
	require.Equal(t, 1, count)

	// Advance time. The silence state management code uses update time when
	// merging, and the logic is "first write wins". So we must advance the clock
	// one tick for updates to take effect.
	clock.Add(1 * time.Millisecond)

	require.NoError(t, s.Expire("pending"))
	require.NoError(t, s.Expire("active"))
	require.NoError(t, s.Expire("expired"))

	// Advance time again. Despite what the function name says, s.Expire() does
	// not expire a silence. It sets the silence to EndAt the current time. This
	// means that the silence is active immediately after calling Expire.
	clock.Add(1 * time.Millisecond)

	// Verify all silences have expired.
	count, err = s.CountState(types.SilenceStatePending)
	require.NoError(t, err)
	require.Equal(t, 0, count)

	count, err = s.CountState(types.SilenceStateActive)
	require.NoError(t, err)
	require.Equal(t, 0, count)

	count, err = s.CountState(types.SilenceStateExpired)
	require.NoError(t, err)
	require.Equal(t, 3, count)
}

func TestSilencer(t *testing.T) {
	ss, err := New(Options{Retention: time.Hour})
	require.NoError(t, err)

	clock := clock.NewMock()
	ss.clock = clock
	now := ss.nowUTC()

	m := types.NewMarker(prometheus.NewRegistry())
	s := NewSilencer(ss, m, log.NewNopLogger())

	require.False(t, s.Mutes(model.LabelSet{"foo": "bar"}), "expected alert not silenced without any silences")

	_, err = ss.Set(&pb.Silence{
		Matchers: []*pb.Matcher{{Name: "foo", Pattern: "baz"}},
		StartsAt: now.Add(-time.Hour),
		EndsAt:   now.Add(5 * time.Minute),
	})
	require.NoError(t, err)

	require.False(t, s.Mutes(model.LabelSet{"foo": "bar"}), "expected alert not silenced by non-matching silence")

	id, err := ss.Set(&pb.Silence{
		Matchers: []*pb.Matcher{{Name: "foo", Pattern: "bar"}},
		StartsAt: now.Add(-time.Hour),
		EndsAt:   now.Add(5 * time.Minute),
	})
	require.NoError(t, err)
	require.NotEmpty(t, id)

	require.True(t, s.Mutes(model.LabelSet{"foo": "bar"}), "expected alert silenced by matching silence")

	// One hour passes, silence expires.
	clock.Add(time.Hour)
	now = ss.nowUTC()

	require.False(t, s.Mutes(model.LabelSet{"foo": "bar"}), "expected alert not silenced by expired silence")

	// Update silence to start in the future.
	_, err = ss.Set(&pb.Silence{
		Id:       id,
		Matchers: []*pb.Matcher{{Name: "foo", Pattern: "bar"}},
		StartsAt: now.Add(time.Hour),
		EndsAt:   now.Add(3 * time.Hour),
	})
	require.NoError(t, err)

	require.False(t, s.Mutes(model.LabelSet{"foo": "bar"}), "expected alert not silenced by future silence")

	// Two hours pass, silence becomes active.
	clock.Add(2 * time.Hour)
	now = ss.nowUTC()

	// Exposes issue #2426.
	require.True(t, s.Mutes(model.LabelSet{"foo": "bar"}), "expected alert silenced by activated silence")

	_, err = ss.Set(&pb.Silence{
		Matchers: []*pb.Matcher{{Name: "foo", Pattern: "b..", Type: pb.Matcher_REGEXP}},
		StartsAt: now.Add(time.Hour),
		EndsAt:   now.Add(3 * time.Hour),
	})
	require.NoError(t, err)

	// Note that issue #2426 doesn't apply anymore because we added a new silence.
	require.True(t, s.Mutes(model.LabelSet{"foo": "bar"}), "expected alert still silenced by activated silence")

	// Two hours pass, first silence expires, overlapping second silence becomes active.
	clock.Add(2 * time.Hour)

	// Another variant of issue #2426 (overlapping silences).
	require.True(t, s.Mutes(model.LabelSet{"foo": "bar"}), "expected alert silenced by activated second silence")
}

func TestValidateMatcher(t *testing.T) {
	cases := []struct {
		m   *pb.Matcher
		err string
	}{
		{
			m: &pb.Matcher{
				Name:    "a",
				Pattern: "b",
				Type:    pb.Matcher_EQUAL,
			},
			err: "",
		}, {
			m: &pb.Matcher{
				Name:    "a",
				Pattern: "b",
				Type:    pb.Matcher_NOT_EQUAL,
			},
			err: "",
		}, {
			m: &pb.Matcher{
				Name:    "a",
				Pattern: "b",
				Type:    pb.Matcher_REGEXP,
			},
			err: "",
		}, {
			m: &pb.Matcher{
				Name:    "a",
				Pattern: "b",
				Type:    pb.Matcher_NOT_REGEXP,
			},
			err: "",
		}, {
			m: &pb.Matcher{
				Name:    "00",
				Pattern: "a",
				Type:    pb.Matcher_EQUAL,
			},
			err: "invalid label name",
		}, {
			m: &pb.Matcher{
				Name:    "a",
				Pattern: "((",
				Type:    pb.Matcher_REGEXP,
			},
			err: "invalid regular expression",
		}, {
			m: &pb.Matcher{
				Name:    "a",
				Pattern: "))",
				Type:    pb.Matcher_NOT_REGEXP,
			},
			err: "invalid regular expression",
		}, {
			m: &pb.Matcher{
				Name:    "a",
				Pattern: "\xff",
				Type:    pb.Matcher_EQUAL,
			},
			err: "invalid label value",
		}, {
			m: &pb.Matcher{
				Name:    "a",
				Pattern: "b",
				Type:    333,
			},
			err: "unknown matcher type",
		},
	}

	for _, c := range cases {
		checkErr(t, c.err, ValidateMatcher(c.m))
	}
}

func TestValidateSilence(t *testing.T) {
	var (
		now            = time.Now().UTC()
		zeroTimestamp  = time.Time{}
		validTimestamp = now
	)
	cases := []struct {
		s   *pb.Silence
		err string
	}{
		{
			s: &pb.Silence{
				Id: "some_id",
				Matchers: []*pb.Matcher{
					{Name: "a", Pattern: "b"},
				},
				StartsAt:  validTimestamp,
				EndsAt:    validTimestamp,
				UpdatedAt: validTimestamp,
			},
			err: "",
		},
		{
			s: &pb.Silence{
				Id: "",
				Matchers: []*pb.Matcher{
					{Name: "a", Pattern: "b"},
				},
				StartsAt:  validTimestamp,
				EndsAt:    validTimestamp,
				UpdatedAt: validTimestamp,
			},
			err: "ID missing",
		},
		{
			s: &pb.Silence{
				Id:        "some_id",
				Matchers:  []*pb.Matcher{},
				StartsAt:  validTimestamp,
				EndsAt:    validTimestamp,
				UpdatedAt: validTimestamp,
			},
			err: "at least one matcher required",
		},
		{
			s: &pb.Silence{
				Id: "some_id",
				Matchers: []*pb.Matcher{
					{Name: "a", Pattern: "b"},
					{Name: "00", Pattern: "b"},
				},
				StartsAt:  validTimestamp,
				EndsAt:    validTimestamp,
				UpdatedAt: validTimestamp,
			},
			err: "invalid label matcher",
		},
		{
			s: &pb.Silence{
				Id: "some_id",
				Matchers: []*pb.Matcher{
					{Name: "a", Pattern: ""},
					{Name: "b", Pattern: ".*", Type: pb.Matcher_REGEXP},
				},
				StartsAt:  validTimestamp,
				EndsAt:    validTimestamp,
				UpdatedAt: validTimestamp,
			},
			err: "at least one matcher must not match the empty string",
		},
		{
			s: &pb.Silence{
				Id: "some_id",
				Matchers: []*pb.Matcher{
					{Name: "a", Pattern: "b"},
				},
				StartsAt:  now,
				EndsAt:    now.Add(-time.Second),
				UpdatedAt: validTimestamp,
			},
			err: "end time must not be before start time",
		},
		{
			s: &pb.Silence{
				Id: "some_id",
				Matchers: []*pb.Matcher{
					{Name: "a", Pattern: "b"},
				},
				StartsAt:  zeroTimestamp,
				EndsAt:    validTimestamp,
				UpdatedAt: validTimestamp,
			},
			err: "invalid zero start timestamp",
		},
		{
			s: &pb.Silence{
				Id: "some_id",
				Matchers: []*pb.Matcher{
					{Name: "a", Pattern: "b"},
				},
				StartsAt:  validTimestamp,
				EndsAt:    zeroTimestamp,
				UpdatedAt: validTimestamp,
			},
			err: "invalid zero end timestamp",
		},
		{
			s: &pb.Silence{
				Id: "some_id",
				Matchers: []*pb.Matcher{
					{Name: "a", Pattern: "b"},
				},
				StartsAt:  validTimestamp,
				EndsAt:    validTimestamp,
				UpdatedAt: zeroTimestamp,
			},
			err: "invalid zero update timestamp",
		},
	}
	for _, c := range cases {
		checkErr(t, c.err, validateSilence(c.s))
	}
}

func TestStateMerge(t *testing.T) {
	now := time.Now().UTC()

	// We only care about key names and timestamps for the
	// merging logic.
	newSilence := func(id string, ts, exp time.Time) *pb.MeshSilence {
		return &pb.MeshSilence{
			Silence:   &pb.Silence{Id: id, UpdatedAt: ts},
			ExpiresAt: exp,
		}
	}

	exp := now.Add(time.Minute)

	cases := []struct {
		a, b  state
		final state
	}{
		{
			a: state{
				"a1": newSilence("a1", now, exp),
				"a2": newSilence("a2", now, exp),
				"a3": newSilence("a3", now, exp),
			},
			b: state{
				"b1": newSilence("b1", now, exp),                                          // new key, should be added
				"a2": newSilence("a2", now.Add(-time.Minute), exp),                        // older timestamp, should be dropped
				"a3": newSilence("a3", now.Add(time.Minute), exp),                         // newer timestamp, should overwrite
				"a4": newSilence("a4", now.Add(-time.Minute), now.Add(-time.Millisecond)), // new key, expired, should not be added
			},
			final: state{
				"a1": newSilence("a1", now, exp),
				"a2": newSilence("a2", now, exp),
				"a3": newSilence("a3", now.Add(time.Minute), exp),
				"b1": newSilence("b1", now, exp),
			},
		},
	}

	for _, c := range cases {
		for _, e := range c.b {
			c.a.merge(e, now)
		}

		require.Equal(t, c.final, c.a, "Merge result should match expectation")
	}
}

func TestStateCoding(t *testing.T) {
	// Check whether encoding and decoding the data is symmetric.
	now := time.Now().UTC()

	cases := []struct {
		entries []*pb.MeshSilence
	}{
		{
			entries: []*pb.MeshSilence{
				{
					Silence: &pb.Silence{
						Id: "3be80475-e219-4ee7-b6fc-4b65114e362f",
						Matchers: []*pb.Matcher{
							{Name: "label1", Pattern: "val1", Type: pb.Matcher_EQUAL},
							{Name: "label2", Pattern: "val.+", Type: pb.Matcher_REGEXP},
						},
						StartsAt:  now,
						EndsAt:    now,
						UpdatedAt: now,
					},
					ExpiresAt: now,
				},
				{
					Silence: &pb.Silence{
						Id: "4b1e760d-182c-4980-b873-c1a6827c9817",
						Matchers: []*pb.Matcher{
							{Name: "label1", Pattern: "val1", Type: pb.Matcher_EQUAL},
						},
						StartsAt:  now.Add(time.Hour),
						EndsAt:    now.Add(2 * time.Hour),
						UpdatedAt: now,
					},
					ExpiresAt: now.Add(24 * time.Hour),
				},
				{
					Silence: &pb.Silence{
						Id: "3dfb2528-59ce-41eb-b465-f875a4e744a4",
						Matchers: []*pb.Matcher{
							{Name: "label1", Pattern: "val1", Type: pb.Matcher_NOT_EQUAL},
							{Name: "label2", Pattern: "val.+", Type: pb.Matcher_NOT_REGEXP},
						},
						StartsAt:  now,
						EndsAt:    now,
						UpdatedAt: now,
					},
					ExpiresAt: now,
				},
			},
		},
	}

	for _, c := range cases {
		// Create gossip data from input.
		in := state{}
		for _, e := range c.entries {
			in[e.Silence.Id] = e
		}
		msg, err := in.MarshalBinary()
		require.NoError(t, err)

		out, err := decodeState(bytes.NewReader(msg))
		require.NoError(t, err, "decoding message failed")

		require.Equal(t, in, out, "decoded data doesn't match encoded data")
	}
}

func TestStateDecodingError(t *testing.T) {
	// Check whether decoding copes with erroneous data.
	s := state{"": &pb.MeshSilence{}}

	msg, err := s.MarshalBinary()
	require.NoError(t, err)

	_, err = decodeState(bytes.NewReader(msg))
	require.Equal(t, ErrInvalidState, err)
}

func benchmarkSilencesQuery(b *testing.B, numSilences int) {
	s, err := New(Options{})
	require.NoError(b, err)

	clock := clock.NewMock()
	s.clock = clock
	now := clock.Now()

	lset := model.LabelSet{"aaaa": "AAAA", "bbbb": "BBBB", "cccc": "CCCC"}

	s.st = state{}
	for i := 0; i < numSilences; i++ {
		id := fmt.Sprint("ID", i)
		// Patterns also contain the ID to bust any caches that might be used under the hood.
		patA := "A{4}|" + id
		patB := id // Does not match.
		if i%10 == 0 {
			// Every 10th time, have an actually matching pattern.
			patB = "B(B|C)B.|" + id
		}

		s.st[id] = &pb.MeshSilence{Silence: &pb.Silence{
			Id: id,
			Matchers: []*pb.Matcher{
				{Type: pb.Matcher_REGEXP, Name: "aaaa", Pattern: patA},
				{Type: pb.Matcher_REGEXP, Name: "bbbb", Pattern: patB},
			},
			StartsAt:  now.Add(-time.Minute),
			EndsAt:    now.Add(time.Hour),
			UpdatedAt: now.Add(-time.Hour),
		}}
	}

	// Run things once to populate the matcherCache.
	sils, _, err := s.Query(
		QState(types.SilenceStateActive),
		QMatches(lset),
	)
	require.NoError(b, err)
	require.Equal(b, numSilences/10, len(sils))

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		sils, _, err := s.Query(
			QState(types.SilenceStateActive),
			QMatches(lset),
		)
		require.NoError(b, err)
		require.Equal(b, numSilences/10, len(sils))
	}
}

func Benchmark100SilencesQuery(b *testing.B) {
	benchmarkSilencesQuery(b, 100)
}

func Benchmark1000SilencesQuery(b *testing.B) {
	benchmarkSilencesQuery(b, 1000)
}

func Benchmark10000SilencesQuery(b *testing.B) {
	benchmarkSilencesQuery(b, 10000)
}

// runtime.Gosched() does not "suspend" the current goroutine so there's no guarantee that the main goroutine won't
// be able to continue. For more see https://pkg.go.dev/runtime#Gosched.
func gosched() {
	time.Sleep(1 * time.Millisecond)
}