prometheus/tsdb/ooo_isolation_test.go
Charles Korn 59844498f7
Fix issue where queries can fail or omit OOO samples if OOO head compaction occurs between creating a querier and reading chunks (#13115)
* Add failing test.

Signed-off-by: Charles Korn <charles.korn@grafana.com>

* Don't run OOO head garbage collection while reads are running.

Signed-off-by: Charles Korn <charles.korn@grafana.com>

* Add further test cases for different order of operations.

Signed-off-by: Charles Korn <charles.korn@grafana.com>

* Ensure all queriers are closed if `DB.blockChunkQuerierForRange()` fails.

Signed-off-by: Charles Korn <charles.korn@grafana.com>

* Ensure all queriers are closed if `DB.Querier()` fails.

Signed-off-by: Charles Korn <charles.korn@grafana.com>

* Invert error handling in `DB.Querier()` and `DB.blockChunkQuerierForRange()` to make it clearer

Signed-off-by: Charles Korn <charles.korn@grafana.com>

* Ensure that queries that touch OOO data can't block OOO head garbage collection forever.

Signed-off-by: Charles Korn <charles.korn@grafana.com>

* Address PR feedback: fix parameter name in comment

Co-authored-by: Jesus Vazquez <jesusvazquez@users.noreply.github.com>
Signed-off-by: Charles Korn <charleskorn@users.noreply.github.com>

* Address PR feedback: use `lastGarbageCollectedMmapRef`

Signed-off-by: Charles Korn <charles.korn@grafana.com>

* Address PR feedback: ensure pending reads are cleaned up if creating an OOO querier fails

Signed-off-by: Charles Korn <charles.korn@grafana.com>

---------

Signed-off-by: Charles Korn <charles.korn@grafana.com>
Signed-off-by: Charles Korn <charleskorn@users.noreply.github.com>
Co-authored-by: Jesus Vazquez <jesusvazquez@users.noreply.github.com>
2023-11-24 12:38:38 +01:00

61 lines
2.0 KiB
Go

// Copyright 2023 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tsdb
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestOOOIsolation(t *testing.T) {
i := newOOOIsolation()
// Empty state shouldn't have any open reads.
require.False(t, i.HasOpenReadsAtOrBefore(0))
require.False(t, i.HasOpenReadsAtOrBefore(1))
require.False(t, i.HasOpenReadsAtOrBefore(2))
require.False(t, i.HasOpenReadsAtOrBefore(3))
// Add a read.
read1 := i.TrackReadAfter(1)
require.False(t, i.HasOpenReadsAtOrBefore(0))
require.False(t, i.HasOpenReadsAtOrBefore(1))
require.True(t, i.HasOpenReadsAtOrBefore(2))
// Add another overlapping read.
read2 := i.TrackReadAfter(0)
require.False(t, i.HasOpenReadsAtOrBefore(0))
require.True(t, i.HasOpenReadsAtOrBefore(1))
require.True(t, i.HasOpenReadsAtOrBefore(2))
// Close the second read, should now only report open reads for the first read's ref.
read2.Close()
require.False(t, i.HasOpenReadsAtOrBefore(0))
require.False(t, i.HasOpenReadsAtOrBefore(1))
require.True(t, i.HasOpenReadsAtOrBefore(2))
// Close the second read again: this should do nothing and ensures we can safely call Close() multiple times.
read2.Close()
require.False(t, i.HasOpenReadsAtOrBefore(0))
require.False(t, i.HasOpenReadsAtOrBefore(1))
require.True(t, i.HasOpenReadsAtOrBefore(2))
// Closing the first read should indicate no further open reads.
read1.Close()
require.False(t, i.HasOpenReadsAtOrBefore(0))
require.False(t, i.HasOpenReadsAtOrBefore(1))
require.False(t, i.HasOpenReadsAtOrBefore(2))
}