Skip to content

Commit

Permalink
Merge pull request #563 from grafana/cedwards/ooo-native-histograms-g…
Browse files Browse the repository at this point in the history 
…auge-tests

Expand OOO native histogram tests to include gauge histograms
  • Loading branch information
@carrieedwards
carrieedwards authored Nov 21, 2023
2 parents 723b8d0 + 8989cf4 commit 86a2683
Show file tree
Hide file tree
Showing 4 changed files with 76 additions and 70 deletions.
20 changes: 1 addition & 19 deletions tsdb/db_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4474,25 +4474,7 @@ func testOOOCompaction(t *testing.T, scenario sampleTypeScenario) {
require.NoError(t, err)

actRes := query(t, q, labels.MustNewMatcher(labels.MatchRegexp, "foo", "bar.*"))
gotSamples := make(map[string][]chunks.Sample)
for k, v := range actRes {
var samples []chunks.Sample
for _, sample := range v {
switch sample.Type() {
case chunkenc.ValFloat:
samples = append(samples, sample)
case chunkenc.ValHistogram:
sample.H().CounterResetHint = histogram.UnknownCounterReset
samples = append(samples, sample)
case chunkenc.ValFloatHistogram:
sample.FH().CounterResetHint = histogram.UnknownCounterReset
samples = append(samples, sample)
}
}
gotSamples[k] = samples
}
require.Equal(t, expRes, gotSamples)
require.Equal(t, expRes, actRes)
requireEqualSamples(t, expRes, actRes, true)
}

// Checking for expected data in the blocks.
Expand Down
22 changes: 6 additions & 16 deletions tsdb/head_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4812,7 +4812,7 @@ func testWBLReplay(t *testing.T, scenario sampleTypeScenario) {
require.NoError(t, err)
require.NoError(t, h.Init(0))

var expOOOSamples []sample
var expOOOSamples []chunks.Sample
l := labels.FromStrings("foo", "bar")
appendSample := func(mins int64, isOOO bool) {
app := h.Appender(context.Background())
Expand Down Expand Up @@ -4863,7 +4863,7 @@ func testWBLReplay(t *testing.T, scenario sampleTypeScenario) {
require.Len(t, chks, 1)

it := chks[0].chunk.Iterator(nil)
actOOOSamples := make([]sample, 0, len(expOOOSamples))
actOOOSamples := make([]chunks.Sample, 0, len(expOOOSamples))
for {
valType := it.Next()
if valType == chunkenc.ValNone {
Expand All @@ -4886,23 +4886,13 @@ func testWBLReplay(t *testing.T, scenario sampleTypeScenario) {

// OOO chunk will be sorted. Hence sort the expected samples.
sort.Slice(expOOOSamples, func(i, j int) bool {
return expOOOSamples[i].t < expOOOSamples[j].t
return expOOOSamples[i].T() < expOOOSamples[j].T()
})

// Sets the counter reset hint to the values currently returned (first sample has counter reset unknown, others have not counter reset)
// Passing in true for the 'ignoreCounterResets' parameter prevents differences in counter reset headers
// from being factored in to the sample comparison
// TODO(fionaliao): understand counter reset behaviour, might want to modify this later
for i := range expOOOSamples {
if i != 0 {
if expOOOSamples[i].h != nil {
expOOOSamples[i].h.CounterResetHint = histogram.NotCounterReset
}
if expOOOSamples[i].fh != nil {
expOOOSamples[i].fh.CounterResetHint = histogram.NotCounterReset
}
}
}

require.Equal(t, expOOOSamples, actOOOSamples)
compareSamples(t, l.String(), expOOOSamples, actOOOSamples, true)

require.NoError(t, h.Close())
}
Expand Down
18 changes: 13 additions & 5 deletions tsdb/ooo_head_read_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -866,11 +866,15 @@ func testOOOHeadChunkReader_Chunk(t *testing.T, scenario sampleTypeScenario) {
resultSamples = append(resultSamples, sample{t: t, f: v})
case chunkenc.ValHistogram:
t, v := it.AtHistogram()
v.CounterResetHint = histogram.UnknownCounterReset
if v.CounterResetHint != histogram.GaugeType {
v.CounterResetHint = histogram.UnknownCounterReset
}
resultSamples = append(resultSamples, sample{t: t, h: v})
case chunkenc.ValFloatHistogram:
t, v := it.AtFloatHistogram()
v.CounterResetHint = histogram.UnknownCounterReset
if v.CounterResetHint != histogram.GaugeType {
v.CounterResetHint = histogram.UnknownCounterReset
}
resultSamples = append(resultSamples, sample{t: t, fh: v})
}
}
Expand Down Expand Up @@ -958,7 +962,7 @@ func testOOOHeadChunkReader_Chunk_ConsistentQueryResponseDespiteOfHeadExpanding(
},
},
{
name: "After Series() prev head gets mmapped after getting samples, new head gets new samples also overlapping, none of these should appear in the response.",
name: "After Series() prev head gets mmapped after getting samples, new head gets new samples also overlapping, none of these should appear in response.",
queryMinT: minutes(0),
queryMaxT: minutes(100),
firstInOrderSampleAt: minutes(120),
Expand Down Expand Up @@ -1054,11 +1058,15 @@ func testOOOHeadChunkReader_Chunk_ConsistentQueryResponseDespiteOfHeadExpanding(
resultSamples = append(resultSamples, sample{t: t, f: v})
case chunkenc.ValHistogram:
t, v := it.AtHistogram()
v.CounterResetHint = histogram.UnknownCounterReset
if v.CounterResetHint != histogram.GaugeType {
v.CounterResetHint = histogram.UnknownCounterReset
}
resultSamples = append(resultSamples, sample{t: t, h: v})
case chunkenc.ValFloatHistogram:
t, v := it.AtFloatHistogram()
v.CounterResetHint = histogram.UnknownCounterReset
if v.CounterResetHint != histogram.GaugeType {
v.CounterResetHint = histogram.UnknownCounterReset
}
resultSamples = append(resultSamples, sample{t: t, fh: v})
}
}
Expand Down
86 changes: 56 additions & 30 deletions tsdb/testutil.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,28 @@ var sampleTypeScenarios = map[string]sampleTypeScenario{
return sample{t: ts, fh: tsdbutil.GenerateTestFloatHistogram(int(value))}
},
},
"gauge int histogram": {
sampleType: sampleMetricTypeHistogram,
appendFunc: func(appender storage.Appender, lbls labels.Labels, ts, value int64) (storage.SeriesRef, error, sample) {
s := sample{t: ts, h: tsdbutil.GenerateTestGaugeHistogram(int(value))}
ref, err := appender.AppendHistogram(0, lbls, ts, s.h, nil)
return ref, err, s
},
sampleFunc: func(ts, value int64) sample {
return sample{t: ts, h: tsdbutil.GenerateTestGaugeHistogram(int(value))}
},
},
"gauge float histogram": {
sampleType: sampleMetricTypeHistogram,
appendFunc: func(appender storage.Appender, lbls labels.Labels, ts, value int64) (storage.SeriesRef, error, sample) {
s := sample{t: ts, fh: tsdbutil.GenerateTestGaugeFloatHistogram(int(value))}
ref, err := appender.AppendHistogram(0, lbls, ts, nil, s.fh)
return ref, err, s
},
sampleFunc: func(ts, value int64) sample {
return sample{t: ts, fh: tsdbutil.GenerateTestGaugeFloatHistogram(int(value))}
},
},
}

// requireEqualSamples checks that the actual series are equal to the expected ones. It ignores the counter reset hints for histograms.
Expand All @@ -66,39 +88,43 @@ func requireEqualSamples(t *testing.T, expected, actual map[string][]chunks.Samp
actualItem, ok := actual[name]
require.True(t, ok, "Expected series %s not found", name)
require.Equal(t, len(expectedItem), len(actualItem), "Length not expected for %s", name)
for i, s := range expectedItem {
expectedSample := s
actualSample := actualItem[i]
require.Equal(t, expectedSample.T(), expectedSample.T(), "Different timestamps for %s[%d]", name, i)
require.Equal(t, expectedSample.Type().String(), actualSample.Type().String(), "Different types for %s[%d] at ts %d", name, i, expectedSample.T())
switch {
case s.H() != nil:
{
expectedHist := expectedSample.H()
actualHist := actualSample.H()
if ignoreCounterResets {
expectedHist.CounterResetHint = histogram.UnknownCounterReset
actualHist.CounterResetHint = histogram.UnknownCounterReset
}
require.Equal(t, expectedHist, actualHist, "Sample doesn't match for %s[%d] at ts %d", name, i, expectedSample.T())
}
case s.FH() != nil:
{
expectedHist := expectedSample.FH()
actualHist := actualSample.FH()
if ignoreCounterResets {
expectedHist.CounterResetHint = histogram.UnknownCounterReset
actualHist.CounterResetHint = histogram.UnknownCounterReset
}
require.Equal(t, expectedHist, actualHist, "Sample doesn't match for %s[%d] at ts %d", name, i, expectedSample.T())
}
default:
require.Equal(t, expectedSample, expectedSample, "Sample doesn't match for %s[%d] at ts %d", name, i, expectedSample.T())
}
}
compareSamples(t, name, expectedItem, actualItem, ignoreCounterResets)
}
for name := range actual {
_, ok := expected[name]
require.True(t, ok, "Unexpected series %s", name)
}
}

func compareSamples(t *testing.T, name string, expected, actual []chunks.Sample, ignoreCounterResets bool) {
for i, s := range expected {
expectedSample := s
actualSample := actual[i]
require.Equal(t, expectedSample.T(), expectedSample.T(), "Different timestamps for %s[%d]", name, i)
require.Equal(t, expectedSample.Type().String(), actualSample.Type().String(), "Different types for %s[%d] at ts %d", name, i, expectedSample.T())
switch {
case s.H() != nil:
{
expectedHist := expectedSample.H()
actualHist := actualSample.H()
if ignoreCounterResets && expectedHist.CounterResetHint != histogram.GaugeType {
expectedHist.CounterResetHint = histogram.UnknownCounterReset
actualHist.CounterResetHint = histogram.UnknownCounterReset
}
require.Equal(t, expectedHist, actualHist, "Sample doesn't match for %s[%d] at ts %d", name, i, expectedSample.T())
}
case s.FH() != nil:
{
expectedHist := expectedSample.FH()
actualHist := actualSample.FH()
if ignoreCounterResets {
expectedHist.CounterResetHint = histogram.UnknownCounterReset
actualHist.CounterResetHint = histogram.UnknownCounterReset
}
require.Equal(t, expectedHist, actualHist, "Sample doesn't match for %s[%d] at ts %d", name, i, expectedSample.T())
}
default:
require.Equal(t, expectedSample, expectedSample, "Sample doesn't match for %s[%d] at ts %d", name, i, expectedSample.T())
}
}
}

0 comments on commit 86a2683

Please sign in to comment.