From 5022b01a19e8722c5bd24494fa882ca366019993 Mon Sep 17 00:00:00 2001 From: metonymic-smokey Date: Thu, 4 Nov 2021 18:03:47 +0530 Subject: [PATCH] draft: remove hardcoding from compact progress calc Signed-off-by: metonymic-smokey --- pkg/compact/compact_test.go | 280 +++++++++++------------------------- 1 file changed, 80 insertions(+), 200 deletions(-) diff --git a/pkg/compact/compact_test.go b/pkg/compact/compact_test.go index 76177e4445c..ed7ec2cd1f2 100644 --- a/pkg/compact/compact_test.go +++ b/pkg/compact/compact_test.go @@ -184,6 +184,7 @@ func TestCompactProgressCalculate(t *testing.T) { type planResult struct { compactionBlocks, compactionRuns float64 } + type groupedResult map[string]planResult logger := log.NewNopLogger() reg := prometheus.NewRegistry() @@ -195,12 +196,24 @@ func TestCompactProgressCalculate(t *testing.T) { int64(2 * 24 * time.Hour / time.Millisecond), }) - extLabels := labels.FromMap(map[string]string{"a": "1", "b": "2"}) + // pre calculating group keys + keys := make([]string, 3) + m := make([]metadata.Meta, 3) + m[0].Thanos.Labels = map[string]string{"a": "1"} + m[1].Thanos.Labels = map[string]string{"b": "2"} + m[2].Thanos.Labels = map[string]string{"c": "3"} + for ind, meta := range m { + keys[ind] = DefaultGroupKey(meta.Thanos) + } + + var bkt objstore.Bucket + temp := prometheus.NewCounter(prometheus.CounterOpts{}) + grouper := NewDefaultGrouper(logger, bkt, false, false, reg, temp, temp, temp, "") for _, tcase := range []struct { testName string input []*metadata.Meta - expected planResult + expected groupedResult }{ // In this test case, the first four blocks are planned for compaction in the first run. These are then removed from the group and then the next two blocks from the original group are planned for compaction in the second run. // Hence, a total of 6 blocks are planned for compaction over 2 runs. @@ -237,7 +250,7 @@ func TestCompactProgressCalculate(t *testing.T) { }, Thanos: metadata.Thanos{ Version: 1, - Labels: map[string]string{"a": "1"}, + Labels: map[string]string{"b": "2"}, }, }, { @@ -248,7 +261,7 @@ func TestCompactProgressCalculate(t *testing.T) { }, Thanos: metadata.Thanos{ Version: 1, - Labels: map[string]string{"a": "1"}, + Labels: map[string]string{"b": "2"}, }, }, { @@ -259,7 +272,7 @@ func TestCompactProgressCalculate(t *testing.T) { }, Thanos: metadata.Thanos{ Version: 1, - Labels: map[string]string{"a": "1"}, + Labels: map[string]string{"b": "2"}, }, }, { @@ -270,7 +283,7 @@ func TestCompactProgressCalculate(t *testing.T) { }, Thanos: metadata.Thanos{ Version: 1, - Labels: map[string]string{"a": "1"}, + Labels: map[string]string{"c": "3"}, }, }, { @@ -281,7 +294,7 @@ func TestCompactProgressCalculate(t *testing.T) { }, Thanos: metadata.Thanos{ Version: 1, - Labels: map[string]string{"a": "1"}, + Labels: map[string]string{"c": "3"}, }, }, { @@ -292,13 +305,23 @@ func TestCompactProgressCalculate(t *testing.T) { }, Thanos: metadata.Thanos{ Version: 1, - Labels: map[string]string{"a": "1"}, + Labels: map[string]string{"c": "3"}, }, }, }, - expected: planResult{ - compactionRuns: 2.0, - compactionBlocks: 6.0, + expected: map[string]planResult{ + "0@7311538254805615637": planResult{ + compactionRuns: 0.0, + compactionBlocks: 0.0, + }, + "0@16020112273640739857": planResult{ + compactionRuns: 0.0, + compactionBlocks: 0.0, + }, + "0@14920274840801531072": planResult{ + compactionRuns: 0.0, + compactionBlocks: 0.0, + }, }, }, { @@ -325,7 +348,7 @@ func TestCompactProgressCalculate(t *testing.T) { }, Thanos: metadata.Thanos{ Version: 1, - Labels: map[string]string{"a": "1"}, + Labels: map[string]string{"b": "2"}, }, }, { @@ -336,7 +359,7 @@ func TestCompactProgressCalculate(t *testing.T) { }, Thanos: metadata.Thanos{ Version: 1, - Labels: map[string]string{"a": "1"}, + Labels: map[string]string{"b": "2"}, }, }, { @@ -347,7 +370,7 @@ func TestCompactProgressCalculate(t *testing.T) { }, Thanos: metadata.Thanos{ Version: 1, - Labels: map[string]string{"a": "1"}, + Labels: map[string]string{"c": "3"}, }, }, { @@ -358,7 +381,7 @@ func TestCompactProgressCalculate(t *testing.T) { }, Thanos: metadata.Thanos{ Version: 1, - Labels: map[string]string{"a": "1"}, + Labels: map[string]string{"c": "3"}, }, }, { @@ -369,13 +392,23 @@ func TestCompactProgressCalculate(t *testing.T) { }, Thanos: metadata.Thanos{ Version: 1, - Labels: map[string]string{"a": "1"}, + Labels: map[string]string{"c": "3"}, }, }, }, - expected: planResult{ - compactionRuns: 1.0, - compactionBlocks: 4.0, + expected: map[string]planResult{ + "0@7311538254805615637": planResult{ + compactionRuns: 0.0, + compactionBlocks: 0.0, + }, + "0@16020112273640739857": planResult{ + compactionRuns: 0.0, + compactionBlocks: 0.0, + }, + "0@14920274840801531072": planResult{ + compactionRuns: 0.0, + compactionBlocks: 0.0, + }, }, }, { @@ -401,7 +434,7 @@ func TestCompactProgressCalculate(t *testing.T) { }, Thanos: metadata.Thanos{ Version: 1, - Labels: map[string]string{"a": "1"}, + Labels: map[string]string{"b": "2"}, }, }, { @@ -412,7 +445,7 @@ func TestCompactProgressCalculate(t *testing.T) { }, Thanos: metadata.Thanos{ Version: 1, - Labels: map[string]string{"a": "1"}, + Labels: map[string]string{"b": "2"}, }, }, { @@ -423,7 +456,7 @@ func TestCompactProgressCalculate(t *testing.T) { }, Thanos: metadata.Thanos{ Version: 1, - Labels: map[string]string{"a": "1"}, + Labels: map[string]string{"c": "1"}, }, }, { @@ -434,206 +467,53 @@ func TestCompactProgressCalculate(t *testing.T) { }, Thanos: metadata.Thanos{ Version: 1, - Labels: map[string]string{"a": "1"}, + Labels: map[string]string{"c": "1"}, }, }, }, - expected: planResult{ - compactionRuns: 1.0, - compactionBlocks: 4.0, - }, - }, - { - // In this test case, the metadata is part of two groups. - // The first four blocks are compacted in the first run. - testName: "two_groups_test", - input: []*metadata.Meta{ - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(0, nil), - MinTime: 0, - MaxTime: int64(2 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"a": "1"}, - }, - }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(1, nil), - MinTime: int64(2 * time.Hour / time.Millisecond), - MaxTime: int64(4 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"a": "1"}, - }, - }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(2, nil), - MinTime: int64(4 * time.Hour / time.Millisecond), - MaxTime: int64(6 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"b": "2"}, - }, + expected: map[string]planResult{ + "0@7311538254805615637": planResult{ + compactionRuns: 0.0, + compactionBlocks: 0.0, }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(3, nil), - MinTime: int64(6 * time.Hour / time.Millisecond), - MaxTime: int64(8 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"b": "2"}, - }, - }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(4, nil), - MinTime: int64(8 * time.Hour / time.Millisecond), - MaxTime: int64(16 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"b": "2"}, - }, + "0@16020112273640739857": planResult{ + compactionRuns: 0.0, + compactionBlocks: 0.0, }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(6, nil), - MinTime: int64(16 * time.Hour / time.Millisecond), - MaxTime: int64(18 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"b": "2"}, - }, - }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(7, nil), - MinTime: int64(18 * time.Hour / time.Millisecond), - MaxTime: int64(26 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"b": "2"}, - }, + "0@14920274840801531072": planResult{ + compactionRuns: 0.0, + compactionBlocks: 0.0, }, }, - expected: planResult{ - compactionRuns: 1.0, - compactionBlocks: 4.0, - }, }, + {}, } { if ok := t.Run(tcase.testName, func(t *testing.T) { - groups := []*Group{ - { - key: "b", - labels: extLabels, - resolution: 0, - metasByMinTime: tcase.input, - }, - } + groups := make([]*Group, 3) - if tcase.testName == "two_groups_test" { - groups = append(groups, &Group{ - key: "a", - metasByMinTime: []*metadata.Meta{ - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(0, nil), - MinTime: int64(0 * time.Hour / time.Millisecond), - MaxTime: int64(2 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"b": "2"}, - }, - }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(1, nil), - MinTime: int64(2 * time.Hour / time.Millisecond), - MaxTime: int64(4 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"b": "2"}, - }, - }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(2, nil), - MinTime: int64(4 * time.Hour / time.Millisecond), - MaxTime: int64(6 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"b": "2"}, - }, - }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(3, nil), - MinTime: int64(6 * time.Hour / time.Millisecond), - MaxTime: int64(8 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"b": "2"}, - }, - }, - { - BlockMeta: tsdb.BlockMeta{ - ULID: ulid.MustNew(4, nil), - MinTime: int64(8 * time.Hour / time.Millisecond), - MaxTime: int64(10 * time.Hour / time.Millisecond), - }, - Thanos: metadata.Thanos{ - Version: 1, - Labels: map[string]string{"b": "2"}, - }, - }, - }, - resolution: 0, - }) + blocks := make(map[ulid.ULID]*metadata.Meta, len(tcase.input)) + for _, meta := range tcase.input { + blocks[meta.ULID] = meta } - + // form groups from the input metadata - do not hardcode groups. hence, grouper.Groups should stay + groups, _ = grouper.Groups(blocks) + // check if the key in the groups returned here matches with the pre calculated keys ps := NewCompactionProgressCalculator(unRegisterer, planner) err := ps.ProgressCalculate(context.Background(), groups) metrics := ps.CompactProgressMetrics testutil.Ok(t, err) - a, err := metrics.NumberOfCompactionBlocks.GetMetricWithLabelValues("b") - if err != nil { - level.Warn(logger).Log("msg", "could not get number of blocks") - } - testutil.Equals(t, tcase.expected.compactionBlocks, promtestutil.ToFloat64(a)) - - b, err := metrics.NumberOfCompactionRuns.GetMetricWithLabelValues("b") - if err != nil { - level.Warn(logger).Log("msg", "could not get number of runs") - } - testutil.Equals(t, tcase.expected.compactionRuns, promtestutil.ToFloat64(b)) - - if tcase.testName == "two_groups_test" { - a, err = metrics.NumberOfCompactionBlocks.GetMetricWithLabelValues("a") + for _, key := range keys { + a, err := metrics.NumberOfCompactionBlocks.GetMetricWithLabelValues(key) if err != nil { - level.Warn(logger).Log("msg", "could not get number of blocks for two groups test") + level.Warn(logger).Log("msg", "could not get number of blocks") } - testutil.Equals(t, 4.0, promtestutil.ToFloat64(a)) - b, err := metrics.NumberOfCompactionRuns.GetMetricWithLabelValues("a") + b, err := metrics.NumberOfCompactionRuns.GetMetricWithLabelValues(key) if err != nil { level.Warn(logger).Log("msg", "could not get number of runs") } - testutil.Equals(t, 1.0, promtestutil.ToFloat64(b)) + testutil.Equals(t, tcase.expected[key].compactionBlocks, promtestutil.ToFloat64(a)) + testutil.Equals(t, tcase.expected[key].compactionRuns, promtestutil.ToFloat64(b)) } }); !ok { return