From 24602c428e8d55d3aa4cbccd12a1e84ec18c22af Mon Sep 17 00:00:00 2001
From: SriKrishna Paparaju <paparaju@gmail.com>
Date: Wed, 27 Oct 2021 19:01:52 -0400
Subject: [PATCH] adds new metric for tracking last loaded block (#4710)

Signed-off-by: SriKrishna Paparaju <paparaju@gmail.com>
---
 CHANGELOG.md                 |  1 +
 pkg/store/bucket.go          |  7 ++++++-
 pkg/store/bucket_e2e_test.go | 30 ++++++++++++++++++++++++++++++
 3 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 80036ed4be..40e8055db7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -15,6 +15,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 - [#4680](https://github.com/thanos-io/thanos/pull/4680) Query: add `exemplar.partial-response` flag to control partial response.
 - [#4679](https://github.com/thanos-io/thanos/pull/4679) Added `enable-feature` flag to enable negative offsets and @ modifier, similar to Prometheus.
 - [#4696](https://github.com/thanos-io/thanos/pull/4696) Query: add cache name to tracing spans.
+- [#4710](https://github.com/thanos-io/thanos/pull/4710) Store: add metric to capture timestamp of the last loaded block.
 - [#4736](https://github.com/thanos-io/thanos/pull/4736) S3: Add capability to use custom AWS STS Endpoint.
 - [#4764](https://github.com/thanos-io/thanos/pull/4764) Compactor: add `block-viewer.global.sync-block-timeout` flag to set the timeout of synchronization block metas.
 
diff --git a/pkg/store/bucket.go b/pkg/store/bucket.go
index be0e4bec97..0d8066e2bf 100644
--- a/pkg/store/bucket.go
+++ b/pkg/store/bucket.go
@@ -104,6 +104,7 @@ type bucketStoreMetrics struct {
 	blocksLoaded          prometheus.Gauge
 	blockLoads            prometheus.Counter
 	blockLoadFailures     prometheus.Counter
+	lastLoadedBlock       prometheus.Gauge
 	blockDrops            prometheus.Counter
 	blockDropFailures     prometheus.Counter
 	seriesDataTouched     *prometheus.SummaryVec
@@ -151,6 +152,10 @@ func newBucketStoreMetrics(reg prometheus.Registerer) *bucketStoreMetrics {
 		Name: "thanos_bucket_store_blocks_loaded",
 		Help: "Number of currently loaded blocks.",
 	})
+	m.lastLoadedBlock = promauto.With(reg).NewGauge(prometheus.GaugeOpts{
+		Name: "thanos_bucket_store_blocks_last_loaded_timestamp_seconds",
+		Help: "Timestamp when last block got loaded.",
+	})
 
 	m.seriesDataTouched = promauto.With(reg).NewSummaryVec(prometheus.SummaryOpts{
 		Name: "thanos_bucket_store_series_data_touched",
@@ -628,7 +633,7 @@ func (s *BucketStore) addBlock(ctx context.Context, meta *metadata.Meta) (err er
 	s.blocks[b.meta.ULID] = b
 
 	s.metrics.blocksLoaded.Inc()
-
+	s.metrics.lastLoadedBlock.SetToCurrentTime()
 	return nil
 }
 
diff --git a/pkg/store/bucket_e2e_test.go b/pkg/store/bucket_e2e_test.go
index fda9f6a1f9..e1dd747315 100644
--- a/pkg/store/bucket_e2e_test.go
+++ b/pkg/store/bucket_e2e_test.go
@@ -17,6 +17,7 @@ import (
 	"github.com/gogo/status"
 	"github.com/oklog/ulid"
 	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
 	"github.com/prometheus/prometheus/pkg/labels"
 	"github.com/prometheus/prometheus/pkg/relabel"
 	"github.com/prometheus/prometheus/pkg/timestamp"
@@ -178,6 +179,7 @@ func prepareStoreWithTestBlocks(t testing.TB, dir string, bkt objstore.Bucket, m
 	}, nil)
 	testutil.Ok(t, err)
 
+	reg := prometheus.NewRegistry()
 	store, err := NewBucketStore(
 		objstore.WithNoopInstr(bkt),
 		metaFetcher,
@@ -194,6 +196,7 @@ func prepareStoreWithTestBlocks(t testing.TB, dir string, bkt objstore.Bucket, m
 		WithLogger(s.logger),
 		WithIndexCache(s.cache),
 		WithFilterConfig(filterConf),
+		WithRegistry(reg),
 	)
 	testutil.Ok(t, err)
 	defer func() { testutil.Ok(t, store.Close()) }()
@@ -207,10 +210,37 @@ func prepareStoreWithTestBlocks(t testing.TB, dir string, bkt objstore.Bucket, m
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
 
+	// Check if the blocks are being loaded.
+	start := time.Now()
+	time.Sleep(time.Second * 1)
 	testutil.Ok(t, store.SyncBlocks(ctx))
+
+	// Get the value of the metric 'thanos_bucket_store_blocks_last_loaded_timestamp_seconds' to capture the timestamp of the last loaded block.
+	m := gatherFamily(t, reg, "thanos_bucket_store_blocks_last_loaded_timestamp_seconds")
+	lastUploaded := time.Unix(int64(m.Metric[0].Gauge.GetValue()), 0)
+
+	if lastUploaded.Before(start) {
+		t.Fatalf("no blocks are loaded ")
+	}
+
 	return s
 }
 
+func gatherFamily(t testing.TB, reg prometheus.Gatherer, familyName string) *dto.MetricFamily {
+
+	families, err := reg.Gather()
+	testutil.Ok(t, err)
+
+	for _, f := range families {
+		if f.GetName() == familyName {
+			return f
+		}
+	}
+
+	t.Fatalf("could not find family %s", familyName)
+	return nil
+}
+
 // TODO(bwplotka): Benchmark Series.
 func testBucketStore_e2e(t *testing.T, ctx context.Context, s *storeSuite) {
 	t.Helper()