Add flag to disable store gateway query stats (#5749)

* add flag to enable store gateway query stats

Signed-off-by: Ben Ye <benye@amazon.com>

* docs update

Signed-off-by: Ben Ye <benye@amazon.com>

* fix tests

Signed-off-by: Ben Ye <benye@amazon.com>

---------

Signed-off-by: Ben Ye <benye@amazon.com>

CHANGELOG.md

@@ -16,6 +16,7 @@
 * [ENHANCEMENT] Index Cache: Multi level cache adds config `max_backfill_items` to cap max items to backfill per async operation. #5686
 * [ENHANCEMENT] Query Frontend: Log number of split queries in `query stats` log. #5703
 * [ENHANCEMENT] Logging: Added new options for logging HTTP request headers: `-server.log-request-headers` enables logging HTTP request headers, `-server.log-request-headers-exclude-list` allows users to specify headers which should not be logged. #5744
+* [ENHANCEMENT] Querier: Added `querier.store-gateway-query-stats-enabled` to enable or disable store gateway query stats log. #5749
 * [BUGFIX] Distributor: Do not use label with empty values for sharding #5717
 * [BUGFIX] Query Frontend: queries with negative offset should check whether it is cacheable or not. #5719
 * [BUGFIX] Redis Cache: pass `cache_size` config correctly. #5734

docs/blocks-storage/querier.md

@@ -214,6 +214,10 @@ querier:
     # CLI flag: -querier.store-gateway-client.grpc-compression
     [grpc_compression: <string> | default = ""]
 
+  # If enabled, store gateway query stats will be logged using `info` log level.
+  # CLI flag: -querier.store-gateway-query-stats-enabled
+  [store_gateway_query_stats: <boolean> | default = true]
+
   # When distributor's sharding strategy is shuffle-sharding and this setting is
   # > 0, queriers fetch in-memory series from the minimum set of required
   # ingesters, selecting only ingesters which may have received series since

docs/configuration/config-file-reference.md

@@ -3527,6 +3527,10 @@ store_gateway_client:
   # CLI flag: -querier.store-gateway-client.grpc-compression
   [grpc_compression: <string> | default = ""]
 
+# If enabled, store gateway query stats will be logged using `info` log level.
+# CLI flag: -querier.store-gateway-query-stats-enabled
+[store_gateway_query_stats: <boolean> | default = true]
+
 # When distributor's sharding strategy is shuffle-sharding and this setting is >
 # 0, queriers fetch in-memory series from the minimum set of required ingesters,
 # selecting only ingesters which may have received series since 'now - lookback

pkg/querier/blocks_store_queryable.go

@@ -138,6 +138,8 @@ type BlocksStoreQueryable struct {
 	metrics         *blocksStoreQueryableMetrics
 	limits          BlocksStoreLimits
 
+	storeGatewayQueryStatsEnabled bool
+
 	// Subservices manager.
 	subservices        *services.Manager
 	subservicesWatcher *services.FailureWatcher
@@ -149,6 +151,7 @@ func NewBlocksStoreQueryable(
 	consistency *BlocksConsistencyChecker,
 	limits BlocksStoreLimits,
 	queryStoreAfter time.Duration,
+	storeGatewayQueryStatsEnabled bool,
 	logger log.Logger,
 	reg prometheus.Registerer,
 ) (*BlocksStoreQueryable, error) {
@@ -158,15 +161,16 @@ func NewBlocksStoreQueryable(
 	}
 
 	q := &BlocksStoreQueryable{
-		stores:             stores,
-		finder:             finder,
-		consistency:        consistency,
-		queryStoreAfter:    queryStoreAfter,
-		logger:             logger,
-		subservices:        manager,
-		subservicesWatcher: services.NewFailureWatcher(),
-		metrics:            newBlocksStoreQueryableMetrics(reg),
-		limits:             limits,
+		stores:                        stores,
+		finder:                        finder,
+		consistency:                   consistency,
+		queryStoreAfter:               queryStoreAfter,
+		logger:                        logger,
+		subservices:                   manager,
+		subservicesWatcher:            services.NewFailureWatcher(),
+		metrics:                       newBlocksStoreQueryableMetrics(reg),
+		limits:                        limits,
+		storeGatewayQueryStatsEnabled: storeGatewayQueryStatsEnabled,
 	}
 
 	q.Service = services.NewBasicService(q.starting, q.running, q.stopping)
@@ -256,7 +260,7 @@ func NewBlocksStoreQueryableFromConfig(querierCfg Config, gatewayCfg storegatewa
 		reg,
 	)
 
-	return NewBlocksStoreQueryable(stores, finder, consistency, limits, querierCfg.QueryStoreAfter, logger, reg)
+	return NewBlocksStoreQueryable(stores, finder, consistency, limits, querierCfg.QueryStoreAfter, querierCfg.StoreGatewayQueryStatsEnabled, logger, reg)
 }
 
 func (q *BlocksStoreQueryable) starting(ctx context.Context) error {
@@ -291,15 +295,16 @@ func (q *BlocksStoreQueryable) Querier(mint, maxt int64) (storage.Querier, error
 	}
 
 	return &blocksStoreQuerier{
-		minT:            mint,
-		maxT:            maxt,
-		finder:          q.finder,
-		stores:          q.stores,
-		metrics:         q.metrics,
-		limits:          q.limits,
-		consistency:     q.consistency,
-		logger:          q.logger,
-		queryStoreAfter: q.queryStoreAfter,
+		minT:                          mint,
+		maxT:                          maxt,
+		finder:                        q.finder,
+		stores:                        q.stores,
+		metrics:                       q.metrics,
+		limits:                        q.limits,
+		consistency:                   q.consistency,
+		logger:                        q.logger,
+		queryStoreAfter:               q.queryStoreAfter,
+		storeGatewayQueryStatsEnabled: q.storeGatewayQueryStatsEnabled,
 	}, nil
 }
 
@@ -315,6 +320,10 @@ type blocksStoreQuerier struct {
 	// If set, the querier manipulates the max time to not be greater than
 	// "now - queryStoreAfter" so that most recent blocks are not queried.
 	queryStoreAfter time.Duration
+
+	// If enabled, query stats of store gateway requests will be logged
+	// using `info` level.
+	storeGatewayQueryStatsEnabled bool
 }
 
 // Select implements storage.Querier interface.
@@ -753,7 +762,7 @@ func (q *blocksStoreQuerier) fetchSeriesFromStores(
 			// Use number of blocks queried to check whether we should log the query
 			// or not. It might be logging too much but good to understand per request
 			// performance.
-			if seriesQueryStats.BlocksQueried > 0 {
+			if q.storeGatewayQueryStatsEnabled && seriesQueryStats.BlocksQueried > 0 {
 				level.Info(spanLog).Log("msg", "store gateway series request stats",
 					"instance", c.RemoteAddress(),
 					"queryable_chunk_bytes_fetched", chunkBytes,

pkg/querier/blocks_store_queryable_test.go

@@ -1588,7 +1588,7 @@ func TestBlocksStoreQuerier_PromQLExecution(t *testing.T) {
 			}
 
 			// Instance the querier that will be executed to run the query.
-			queryable, err := NewBlocksStoreQueryable(stores, finder, NewBlocksConsistencyChecker(0, 0, logger, nil), &blocksStoreLimitsMock{}, 0, logger, nil)
+			queryable, err := NewBlocksStoreQueryable(stores, finder, NewBlocksConsistencyChecker(0, 0, logger, nil), &blocksStoreLimitsMock{}, 0, false, logger, nil)
 			require.NoError(t, err)
 			require.NoError(t, services.StartAndAwaitRunning(context.Background(), queryable))
 			defer services.StopAndAwaitTerminated(context.Background(), queryable) // nolint:errcheck

pkg/querier/querier.go

@@ -75,8 +75,9 @@ type Config struct {
 	LookbackDelta time.Duration `yaml:"lookback_delta"`
 
 	// Blocks storage only.
-	StoreGatewayAddresses string       `yaml:"store_gateway_addresses"`
-	StoreGatewayClient    ClientConfig `yaml:"store_gateway_client"`
+	StoreGatewayAddresses         string       `yaml:"store_gateway_addresses"`
+	StoreGatewayClient            ClientConfig `yaml:"store_gateway_client"`
+	StoreGatewayQueryStatsEnabled bool         `yaml:"store_gateway_query_stats"`
 
 	ShuffleShardingIngestersLookbackPeriod time.Duration `yaml:"shuffle_sharding_ingesters_lookback_period"`
 
@@ -112,6 +113,7 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
 	f.DurationVar(&cfg.QueryStoreAfter, "querier.query-store-after", 0, "The time after which a metric should be queried from storage and not just ingesters. 0 means all queries are sent to store. When running the blocks storage, if this option is enabled, the time range of the query sent to the store will be manipulated to ensure the query end is not more recent than 'now - query-store-after'.")
 	f.StringVar(&cfg.ActiveQueryTrackerDir, "querier.active-query-tracker-dir", "./active-query-tracker", "Active query tracker monitors active queries, and writes them to the file in given directory. If Cortex discovers any queries in this log during startup, it will log them to the log file. Setting to empty value disables active query tracker, which also disables -querier.max-concurrent option.")
 	f.StringVar(&cfg.StoreGatewayAddresses, "querier.store-gateway-addresses", "", "Comma separated list of store-gateway addresses in DNS Service Discovery format. This option should be set when using the blocks storage and the store-gateway sharding is disabled (when enabled, the store-gateway instances form a ring and addresses are picked from the ring).")
+	f.BoolVar(&cfg.StoreGatewayQueryStatsEnabled, "querier.store-gateway-query-stats-enabled", true, "If enabled, store gateway query stats will be logged using `info` log level.")
 	f.DurationVar(&cfg.LookbackDelta, "querier.lookback-delta", 5*time.Minute, "Time since the last sample after which a time series is considered stale and ignored by expression evaluations.")
 	f.DurationVar(&cfg.ShuffleShardingIngestersLookbackPeriod, "querier.shuffle-sharding-ingesters-lookback-period", 0, "When distributor's sharding strategy is shuffle-sharding and this setting is > 0, queriers fetch in-memory series from the minimum set of required ingesters, selecting only ingesters which may have received series since 'now - lookback period'. The lookback period should be greater or equal than the configured 'query store after' and 'query ingesters within'. If this setting is 0, queriers always query all ingesters (ingesters shuffle sharding on read path is disabled).")
 	f.BoolVar(&cfg.ThanosEngine, "querier.thanos-engine", false, "Experimental. Use Thanos promql engine https://github.com/thanos-io/promql-engine rather than the Prometheus promql engine.")