diff --git a/CHANGELOG.md b/CHANGELOG.md index 64fa3fd564..b54c6abb1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ * [ENHANCEMENT] Index Cache: Multi level cache adds config `max_backfill_items` to cap max items to backfill per async operation. #5686 * [ENHANCEMENT] Query Frontend: Log number of split queries in `query stats` log. #5703 * [ENHANCEMENT] Logging: Added new options for logging HTTP request headers: `-server.log-request-headers` enables logging HTTP request headers, `-server.log-request-headers-exclude-list` allows users to specify headers which should not be logged. #5744 +* [ENHANCEMENT] Querier: Added `querier.store-gateway-query-stats-enabled` to enable or disable store gateway query stats log. #5749 * [BUGFIX] Distributor: Do not use label with empty values for sharding #5717 * [BUGFIX] Query Frontend: queries with negative offset should check whether it is cacheable or not. #5719 * [BUGFIX] Redis Cache: pass `cache_size` config correctly. #5734 diff --git a/docs/blocks-storage/querier.md b/docs/blocks-storage/querier.md index 4c303613ca..aa7e138a50 100644 --- a/docs/blocks-storage/querier.md +++ b/docs/blocks-storage/querier.md @@ -214,6 +214,10 @@ querier: # CLI flag: -querier.store-gateway-client.grpc-compression [grpc_compression: | default = ""] + # If enabled, store gateway query stats will be logged using `info` log level. + # CLI flag: -querier.store-gateway-query-stats-enabled + [store_gateway_query_stats: | default = true] + # When distributor's sharding strategy is shuffle-sharding and this setting is # > 0, queriers fetch in-memory series from the minimum set of required # ingesters, selecting only ingesters which may have received series since diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index cac4fd15c2..efea8a6e1b 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -3527,6 +3527,10 @@ store_gateway_client: # CLI flag: -querier.store-gateway-client.grpc-compression [grpc_compression: | default = ""] +# If enabled, store gateway query stats will be logged using `info` log level. +# CLI flag: -querier.store-gateway-query-stats-enabled +[store_gateway_query_stats: | default = true] + # When distributor's sharding strategy is shuffle-sharding and this setting is > # 0, queriers fetch in-memory series from the minimum set of required ingesters, # selecting only ingesters which may have received series since 'now - lookback diff --git a/pkg/querier/blocks_store_queryable.go b/pkg/querier/blocks_store_queryable.go index c4adf5bdab..a8b0f0027c 100644 --- a/pkg/querier/blocks_store_queryable.go +++ b/pkg/querier/blocks_store_queryable.go @@ -138,6 +138,8 @@ type BlocksStoreQueryable struct { metrics *blocksStoreQueryableMetrics limits BlocksStoreLimits + storeGatewayQueryStatsEnabled bool + // Subservices manager. subservices *services.Manager subservicesWatcher *services.FailureWatcher @@ -149,6 +151,7 @@ func NewBlocksStoreQueryable( consistency *BlocksConsistencyChecker, limits BlocksStoreLimits, queryStoreAfter time.Duration, + storeGatewayQueryStatsEnabled bool, logger log.Logger, reg prometheus.Registerer, ) (*BlocksStoreQueryable, error) { @@ -158,15 +161,16 @@ func NewBlocksStoreQueryable( } q := &BlocksStoreQueryable{ - stores: stores, - finder: finder, - consistency: consistency, - queryStoreAfter: queryStoreAfter, - logger: logger, - subservices: manager, - subservicesWatcher: services.NewFailureWatcher(), - metrics: newBlocksStoreQueryableMetrics(reg), - limits: limits, + stores: stores, + finder: finder, + consistency: consistency, + queryStoreAfter: queryStoreAfter, + logger: logger, + subservices: manager, + subservicesWatcher: services.NewFailureWatcher(), + metrics: newBlocksStoreQueryableMetrics(reg), + limits: limits, + storeGatewayQueryStatsEnabled: storeGatewayQueryStatsEnabled, } q.Service = services.NewBasicService(q.starting, q.running, q.stopping) @@ -256,7 +260,7 @@ func NewBlocksStoreQueryableFromConfig(querierCfg Config, gatewayCfg storegatewa reg, ) - return NewBlocksStoreQueryable(stores, finder, consistency, limits, querierCfg.QueryStoreAfter, logger, reg) + return NewBlocksStoreQueryable(stores, finder, consistency, limits, querierCfg.QueryStoreAfter, querierCfg.StoreGatewayQueryStatsEnabled, logger, reg) } func (q *BlocksStoreQueryable) starting(ctx context.Context) error { @@ -291,15 +295,16 @@ func (q *BlocksStoreQueryable) Querier(mint, maxt int64) (storage.Querier, error } return &blocksStoreQuerier{ - minT: mint, - maxT: maxt, - finder: q.finder, - stores: q.stores, - metrics: q.metrics, - limits: q.limits, - consistency: q.consistency, - logger: q.logger, - queryStoreAfter: q.queryStoreAfter, + minT: mint, + maxT: maxt, + finder: q.finder, + stores: q.stores, + metrics: q.metrics, + limits: q.limits, + consistency: q.consistency, + logger: q.logger, + queryStoreAfter: q.queryStoreAfter, + storeGatewayQueryStatsEnabled: q.storeGatewayQueryStatsEnabled, }, nil } @@ -315,6 +320,10 @@ type blocksStoreQuerier struct { // If set, the querier manipulates the max time to not be greater than // "now - queryStoreAfter" so that most recent blocks are not queried. queryStoreAfter time.Duration + + // If enabled, query stats of store gateway requests will be logged + // using `info` level. + storeGatewayQueryStatsEnabled bool } // Select implements storage.Querier interface. @@ -753,7 +762,7 @@ func (q *blocksStoreQuerier) fetchSeriesFromStores( // Use number of blocks queried to check whether we should log the query // or not. It might be logging too much but good to understand per request // performance. - if seriesQueryStats.BlocksQueried > 0 { + if q.storeGatewayQueryStatsEnabled && seriesQueryStats.BlocksQueried > 0 { level.Info(spanLog).Log("msg", "store gateway series request stats", "instance", c.RemoteAddress(), "queryable_chunk_bytes_fetched", chunkBytes, diff --git a/pkg/querier/blocks_store_queryable_test.go b/pkg/querier/blocks_store_queryable_test.go index 85de2f7615..f624644866 100644 --- a/pkg/querier/blocks_store_queryable_test.go +++ b/pkg/querier/blocks_store_queryable_test.go @@ -1588,7 +1588,7 @@ func TestBlocksStoreQuerier_PromQLExecution(t *testing.T) { } // Instance the querier that will be executed to run the query. - queryable, err := NewBlocksStoreQueryable(stores, finder, NewBlocksConsistencyChecker(0, 0, logger, nil), &blocksStoreLimitsMock{}, 0, logger, nil) + queryable, err := NewBlocksStoreQueryable(stores, finder, NewBlocksConsistencyChecker(0, 0, logger, nil), &blocksStoreLimitsMock{}, 0, false, logger, nil) require.NoError(t, err) require.NoError(t, services.StartAndAwaitRunning(context.Background(), queryable)) defer services.StopAndAwaitTerminated(context.Background(), queryable) // nolint:errcheck diff --git a/pkg/querier/querier.go b/pkg/querier/querier.go index 2a5e03af3b..bcb0bd321f 100644 --- a/pkg/querier/querier.go +++ b/pkg/querier/querier.go @@ -75,8 +75,9 @@ type Config struct { LookbackDelta time.Duration `yaml:"lookback_delta"` // Blocks storage only. - StoreGatewayAddresses string `yaml:"store_gateway_addresses"` - StoreGatewayClient ClientConfig `yaml:"store_gateway_client"` + StoreGatewayAddresses string `yaml:"store_gateway_addresses"` + StoreGatewayClient ClientConfig `yaml:"store_gateway_client"` + StoreGatewayQueryStatsEnabled bool `yaml:"store_gateway_query_stats"` ShuffleShardingIngestersLookbackPeriod time.Duration `yaml:"shuffle_sharding_ingesters_lookback_period"` @@ -112,6 +113,7 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { f.DurationVar(&cfg.QueryStoreAfter, "querier.query-store-after", 0, "The time after which a metric should be queried from storage and not just ingesters. 0 means all queries are sent to store. When running the blocks storage, if this option is enabled, the time range of the query sent to the store will be manipulated to ensure the query end is not more recent than 'now - query-store-after'.") f.StringVar(&cfg.ActiveQueryTrackerDir, "querier.active-query-tracker-dir", "./active-query-tracker", "Active query tracker monitors active queries, and writes them to the file in given directory. If Cortex discovers any queries in this log during startup, it will log them to the log file. Setting to empty value disables active query tracker, which also disables -querier.max-concurrent option.") f.StringVar(&cfg.StoreGatewayAddresses, "querier.store-gateway-addresses", "", "Comma separated list of store-gateway addresses in DNS Service Discovery format. This option should be set when using the blocks storage and the store-gateway sharding is disabled (when enabled, the store-gateway instances form a ring and addresses are picked from the ring).") + f.BoolVar(&cfg.StoreGatewayQueryStatsEnabled, "querier.store-gateway-query-stats-enabled", true, "If enabled, store gateway query stats will be logged using `info` log level.") f.DurationVar(&cfg.LookbackDelta, "querier.lookback-delta", 5*time.Minute, "Time since the last sample after which a time series is considered stale and ignored by expression evaluations.") f.DurationVar(&cfg.ShuffleShardingIngestersLookbackPeriod, "querier.shuffle-sharding-ingesters-lookback-period", 0, "When distributor's sharding strategy is shuffle-sharding and this setting is > 0, queriers fetch in-memory series from the minimum set of required ingesters, selecting only ingesters which may have received series since 'now - lookback period'. The lookback period should be greater or equal than the configured 'query store after' and 'query ingesters within'. If this setting is 0, queriers always query all ingesters (ingesters shuffle sharding on read path is disabled).") f.BoolVar(&cfg.ThanosEngine, "querier.thanos-engine", false, "Experimental. Use Thanos promql engine https://github.com/thanos-io/promql-engine rather than the Prometheus promql engine.")