Query: Allow to ignore No StoreAPIs matched error

CHANGELOG.md

@@ -15,6 +15,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 - [#5990](https://github.com/thanos-io/thanos/pull/5990) Cache/Redis: add support for Redis Sentinel via new option `master_name`.
 - [#6008](https://github.com/thanos-io/thanos/pull/6008) *: Add counter metric `gate_queries_total` to gate.
 - [#5773](https://github.com/thanos-io/thanos/pull/5773) Store: Support disable cache index header file.
+- [#5937](https://github.com/thanos-io/thanos/pull/5937) Query: Allow to ignore No StoreAPIs matched error.
 
 ### Changed
 

cmd/thanos/query.go

@@ -174,6 +174,9 @@ func registerQuery(app *extkingpin.App) {
 	enableQueryPartialResponse := cmd.Flag("query.partial-response", "Enable partial response for queries if no partial_response param is specified. --no-query.partial-response for disabling.").
 		Default("true").Bool()
 
+	enableQueryIgnoreNoStoresMatched := cmd.Flag("query.ignore-no-stores-matched", "Enable to ignore no StoreAPIs matched error for queries if no ignore_no_stores_matched param is specified.").
+		Default("false").Bool()
+
 	enableRulePartialResponse := cmd.Flag("rule.partial-response", "Enable partial response for rules endpoint. --no-rule.partial-response for disabling.").
 		Hidden().Default("true").Bool()
 
@@ -298,6 +301,7 @@ func registerQuery(app *extkingpin.App) {
 			*exemplarEndpoints,
 			*enableAutodownsampling,
 			*enableQueryPartialResponse,
+			*enableQueryIgnoreNoStoresMatched,
 			*enableRulePartialResponse,
 			*enableTargetPartialResponse,
 			*enableMetricMetadataPartialResponse,
@@ -374,6 +378,7 @@ func runQuery(
 	exemplarAddrs []string,
 	enableAutodownsampling bool,
 	enableQueryPartialResponse bool,
+	enableQueryIgnoreNoStoresMatched bool,
 	enableRulePartialResponse bool,
 	enableTargetPartialResponse bool,
 	enableMetricMetadataPartialResponse bool,
@@ -685,6 +690,7 @@ func runQuery(
 			exemplars.NewGRPCClientWithDedup(exemplarsProxy, queryReplicaLabels),
 			enableAutodownsampling,
 			enableQueryPartialResponse,
+			enableQueryIgnoreNoStoresMatched,
 			enableRulePartialResponse,
 			enableTargetPartialResponse,
 			enableMetricMetadataPartialResponse,

docs/components/query.md

@@ -358,6 +358,10 @@ Flags:
                                  = max(rangeSeconds / 250, defaultStep)).
                                  This will not work from Grafana, but Grafana
                                  has __step variable which can be used.
+      --query.ignore-no-stores-matched
+                                 Enable to ignore no StoreAPIs matched error for
+                                 queries if no ignore_no_stores_matched param is
+                                 specified.
       --query.lookback-delta=QUERY.LOOKBACK-DELTA
                                  The maximum lookback duration for retrieving
                                  metrics during expression evaluations.

pkg/api/query/grpc.go

@@ -91,6 +91,7 @@ func (g *GRPCAPI) Query(request *querypb.QueryRequest, server querypb.Query_Quer
 		storeMatchers,
 		maxResolution,
 		request.EnablePartialResponse,
+		request.EnableIgnoreNoStoresMatched,
 		request.EnableQueryPushdown,
 		false,
 		request.ShardInfo,
@@ -166,6 +167,7 @@ func (g *GRPCAPI) QueryRange(request *querypb.QueryRangeRequest, srv querypb.Que
 		storeMatchers,
 		maxResolution,
 		request.EnablePartialResponse,
+		request.EnableIgnoreNoStoresMatched,
 		request.EnableQueryPushdown,
 		false,
 		request.ShardInfo,

pkg/api/query/querypb/query.proto

@@ -41,6 +41,8 @@ message QueryRequest {
   ShardInfo shard_info = 11;
 
   int64 lookback_delta_seconds = 12;
+
+  bool enableIgnoreNoStoresMatched = 13;
 }
 
 message StoreMatchers {
@@ -78,6 +80,8 @@ message QueryRangeRequest {
 
   ShardInfo shard_info = 13;
   int64 lookback_delta_seconds = 14;
+
+  bool enableIgnoreNoStoresMatched = 15;
 }
 
 message QueryRangeResponse {

pkg/api/query/v1.go

@@ -63,16 +63,17 @@ import (
 )
 
 const (
-	DedupParam               = "dedup"
-	PartialResponseParam     = "partial_response"
-	MaxSourceResolutionParam = "max_source_resolution"
-	ReplicaLabelsParam       = "replicaLabels[]"
-	MatcherParam             = "match[]"
-	StoreMatcherParam        = "storeMatch[]"
-	Step                     = "step"
-	Stats                    = "stats"
-	ShardInfoParam           = "shard_info"
-	LookbackDeltaParam       = "lookback_delta"
+	DedupParam                 = "dedup"
+	PartialResponseParam       = "partial_response"
+	MaxSourceResolutionParam   = "max_source_resolution"
+	ReplicaLabelsParam         = "replicaLabels[]"
+	MatcherParam               = "match[]"
+	StoreMatcherParam          = "storeMatch[]"
+	Step                       = "step"
+	Stats                      = "stats"
+	ShardInfoParam             = "shard_info"
+	LookbackDeltaParam         = "lookback_delta"
+	IgnoreNoStoresMatchedParam = "ignore_no_stores_matched"
 )
 
 // QueryAPI is an API used by Thanos Querier.
@@ -91,6 +92,7 @@ type QueryAPI struct {
 
 	enableAutodownsampling              bool
 	enableQueryPartialResponse          bool
+	enableQueryIgnoreNoStoresMatched    bool
 	enableRulePartialResponse           bool
 	enableTargetPartialResponse         bool
 	enableMetricMetadataPartialResponse bool
@@ -128,6 +130,7 @@ func NewQueryAPI(
 	exemplars exemplars.UnaryClient,
 	enableAutodownsampling bool,
 	enableQueryPartialResponse bool,
+	enableQueryIgnoreNoStoresMatched bool,
 	enableRulePartialResponse bool,
 	enableTargetPartialResponse bool,
 	enableMetricMetadataPartialResponse bool,
@@ -159,6 +162,7 @@ func NewQueryAPI(
 		exemplars:                              exemplars,
 		enableAutodownsampling:                 enableAutodownsampling,
 		enableQueryPartialResponse:             enableQueryPartialResponse,
+		enableQueryIgnoreNoStoresMatched:       enableQueryIgnoreNoStoresMatched,
 		enableRulePartialResponse:              enableRulePartialResponse,
 		enableTargetPartialResponse:            enableTargetPartialResponse,
 		enableMetricMetadataPartialResponse:    enableMetricMetadataPartialResponse,
@@ -312,6 +316,18 @@ func (qapi *QueryAPI) parsePartialResponseParam(r *http.Request, defaultEnablePa
 	return defaultEnablePartialResponse, nil
 }
 
+func (qapi *QueryAPI) parseIgnoreNoStoresMatchedParam(r *http.Request, defaultIgnoreNoStoresMatched bool) (enableIgnoreNoStoresMatched bool, _ *api.ApiError) {
+	// Overwrite the cli flag when provided as a query parameter.
+	if val := r.FormValue(IgnoreNoStoresMatchedParam); val != "" {
+		var err error
+		defaultIgnoreNoStoresMatched, err = strconv.ParseBool(val)
+		if err != nil {
+			return false, &api.ApiError{Typ: api.ErrorBadData, Err: errors.Wrapf(err, "'%s' parameter", IgnoreNoStoresMatchedParam)}
+		}
+	}
+	return defaultIgnoreNoStoresMatched, nil
+}
+
 func (qapi *QueryAPI) parseStep(r *http.Request, defaultRangeQueryStep time.Duration, rangeSeconds int64) (time.Duration, *api.ApiError) {
 	// Overwrite the cli flag when provided as a query parameter.
 	if val := r.FormValue(Step); val != "" {
@@ -383,6 +399,11 @@ func (qapi *QueryAPI) query(r *http.Request) (interface{}, []error, *api.ApiErro
 		return nil, nil, apiErr, func() {}
 	}
 
+	enableIgnoreNoStoresMatched, apiErr := qapi.parseIgnoreNoStoresMatchedParam(r, qapi.enableQueryIgnoreNoStoresMatched)
+	if apiErr != nil {
+		return nil, nil, apiErr, func() {}
+	}
+
 	maxSourceResolution, apiErr := qapi.parseDownsamplingParamMillis(r, qapi.defaultInstantQueryMaxSourceResolution)
 	if apiErr != nil {
 		return nil, nil, apiErr, func() {}
@@ -415,6 +436,7 @@ func (qapi *QueryAPI) query(r *http.Request) (interface{}, []error, *api.ApiErro
 			storeDebugMatchers,
 			maxSourceResolution,
 			enablePartialResponse,
+			enableIgnoreNoStoresMatched,
 			qapi.enableQueryPushdown,
 			false,
 			shardInfo,
@@ -536,6 +558,11 @@ func (qapi *QueryAPI) queryRange(r *http.Request) (interface{}, []error, *api.Ap
 		return nil, nil, apiErr, func() {}
 	}
 
+	enableIgnoreNoStoresMatched, apiErr := qapi.parseIgnoreNoStoresMatchedParam(r, qapi.enableQueryIgnoreNoStoresMatched)
+	if apiErr != nil {
+		return nil, nil, apiErr, func() {}
+	}
+
 	shardInfo, apiErr := qapi.parseShardInfo(r)
 	if apiErr != nil {
 		return nil, nil, apiErr, func() {}
@@ -566,6 +593,7 @@ func (qapi *QueryAPI) queryRange(r *http.Request) (interface{}, []error, *api.Ap
 			storeDebugMatchers,
 			maxSourceResolution,
 			enablePartialResponse,
+			enableIgnoreNoStoresMatched,
 			qapi.enableQueryPushdown,
 			false,
 			shardInfo,
@@ -635,6 +663,11 @@ func (qapi *QueryAPI) labelValues(r *http.Request) (interface{}, []error, *api.A
 		return nil, nil, apiErr, func() {}
 	}
 
+	enableIgnoreNoStoresMatched, apiErr := qapi.parseIgnoreNoStoresMatchedParam(r, qapi.enableQueryIgnoreNoStoresMatched)
+	if apiErr != nil {
+		return nil, nil, apiErr, func() {}
+	}
+
 	storeDebugMatchers, apiErr := qapi.parseStoreDebugMatchersParam(r)
 	if apiErr != nil {
 		return nil, nil, apiErr, func() {}
@@ -655,6 +688,7 @@ func (qapi *QueryAPI) labelValues(r *http.Request) (interface{}, []error, *api.A
 		storeDebugMatchers,
 		0,
 		enablePartialResponse,
+		enableIgnoreNoStoresMatched,
 		qapi.enableQueryPushdown,
 		true,
 		nil,
@@ -745,12 +779,18 @@ func (qapi *QueryAPI) series(r *http.Request) (interface{}, []error, *api.ApiErr
 		return nil, nil, apiErr, func() {}
 	}
 
+	enableIgnoreNoStoresMatched, apiErr := qapi.parseIgnoreNoStoresMatchedParam(r, qapi.enableQueryIgnoreNoStoresMatched)
+	if apiErr != nil {
+		return nil, nil, apiErr, func() {}
+	}
+
 	q, err := qapi.queryableCreate(
 		enableDedup,
 		replicaLabels,
 		storeDebugMatchers,
 		math.MaxInt64,
 		enablePartialResponse,
+		enableIgnoreNoStoresMatched,
 		qapi.enableQueryPushdown,
 		true,
 		nil,
@@ -791,6 +831,11 @@ func (qapi *QueryAPI) labelNames(r *http.Request) (interface{}, []error, *api.Ap
 		return nil, nil, apiErr, func() {}
 	}
 
+	enableIgnoreNoStoresMatched, apiErr := qapi.parseIgnoreNoStoresMatchedParam(r, qapi.enableQueryIgnoreNoStoresMatched)
+	if apiErr != nil {
+		return nil, nil, apiErr, func() {}
+	}
+
 	storeDebugMatchers, apiErr := qapi.parseStoreDebugMatchersParam(r)
 	if apiErr != nil {
 		return nil, nil, apiErr, func() {}
@@ -811,6 +856,7 @@ func (qapi *QueryAPI) labelNames(r *http.Request) (interface{}, []error, *api.Ap
 		storeDebugMatchers,
 		0,
 		enablePartialResponse,
+		enableIgnoreNoStoresMatched,
 		qapi.enableQueryPushdown,
 		true,
 		nil,