diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ced30f722f..a7d2d86c7cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,6 +63,7 @@ * `-store-gateway.sharding-ring.etcd.tls-cipher-suites` * `-store-gateway.sharding-ring.etcd.tls-min-version` * [ENHANCEMENT] Store-gateway: Add `-blocks-storage.bucket-store.max-concurrent-reject-over-limit` option to allow requests that exceed the max number of inflight object storage requests to be rejected. #2999 +* [ENHANCEMENT] Query-frontend: allow setting a separate limit on the total (before splitting/sharding) query length of range queries with the new experimental `-query-frontend.max-total-query-length` flag, which defaults to `-store.max-query-length` if unset or set to 0. #3058 * [BUGFIX] Querier: Fix 400 response while handling streaming remote read. #2963 * [BUGFIX] Fix a bug causing query-frontend, query-scheduler, and querier not failing if one of their internal components fail. #2978 * [BUGFIX] Querier: re-balance the querier worker connections when a query-frontend or query-scheduler is terminated. #3005 diff --git a/cmd/mimir/config-descriptor.json b/cmd/mimir/config-descriptor.json index 78f4a2de0a1..5ad58131afe 100644 --- a/cmd/mimir/config-descriptor.json +++ b/cmd/mimir/config-descriptor.json @@ -3076,7 +3076,7 @@ "kind": "field", "name": "max_query_length", "required": false, - "desc": "Limit the query time range (end - start time). This limit is enforced in the query-frontend (on the received query), in the querier (on the query possibly split by the query-frontend) and ruler. 0 to disable.", + "desc": "Limit the query time range (end - start time). This limit is enforced in the querier (on the query possibly split by the query-frontend) and ruler. 0 to disable.", "fieldValue": null, "fieldDefaultValue": 0, "fieldFlag": "store.max-query-length", @@ -3154,6 +3154,17 @@ "fieldType": "duration", "fieldCategory": "experimental" }, + { + "kind": "field", + "name": "max_total_query_length", + "required": false, + "desc": "Limit the total query time range (end - start time). This limit is enforced in the query-frontend on the received query. Defaults to the value of -store.max-query-length if set to 0.", + "fieldValue": null, + "fieldDefaultValue": 0, + "fieldFlag": "query-frontend.max-total-query-length", + "fieldType": "duration", + "fieldCategory": "experimental" + }, { "kind": "field", "name": "cardinality_analysis_enabled", diff --git a/cmd/mimir/help-all.txt.tmpl b/cmd/mimir/help-all.txt.tmpl index 4e6d50f5b63..936e7c9788b 100644 --- a/cmd/mimir/help-all.txt.tmpl +++ b/cmd/mimir/help-all.txt.tmpl @@ -1357,6 +1357,8 @@ Usage of ./cmd/mimir/mimir: Maximum number of queriers that can handle requests for a single tenant. If set to 0 or value higher than number of available queriers, *all* queriers will handle requests for the tenant. Each frontend (or query-scheduler, if used) will select the same set of queriers for the same tenant (given that all queriers are connected to all frontends / query-schedulers). This option only works with queriers connecting to the query-frontend / query-scheduler, not when using downstream URL. -query-frontend.max-retries-per-request int Maximum number of retries for a single request; beyond this, the downstream error is returned. (default 5) + -query-frontend.max-total-query-length duration + [experimental] Limit the total query time range (end - start time). This limit is enforced in the query-frontend on the received query. Defaults to the value of -store.max-query-length if set to 0. -query-frontend.parallelize-shardable-queries True to enable query sharding. -query-frontend.querier-forget-delay duration @@ -1954,7 +1956,7 @@ Usage of ./cmd/mimir/mimir: -store.max-labels-query-length duration Limit the time range (end - start time) of series, label names and values queries. This limit is enforced in the querier. If the requested time range is outside the allowed range, the request will not fail but will be manipulated to only query data within the allowed time range. 0 to disable. -store.max-query-length duration - Limit the query time range (end - start time). This limit is enforced in the query-frontend (on the received query), in the querier (on the query possibly split by the query-frontend) and ruler. 0 to disable. + Limit the query time range (end - start time). This limit is enforced in the querier (on the query possibly split by the query-frontend) and ruler. 0 to disable. -target comma-separated-list-of-strings Comma-separated list of components to include in the instantiated process. The default value 'all' includes all components that are required to form a functional Grafana Mimir instance in single-binary mode. Use the '-modules' command line flag to get a list of available components, and to see which components are included with 'all'. (default all) -tenant-federation.enabled diff --git a/cmd/mimir/help.txt.tmpl b/cmd/mimir/help.txt.tmpl index 582bfa58bf0..ef9304e72fa 100644 --- a/cmd/mimir/help.txt.tmpl +++ b/cmd/mimir/help.txt.tmpl @@ -574,7 +574,7 @@ Usage of ./cmd/mimir/mimir: -store.max-labels-query-length duration Limit the time range (end - start time) of series, label names and values queries. This limit is enforced in the querier. If the requested time range is outside the allowed range, the request will not fail but will be manipulated to only query data within the allowed time range. 0 to disable. -store.max-query-length duration - Limit the query time range (end - start time). This limit is enforced in the query-frontend (on the received query), in the querier (on the query possibly split by the query-frontend) and ruler. 0 to disable. + Limit the query time range (end - start time). This limit is enforced in the querier (on the query possibly split by the query-frontend) and ruler. 0 to disable. -target comma-separated-list-of-strings Comma-separated list of components to include in the instantiated process. The default value 'all' includes all components that are required to form a functional Grafana Mimir instance in single-binary mode. Use the '-modules' command line flag to get a list of available components, and to see which components are included with 'all'. (default all) -tenant-federation.enabled diff --git a/docs/sources/operators-guide/configure/about-versioning.md b/docs/sources/operators-guide/configure/about-versioning.md index 69129db2e76..ab342f4fc47 100644 --- a/docs/sources/operators-guide/configure/about-versioning.md +++ b/docs/sources/operators-guide/configure/about-versioning.md @@ -84,6 +84,7 @@ The following features are currently experimental: - Snapshotting of in-memory TSDB data on disk when shutting down (`-blocks-storage.tsdb.memory-snapshot-on-shutdown`) - Out-of-order samples ingestion (`-ingester.out-of-order-allowance`) - Query-frontend + - `-query-frontend.max-total-query-length` - `-query-frontend.querier-forget-delay` - Instant query splitting (`-query-frontend.split-instant-queries-by-interval`) - Query-scheduler diff --git a/docs/sources/operators-guide/configure/reference-configuration-parameters/index.md b/docs/sources/operators-guide/configure/reference-configuration-parameters/index.md index 73720ef55cb..36b090e075a 100644 --- a/docs/sources/operators-guide/configure/reference-configuration-parameters/index.md +++ b/docs/sources/operators-guide/configure/reference-configuration-parameters/index.md @@ -2478,8 +2478,8 @@ The `limits` block configures default and per-tenant limits imposed by component [max_query_lookback: | default = 0s] # Limit the query time range (end - start time). This limit is enforced in the -# query-frontend (on the received query), in the querier (on the query possibly -# split by the query-frontend) and ruler. 0 to disable. +# querier (on the query possibly split by the query-frontend) and ruler. 0 to +# disable. # CLI flag: -store.max-query-length [max_query_length: | default = 0s] @@ -2530,6 +2530,12 @@ The `limits` block configures default and per-tenant limits imposed by component # CLI flag: -query-frontend.split-instant-queries-by-interval [split_instant_queries_by_interval: | default = 0s] +# (experimental) Limit the total query time range (end - start time). This limit +# is enforced in the query-frontend on the received query. Defaults to the value +# of -store.max-query-length if set to 0. +# CLI flag: -query-frontend.max-total-query-length +[max_total_query_length: | default = 0s] + # Enables endpoints used for cardinality analysis. # CLI flag: -querier.cardinality-analysis-enabled [cardinality_analysis_enabled: | default = false] diff --git a/docs/sources/operators-guide/mimir-runbooks/_index.md b/docs/sources/operators-guide/mimir-runbooks/_index.md index 6f715ca3c00..854786c3f93 100644 --- a/docs/sources/operators-guide/mimir-runbooks/_index.md +++ b/docs/sources/operators-guide/mimir-runbooks/_index.md @@ -1376,7 +1376,7 @@ How to **fix** it: ### err-mimir-max-query-length -This error occurs when the time range of a query exceeds the configured maximum length. +This error occurs when the time range of a partial (after possible splitting, sharding by the query-frontend) query exceeds the configured maximum length. For a limit on the total query length, see [err-mimir-max-total-query-length](#err-mimir-max-total-query-length). Both PromQL instant and range queries can fetch metrics data over a period of time. A [range query](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) requires a `start` and `end` timestamp, so the difference of `end` minus `start` is the time range length of the query. @@ -1389,6 +1389,18 @@ Mimir has a limit on the query length. This limit is applied to partial queries, after they've split (according to time) by the query-frontend. This limit protects the system’s stability from potential abuse or mistakes. To configure the limit on a per-tenant basis, use the `-store.max-query-length` option (or `max_query_length` in the runtime configuration). +### err-mimir-max-total-query-length + +This error occurs when the time range of a query exceeds the configured maximum length. For a limit on the partial query length (after query splitting by interval and/or sharding), see [err-mimir-max-query-length](#err-mimir-max-query-length). + +PromQL range queries can fetch metrics data over a period of time. +A [range query](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) requires a `start` and `end` timestamp, so the difference of `end` minus `start` is the time range length of the query. + +Mimir has a limit on the query length. +This limit is applied to range queries before they are split (according to time) or sharded by the query-frontend. This limit protects the system’s stability from potential abuse or mistakes. +To configure the limit on a per-tenant basis, use the `-query-frontend.max-total-query-length` option (or `max_total_query_length` in the runtime configuration). +If this limit is set to 0, it takes its value from `-store.max-query-length`. + ### err-mimir-tenant-max-request-rate This error occurs when the rate of write requests per second is exceeded for this tenant. diff --git a/pkg/frontend/querymiddleware/limits.go b/pkg/frontend/querymiddleware/limits.go index 667c0d61548..7e6c0861f6a 100644 --- a/pkg/frontend/querymiddleware/limits.go +++ b/pkg/frontend/querymiddleware/limits.go @@ -32,7 +32,7 @@ type Limits interface { MaxQueryLookback(userID string) time.Duration // MaxQueryLength returns the limit of the length (in time) of a query. - MaxQueryLength(userID string) time.Duration + MaxTotalQueryLength(userID string) time.Duration // MaxQueryParallelism returns the limit to the number of split queries the // frontend will process in parallel. @@ -112,10 +112,10 @@ func (l limitsMiddleware) Do(ctx context.Context, r Request) (Response, error) { } // Enforce the max query length. - if maxQueryLength := validation.SmallestPositiveNonZeroDurationPerTenant(tenantIDs, l.MaxQueryLength); maxQueryLength > 0 { + if maxQueryLength := validation.SmallestPositiveNonZeroDurationPerTenant(tenantIDs, l.MaxTotalQueryLength); maxQueryLength > 0 { queryLen := timestamp.Time(r.GetEnd()).Sub(timestamp.Time(r.GetStart())) if queryLen > maxQueryLength { - return nil, apierror.New(apierror.TypeBadData, validation.NewMaxQueryLengthError(queryLen, maxQueryLength).Error()) + return nil, apierror.New(apierror.TypeBadData, validation.NewMaxTotalQueryLengthError(queryLen, maxQueryLength).Error()) } } diff --git a/pkg/frontend/querymiddleware/limits_test.go b/pkg/frontend/querymiddleware/limits_test.go index 5cdb78931b2..b21ebcad4f2 100644 --- a/pkg/frontend/querymiddleware/limits_test.go +++ b/pkg/frontend/querymiddleware/limits_test.go @@ -119,10 +119,11 @@ func TestLimitsMiddleware_MaxQueryLength(t *testing.T) { now := time.Now() tests := map[string]struct { - maxQueryLength time.Duration - reqStartTime time.Time - reqEndTime time.Time - expectedErr string + maxQueryLength time.Duration + maxTotalQueryLength time.Duration + reqStartTime time.Time + reqEndTime time.Time + expectedErr string }{ "should skip validation if max length is disabled": { maxQueryLength: 0, @@ -148,13 +149,19 @@ func TestLimitsMiddleware_MaxQueryLength(t *testing.T) { maxQueryLength: thirtyDays, reqStartTime: now.Add(-thirtyDays).Add(-100 * time.Hour), reqEndTime: now, - expectedErr: "the query time range exceeds the limit", + expectedErr: "the total query time range exceeds the limit", }, "should fail on a query on large time range over the limit, ending in the past": { maxQueryLength: thirtyDays, reqStartTime: now.Add(-4 * thirtyDays), reqEndTime: now.Add(-2 * thirtyDays), - expectedErr: "the query time range exceeds the limit", + expectedErr: "the total query time range exceeds the limit", + }, + "should succeed if total query length is higher than query length limit": { + maxQueryLength: thirtyDays, + maxTotalQueryLength: 8 * thirtyDays, + reqStartTime: now.Add(-4 * thirtyDays), + reqEndTime: now.Add(-2 * thirtyDays), }, } @@ -165,7 +172,7 @@ func TestLimitsMiddleware_MaxQueryLength(t *testing.T) { End: util.TimeToMillis(testData.reqEndTime), } - limits := mockLimits{maxQueryLength: testData.maxQueryLength} + limits := mockLimits{maxQueryLength: testData.maxQueryLength, maxTotalQueryLength: testData.maxTotalQueryLength} middleware := newLimitsMiddleware(limits, log.NewNopLogger()) innerRes := newEmptyPrometheusResponse() @@ -198,6 +205,7 @@ func TestLimitsMiddleware_MaxQueryLength(t *testing.T) { type mockLimits struct { maxQueryLookback time.Duration maxQueryLength time.Duration + maxTotalQueryLength time.Duration maxCacheFreshness time.Duration maxQueryParallelism int maxShardedQueries int @@ -214,6 +222,13 @@ func (m mockLimits) MaxQueryLength(string) time.Duration { return m.maxQueryLength } +func (m mockLimits) MaxTotalQueryLength(string) time.Duration { + if m.maxTotalQueryLength == time.Duration(0) { + return m.maxQueryLength + } + return m.maxTotalQueryLength +} + func (m mockLimits) MaxQueryParallelism(string) int { if m.maxQueryParallelism == 0 { return 14 // Flag default. diff --git a/pkg/util/globalerror/errors.go b/pkg/util/globalerror/errors.go index 669a4d39330..f9f13615bc1 100644 --- a/pkg/util/globalerror/errors.go +++ b/pkg/util/globalerror/errors.go @@ -50,6 +50,7 @@ const ( MetricMetadataUnitTooLong ID = "unit-too-long" MaxQueryLength ID = "max-query-length" + MaxTotalQueryLength ID = "max-total-query-length" RequestRateLimited ID = "tenant-max-request-rate" IngestionRateLimited ID = "tenant-max-ingestion-rate" TooManyHAClusters ID = "tenant-too-many-ha-clusters" diff --git a/pkg/util/validation/errors.go b/pkg/util/validation/errors.go index 3e132dfa09c..457e588db7f 100644 --- a/pkg/util/validation/errors.go +++ b/pkg/util/validation/errors.go @@ -263,6 +263,12 @@ func NewMaxQueryLengthError(actualQueryLen, maxQueryLength time.Duration) LimitE maxQueryLengthFlag)) } +func NewMaxTotalQueryLengthError(actualQueryLen, maxTotalQueryLength time.Duration) LimitError { + return LimitError(globalerror.MaxTotalQueryLength.MessageWithPerTenantLimitConfig( + fmt.Sprintf("the total query time range exceeds the limit (query length: %s, limit: %s)", actualQueryLen, maxTotalQueryLength), + maxTotalQueryLengthFlag)) +} + func NewRequestRateLimitedError(limit float64, burst int) LimitError { return LimitError(globalerror.RequestRateLimited.MessageWithPerTenantLimitConfig( fmt.Sprintf("the request has been rejected because the tenant exceeded the request rate limit, set to %v requests/s across all distributors with a maximum allowed burst of %d", limit, burst), diff --git a/pkg/util/validation/errors_test.go b/pkg/util/validation/errors_test.go index 5763d129324..1419065221a 100644 --- a/pkg/util/validation/errors_test.go +++ b/pkg/util/validation/errors_test.go @@ -31,6 +31,11 @@ func TestNewMaxQueryLengthError(t *testing.T) { assert.Equal(t, "the query time range exceeds the limit (query length: 1h0m0s, limit: 1m0s) (err-mimir-max-query-length). To adjust the related per-tenant limit, configure -store.max-query-length, or contact your service administrator.", err.Error()) } +func TestNewTotalMaxQueryLengthError(t *testing.T) { + err := NewMaxTotalQueryLengthError(time.Hour, time.Minute) + assert.Equal(t, "the total query time range exceeds the limit (query length: 1h0m0s, limit: 1m0s) (err-mimir-max-total-query-length). To adjust the related per-tenant limit, configure -query-frontend.max-total-query-length, or contact your service administrator.", err.Error()) +} + func TestNewRequestRateLimitedError(t *testing.T) { err := NewRequestRateLimitedError(10, 5) assert.Equal(t, "the request has been rejected because the tenant exceeded the request rate limit, set to 10 requests/s across all distributors with a maximum allowed burst of 5 (err-mimir-tenant-max-request-rate). To adjust the related per-tenant limits, configure -distributor.request-rate-limit and -distributor.request-burst-size, or contact your service administrator.", err.Error()) diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 9da17c4b86e..159a4f7d321 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -38,6 +38,7 @@ const ( maxMetadataLengthFlag = "validation.max-metadata-length" creationGracePeriodFlag = "validation.create-grace-period" maxQueryLengthFlag = "store.max-query-length" + maxTotalQueryLengthFlag = "query-frontend.max-total-query-length" requestRateFlag = "distributor.request-rate-limit" requestBurstSizeFlag = "distributor.request-burst-size" ingestionRateFlag = "distributor.ingestion-rate-limit" @@ -118,6 +119,10 @@ type Limits struct { QueryShardingTotalShards int `yaml:"query_sharding_total_shards" json:"query_sharding_total_shards"` QueryShardingMaxShardedQueries int `yaml:"query_sharding_max_sharded_queries" json:"query_sharding_max_sharded_queries"` SplitInstantQueriesByInterval model.Duration `yaml:"split_instant_queries_by_interval" json:"split_instant_queries_by_interval" category:"experimental"` + + // Query-frontend limits. + MaxTotalQueryLength model.Duration `yaml:"max_total_query_length,omitempty" json:"max_total_query_length,omitempty" category:"experimental"` + // Cardinality CardinalityAnalysisEnabled bool `yaml:"cardinality_analysis_enabled" json:"cardinality_analysis_enabled"` LabelNamesAndValuesResultsMaxSizeBytes int `yaml:"label_names_and_values_results_max_size_bytes" json:"label_names_and_values_results_max_size_bytes"` @@ -196,7 +201,7 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) { f.IntVar(&l.MaxChunksPerQuery, MaxChunksPerQueryFlag, 2e6, "Maximum number of chunks that can be fetched in a single query from ingesters and long-term storage. This limit is enforced in the querier, ruler and store-gateway. 0 to disable.") f.IntVar(&l.MaxFetchedSeriesPerQuery, MaxSeriesPerQueryFlag, 0, "The maximum number of unique series for which a query can fetch samples from each ingesters and storage. This limit is enforced in the querier and ruler. 0 to disable") f.IntVar(&l.MaxFetchedChunkBytesPerQuery, MaxChunkBytesPerQueryFlag, 0, "The maximum size of all chunks in bytes that a query can fetch from each ingester and storage. This limit is enforced in the querier and ruler. 0 to disable.") - f.Var(&l.MaxQueryLength, maxQueryLengthFlag, "Limit the query time range (end - start time). This limit is enforced in the query-frontend (on the received query), in the querier (on the query possibly split by the query-frontend) and ruler. 0 to disable.") + f.Var(&l.MaxQueryLength, maxQueryLengthFlag, "Limit the query time range (end - start time). This limit is enforced in the querier (on the query possibly split by the query-frontend) and ruler. 0 to disable.") f.Var(&l.MaxQueryLookback, "querier.max-query-lookback", "Limit how long back data (series and metadata) can be queried, up until duration ago. This limit is enforced in the query-frontend, querier and ruler. If the requested time range is outside the allowed range, the request will not fail but will be manipulated to only query data within the allowed time range. 0 to disable.") f.IntVar(&l.MaxQueryParallelism, "querier.max-query-parallelism", 14, "Maximum number of split (by time) or partial (by shard) queries that will be scheduled in parallel by the query-frontend for a single input query. This limit is introduced to have a fairer query scheduling and avoid a single query over a large time range saturating all available queriers.") f.Var(&l.MaxLabelsQueryLength, "store.max-labels-query-length", "Limit the time range (end - start time) of series, label names and values queries. This limit is enforced in the querier. If the requested time range is outside the allowed range, the request will not fail but will be manipulated to only query data within the allowed time range. 0 to disable.") @@ -222,6 +227,9 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) { f.Var(&l.CompactorPartialBlockDeletionDelay, "compactor.partial-block-deletion-delay", fmt.Sprintf("If a partial block (unfinished block without %s file) hasn't been modified for this time, it will be marked for deletion. The minimum accepted value is %s: a lower value will be ignored and the feature disabled. 0 to disable.", block.MetaFilename, MinCompactorPartialBlockDeletionDelay.String())) f.BoolVar(&l.CompactorBlockUploadEnabled, "compactor.block-upload-enabled", false, "Enable block upload API for the tenant.") + // Query-frontend. + f.Var(&l.MaxTotalQueryLength, maxTotalQueryLengthFlag, fmt.Sprintf("Limit the total query time range (end - start time). This limit is enforced in the query-frontend on the received query. Defaults to the value of -%s if set to 0.", maxQueryLengthFlag)) + // Store-gateway. f.IntVar(&l.StoreGatewayTenantShardSize, "store-gateway.tenant-shard-size", 0, "The tenant's shard size, used when store-gateway sharding is enabled. Value of 0 disables shuffle sharding for the tenant, that is all tenant blocks are sharded across all store-gateway replicas.") @@ -458,6 +466,15 @@ func (o *Overrides) MaxQueryLength(userID string) time.Duration { return time.Duration(o.getOverridesForUser(userID).MaxQueryLength) } +// MaxTotalQueryLength returns the limit of the total length (in time) of a query. +func (o *Overrides) MaxTotalQueryLength(userID string) time.Duration { + t := time.Duration(o.getOverridesForUser(userID).MaxTotalQueryLength) + if t == time.Duration(0) { + return o.MaxQueryLength(userID) + } + return t +} + // MaxLabelsQueryLength returns the limit of the length (in time) of a label names or values request. func (o *Overrides) MaxLabelsQueryLength(userID string) time.Duration { return time.Duration(o.getOverridesForUser(userID).MaxLabelsQueryLength) diff --git a/pkg/util/validation/limits_test.go b/pkg/util/validation/limits_test.go index a271d00b19a..c7eb2a5736d 100644 --- a/pkg/util/validation/limits_test.go +++ b/pkg/util/validation/limits_test.go @@ -286,6 +286,67 @@ func TestSmallestPositiveNonZeroDurationPerTenant(t *testing.T) { } } +func TestMaxTotalQueryLengthWithoutDefault(t *testing.T) { + tenantLimits := map[string]*Limits{ + "tenant-a": { + MaxQueryLength: model.Duration(time.Hour), + }, + "tenant-b": { + MaxQueryLength: model.Duration(time.Hour), + MaxTotalQueryLength: model.Duration(4 * time.Hour), + }, + } + defaults := Limits{ + MaxQueryLength: model.Duration(2 * time.Hour), + } + + ov, err := NewOverrides(defaults, newMockTenantLimits(tenantLimits)) + require.NoError(t, err) + + for _, tc := range []struct { + tenantIDs []string + expLimit time.Duration + }{ + {tenantIDs: []string{}, expLimit: time.Duration(0)}, + {tenantIDs: []string{"tenant-a"}, expLimit: time.Hour}, + {tenantIDs: []string{"tenant-b"}, expLimit: 4 * time.Hour}, + {tenantIDs: []string{"tenant-c"}, expLimit: 2 * time.Hour}, + } { + assert.Equal(t, tc.expLimit, SmallestPositiveNonZeroDurationPerTenant(tc.tenantIDs, ov.MaxTotalQueryLength)) + } +} + +func TestMaxTotalQueryLengthWithDefault(t *testing.T) { + tenantLimits := map[string]*Limits{ + "tenant-a": { + MaxQueryLength: model.Duration(time.Hour), + }, + "tenant-b": { + MaxQueryLength: model.Duration(time.Hour), + MaxTotalQueryLength: model.Duration(4 * time.Hour), + }, + } + defaults := Limits{ + MaxQueryLength: model.Duration(2 * time.Hour), + MaxTotalQueryLength: model.Duration(3 * time.Hour), + } + + ov, err := NewOverrides(defaults, newMockTenantLimits(tenantLimits)) + require.NoError(t, err) + + for _, tc := range []struct { + tenantIDs []string + expLimit time.Duration + }{ + {tenantIDs: []string{}, expLimit: time.Duration(0)}, + {tenantIDs: []string{"tenant-a"}, expLimit: time.Hour}, + {tenantIDs: []string{"tenant-b"}, expLimit: 4 * time.Hour}, + {tenantIDs: []string{"tenant-c"}, expLimit: 3 * time.Hour}, + } { + assert.Equal(t, tc.expLimit, SmallestPositiveNonZeroDurationPerTenant(tc.tenantIDs, ov.MaxTotalQueryLength)) + } +} + func TestAlertmanagerNotificationLimits(t *testing.T) { for name, tc := range map[string]struct { inputYAML string