Skip to content

Commit

Permalink
Mimir query engine: report estimated peak memory consumption as a met…
Browse files Browse the repository at this point in the history
…ric and in traces (grafana#8270)

* Add histogram with estimated peak memory consumption of each query

* Include estimated peak memory consumption in traces.

* Add changelog entry
  • Loading branch information
charleskorn authored and narqo committed Jun 6, 2024
1 parent feb0484 commit a8c36f5
Show file tree
Hide file tree
Showing 6 changed files with 149 additions and 56 deletions.
2 changes: 1 addition & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
* [FEATURE] Continuous-test: now runable as a module with `mimir -target=continuous-test`. #7747
* [FEATURE] Store-gateway: Allow specific tenants to be enabled or disabled via `-store-gateway.enabled-tenants` or `-store-gateway.disabled-tenants` CLI flags or their corresponding YAML settings. #7653
* [FEATURE] New `-<prefix>.s3.bucket-lookup-type` flag configures lookup style type, used to access bucket in s3 compatible providers. #7684
* [FEATURE] Querier: add experimental streaming PromQL engine, enabled with `-querier.promql-engine=streaming`. #7693 #7898 #7899 #8023 #8058 #8096 #8121 #8197 #8230 #8247 #8276 #8277
* [FEATURE] Querier: add experimental streaming PromQL engine, enabled with `-querier.promql-engine=streaming`. #7693 #7898 #7899 #8023 #8058 #8096 #8121 #8197 #8230 #8247 #8270 #8276 #8277
* [FEATURE] New `/ingester/unregister-on-shutdown` HTTP endpoint allows dynamic access to ingesters' `-ingester.ring.unregister-on-shutdown` configuration. #7739
* [FEATURE] Server: added experimental [PROXY protocol support](https://www.haproxy.org/download/2.3/doc/proxy-protocol.txt). The PROXY protocol support can be enabled via `-server.proxy-protocol-enabled=true`. When enabled, the support is added both to HTTP and gRPC listening ports. #7698
* [FEATURE] mimirtool: Add `runtime-config verify` sub-command, for verifying Mimir runtime config files. #8123
Expand Down
2 changes: 1 addition & 1 deletion pkg/querier/querier.go
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@ func New(cfg Config, limits *validation.Overrides, distributor Distributor, stor
eng = promql.NewEngine(opts)
case streamingPromQLEngine:
limitsProvider := &tenantQueryLimitsProvider{limits: limits}
streamingEngine, err := streamingpromql.NewEngine(opts, limitsProvider)
streamingEngine, err := streamingpromql.NewEngine(opts, limitsProvider, logger)
if err != nil {
return nil, nil, nil, err
}
Expand Down
6 changes: 3 additions & 3 deletions pkg/streamingpromql/benchmarks/comparison_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ func BenchmarkQuery(b *testing.B) {

opts := streamingpromql.NewTestEngineOpts()
prometheusEngine := promql.NewEngine(opts)
streamingEngine, err := streamingpromql.NewEngine(opts, streamingpromql.NewStaticQueryLimitsProvider(0))
streamingEngine, err := streamingpromql.NewEngine(opts, streamingpromql.NewStaticQueryLimitsProvider(0), log.NewNopLogger())
require.NoError(b, err)

// Important: the names below must remain in sync with the names used in tools/benchmark-query-engine.
Expand Down Expand Up @@ -96,7 +96,7 @@ func TestBothEnginesReturnSameResultsForBenchmarkQueries(t *testing.T) {

opts := streamingpromql.NewTestEngineOpts()
prometheusEngine := promql.NewEngine(opts)
streamingEngine, err := streamingpromql.NewEngine(opts, streamingpromql.NewStaticQueryLimitsProvider(0))
streamingEngine, err := streamingpromql.NewEngine(opts, streamingpromql.NewStaticQueryLimitsProvider(0), log.NewNopLogger())
require.NoError(t, err)

ctx := user.InjectOrgID(context.Background(), UserID)
Expand All @@ -123,7 +123,7 @@ func TestBenchmarkSetup(t *testing.T) {
q := createBenchmarkQueryable(t, []int{1})

opts := streamingpromql.NewTestEngineOpts()
streamingEngine, err := streamingpromql.NewEngine(opts, streamingpromql.NewStaticQueryLimitsProvider(0))
streamingEngine, err := streamingpromql.NewEngine(opts, streamingpromql.NewStaticQueryLimitsProvider(0), log.NewNopLogger())
require.NoError(t, err)

ctx := user.InjectOrgID(context.Background(), UserID)
Expand Down
15 changes: 14 additions & 1 deletion pkg/streamingpromql/engine.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,16 @@ import (
"fmt"
"time"

"github.com/go-kit/log"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/prometheus/prometheus/promql"
"github.com/prometheus/prometheus/storage"
)

const defaultLookbackDelta = 5 * time.Minute // This should be the same value as github.com/prometheus/prometheus/promql.defaultLookbackDelta.

func NewEngine(opts promql.EngineOpts, limitsProvider QueryLimitsProvider) (promql.QueryEngine, error) {
func NewEngine(opts promql.EngineOpts, limitsProvider QueryLimitsProvider, logger log.Logger) (promql.QueryEngine, error) {
lookbackDelta := opts.LookbackDelta
if lookbackDelta == 0 {
lookbackDelta = defaultLookbackDelta
Expand All @@ -40,6 +43,13 @@ func NewEngine(opts promql.EngineOpts, limitsProvider QueryLimitsProvider) (prom
timeout: opts.Timeout,
limitsProvider: limitsProvider,
activeQueryTracker: opts.ActiveQueryTracker,

logger: logger,
estimatedPeakMemoryConsumption: promauto.With(opts.Reg).NewHistogram(prometheus.HistogramOpts{
Name: "cortex_streaming_promql_engine_estimated_query_peak_memory_consumption",
Help: "Estimated peak memory consumption of each query (in bytes)",
NativeHistogramBucketFactor: 1.1,
}),
}, nil
}

Expand All @@ -48,6 +58,9 @@ type Engine struct {
timeout time.Duration
limitsProvider QueryLimitsProvider
activeQueryTracker promql.QueryTracker

logger log.Logger
estimatedPeakMemoryConsumption prometheus.Histogram
}

func (e *Engine) NewInstantQuery(ctx context.Context, q storage.Queryable, opts promql.QueryOpts, qs string, ts time.Time) (promql.Query, error) {
Expand Down
Loading

0 comments on commit a8c36f5

Please sign in to comment.