From 52d520b5f5a9388e538f224a646c5d8daac94ef7 Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Mon, 16 Oct 2023 13:24:59 -0700
Subject: [PATCH 01/49] Separate user queue channel into two - normalQueue and
 highPriorityQueue

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 pkg/frontend/v1/frontend.go             |  5 +++
 pkg/scheduler/queue/queue.go            | 19 ++++++---
 pkg/scheduler/queue/queue_test.go       | 54 +++++++++++++++++++++++--
 pkg/scheduler/queue/user_queues.go      | 28 +++++++++----
 pkg/scheduler/queue/user_queues_test.go | 29 ++++++++-----
 pkg/scheduler/scheduler.go              |  5 +++
 6 files changed, 112 insertions(+), 28 deletions(-)

diff --git a/pkg/frontend/v1/frontend.go b/pkg/frontend/v1/frontend.go
index ac5074dd1c..b36a62162e 100644
--- a/pkg/frontend/v1/frontend.go
+++ b/pkg/frontend/v1/frontend.go
@@ -93,6 +93,11 @@ type request struct {
 	response chan *httpgrpc.HTTPResponse
 }
 
+func (r request) IsHighPriority() bool {
+	//TODO implement me
+	return true
+}
+
 // New creates a new frontend. Frontend implements service, and must be started and stopped.
 func New(cfg Config, limits Limits, log log.Logger, registerer prometheus.Registerer, retry *transport.Retry) (*Frontend, error) {
 	f := &Frontend{
diff --git a/pkg/scheduler/queue/queue.go b/pkg/scheduler/queue/queue.go
index 42b8238b56..93095d58f7 100644
--- a/pkg/scheduler/queue/queue.go
+++ b/pkg/scheduler/queue/queue.go
@@ -44,7 +44,9 @@ func FirstUser() UserIndex {
 }
 
 // Request stored into the queue.
-type Request interface{}
+type Request interface {
+	IsHighPriority() bool
+}
 
 // RequestQueue holds incoming requests in per-user queues. It also assigns each user specified number of queriers,
 // and when querier asks for next request to handle (using GetNextRequestForQuerier), it returns requests
@@ -96,13 +98,21 @@ func (q *RequestQueue) EnqueueRequest(userID string, req Request, maxQueriers fl
 	}
 
 	shardSize := util.DynamicShardSize(maxQueriers, len(q.queues.queriers))
-	queue := q.queues.getOrAddQueue(userID, shardSize)
+	queue := q.queues.getOrAddQueue(userID, shardSize, req.IsHighPriority())
+	maxOutstandingRequests := q.queues.limits.MaxOutstandingPerTenant(userID)
+
 	if queue == nil {
 		// This can only happen if userID is "".
 		return errors.New("no queue found")
 	}
 
 	q.totalRequests.WithLabelValues(userID).Inc()
+
+	if q.queues.getTotalQueueSize(userID) >= maxOutstandingRequests {
+		q.discardedRequests.WithLabelValues(userID).Inc()
+		return ErrTooManyRequests
+	}
+
 	select {
 	case queue <- req:
 		q.queueLength.WithLabelValues(userID).Inc()
@@ -112,9 +122,6 @@ func (q *RequestQueue) EnqueueRequest(userID string, req Request, maxQueriers fl
 			successFn()
 		}
 		return nil
-	default:
-		q.discardedRequests.WithLabelValues(userID).Inc()
-		return ErrTooManyRequests
 	}
 }
 
@@ -152,7 +159,7 @@ FindQueue:
 		// Pick next request from the queue.
 		for {
 			request := <-queue
-			if len(queue) == 0 {
+			if q.queues.getTotalQueueSize(userID) == 0 {
 				q.queues.deleteQueue(userID)
 			}
 
diff --git a/pkg/scheduler/queue/queue_test.go b/pkg/scheduler/queue/queue_test.go
index 8e8f0a94d0..18edb80989 100644
--- a/pkg/scheduler/queue/queue_test.go
+++ b/pkg/scheduler/queue/queue_test.go
@@ -39,7 +39,7 @@ func BenchmarkGetNextRequest(b *testing.B) {
 			for j := 0; j < numTenants; j++ {
 				userID := strconv.Itoa(j)
 
-				err := queue.EnqueueRequest(userID, "request", 0, nil)
+				err := queue.EnqueueRequest(userID, MockRequest{}, 0, nil)
 				if err != nil {
 					b.Fatal(err)
 				}
@@ -79,7 +79,7 @@ func BenchmarkQueueRequest(b *testing.B) {
 
 	queues := make([]*RequestQueue, 0, b.N)
 	users := make([]string, 0, numTenants)
-	requests := make([]string, 0, numTenants)
+	requests := make([]MockRequest, 0, numTenants)
 
 	for n := 0; n < b.N; n++ {
 		q := NewRequestQueue(maxOutstandingPerTenant, 0,
@@ -96,7 +96,7 @@ func BenchmarkQueueRequest(b *testing.B) {
 		queues = append(queues, q)
 
 		for j := 0; j < numTenants; j++ {
-			requests = append(requests, fmt.Sprintf("%d-%d", n, j))
+			requests = append(requests, MockRequest{id: fmt.Sprintf("%d-%d", n, j)})
 			users = append(users, strconv.Itoa(j))
 		}
 	}
@@ -149,7 +149,7 @@ func TestRequestQueue_GetNextRequestForQuerier_ShouldGetRequestAfterReshardingBe
 
 	// Enqueue a request from an user which would be assigned to querier-1.
 	// NOTE: "user-1" hash falls in the querier-1 shard.
-	require.NoError(t, queue.EnqueueRequest("user-1", "request", 1, nil))
+	require.NoError(t, queue.EnqueueRequest("user-1", MockRequest{}, 1, nil))
 
 	startTime := time.Now()
 	querier2wg.Wait()
@@ -158,3 +158,49 @@ func TestRequestQueue_GetNextRequestForQuerier_ShouldGetRequestAfterReshardingBe
 	// We expect that querier-2 got the request only after querier-1 forget delay is passed.
 	assert.GreaterOrEqual(t, waitTime.Milliseconds(), forgetDelay.Milliseconds())
 }
+
+func TestRequestQueue_HighPriority(t *testing.T) {
+	queue := NewRequestQueue(3, 0,
+		prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user"}),
+		prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user"}),
+		MockLimits{MaxOutstanding: 3},
+		nil,
+	)
+	ctx := context.Background()
+	queue.RegisterQuerierConnection("querier-1")
+
+	normalRequest1 := MockRequest{
+		id:             "normal query 1",
+		isHighPriority: false,
+	}
+	normalRequest2 := MockRequest{
+		id:             "normal query 1",
+		isHighPriority: false,
+	}
+	highPriorityRequest := MockRequest{
+		id:             "high priority query",
+		isHighPriority: true,
+	}
+
+	queue.EnqueueRequest("userID", normalRequest1, 1, func() {})
+	queue.EnqueueRequest("userID", normalRequest2, 1, func() {})
+	queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {})
+
+	assert.Error(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {})) // should fail due to maxOutstandingPerTenant = 3
+	assert.Equal(t, 3, queue.queues.getTotalQueueSize("userID"))
+	nextRequest, _, _ := queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1")
+	assert.Equal(t, highPriorityRequest, nextRequest) // high priority request returned, although it was enqueued the last
+	nextRequest, _, _ = queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1")
+	assert.Equal(t, normalRequest1, nextRequest)
+	nextRequest, _, _ = queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1")
+	assert.Equal(t, normalRequest2, nextRequest)
+}
+
+type MockRequest struct {
+	id             string
+	isHighPriority bool
+}
+
+func (r MockRequest) IsHighPriority() bool {
+	return r.isHighPriority
+}
diff --git a/pkg/scheduler/queue/user_queues.go b/pkg/scheduler/queue/user_queues.go
index 0f9cd1a081..9ab34f9820 100644
--- a/pkg/scheduler/queue/user_queues.go
+++ b/pkg/scheduler/queue/user_queues.go
@@ -53,7 +53,8 @@ type queues struct {
 }
 
 type userQueue struct {
-	ch chan Request
+	normalQueue       chan Request
+	highPriorityQueue chan Request
 
 	// If not nil, only these queriers can handle user requests. If nil, all queriers can.
 	// We set this to nil if number of available queriers <= maxQueriers.
@@ -103,7 +104,7 @@ func (q *queues) deleteQueue(userID string) {
 // MaxQueriers is used to compute which queriers should handle requests for this user.
 // If maxQueriers is <= 0, all queriers can handle this user's requests.
 // If maxQueriers has changed since the last call, queriers for this are recomputed.
-func (q *queues) getOrAddQueue(userID string, maxQueriers int) chan Request {
+func (q *queues) getOrAddQueue(userID string, maxQueriers int, isHighPriority bool) chan Request {
 	// Empty user is not allowed, as that would break our users list ("" is used for free spot).
 	if userID == "" {
 		return nil
@@ -123,9 +124,10 @@ func (q *queues) getOrAddQueue(userID string, maxQueriers int) chan Request {
 			queueSize = q.maxUserQueueSize
 		}
 		uq = &userQueue{
-			ch:    make(chan Request, queueSize),
-			seed:  util.ShuffleShardSeed(userID, ""),
-			index: -1,
+			normalQueue:       make(chan Request, queueSize),
+			highPriorityQueue: make(chan Request, queueSize),
+			seed:              util.ShuffleShardSeed(userID, ""),
+			index:             -1,
 		}
 		q.userQueues[userID] = uq
 
@@ -150,7 +152,15 @@ func (q *queues) getOrAddQueue(userID string, maxQueriers int) chan Request {
 		uq.queriers = shuffleQueriersForUser(uq.seed, maxQueriers, q.sortedQueriers, nil)
 	}
 
-	return uq.ch
+	if isHighPriority {
+		return uq.highPriorityQueue
+	}
+
+	return uq.normalQueue
+}
+
+func (q *queues) getTotalQueueSize(userID string) int {
+	return len(q.userQueues[userID].normalQueue) + len(q.userQueues[userID].highPriorityQueue)
 }
 
 // Finds next queue for the querier. To support fair scheduling between users, client is expected
@@ -182,7 +192,11 @@ func (q *queues) getNextQueueForQuerier(lastUserIndex int, querierID string) (ch
 			}
 		}
 
-		return q.ch, u, uid
+		if len(q.highPriorityQueue) > 0 {
+			return q.highPriorityQueue, u, uid
+		}
+
+		return q.normalQueue, u, uid
 	}
 	return nil, "", uid
 }
diff --git a/pkg/scheduler/queue/user_queues_test.go b/pkg/scheduler/queue/user_queues_test.go
index cc986cac41..56f05c6f48 100644
--- a/pkg/scheduler/queue/user_queues_test.go
+++ b/pkg/scheduler/queue/user_queues_test.go
@@ -5,6 +5,7 @@ import (
 	"math"
 	"math/rand"
 	"sort"
+	"sync"
 	"testing"
 	"time"
 
@@ -22,11 +23,11 @@ func TestQueues(t *testing.T) {
 	assert.Equal(t, "", u)
 
 	// Add queues: [one]
-	qOne := getOrAdd(t, uq, "one", 0)
+	qOne := getOrAdd(t, uq, "one", 0, false)
 	lastUserIndex = confirmOrderForQuerier(t, uq, "querier-1", lastUserIndex, qOne, qOne)
 
 	// [one two]
-	qTwo := getOrAdd(t, uq, "two", 0)
+	qTwo := getOrAdd(t, uq, "two", 0, false)
 	assert.NotEqual(t, qOne, qTwo)
 
 	lastUserIndex = confirmOrderForQuerier(t, uq, "querier-1", lastUserIndex, qTwo, qOne, qTwo, qOne)
@@ -34,7 +35,7 @@ func TestQueues(t *testing.T) {
 
 	// [one two three]
 	// confirm fifo by adding a third queue and iterating to it
-	qThree := getOrAdd(t, uq, "three", 0)
+	qThree := getOrAdd(t, uq, "three", 0, false)
 
 	lastUserIndex = confirmOrderForQuerier(t, uq, "querier-1", lastUserIndex, qTwo, qThree, qOne)
 
@@ -45,7 +46,7 @@ func TestQueues(t *testing.T) {
 	lastUserIndex = confirmOrderForQuerier(t, uq, "querier-1", lastUserIndex, qTwo, qThree, qTwo)
 
 	// "four" is added at the beginning of the list: [four two three]
-	qFour := getOrAdd(t, uq, "four", 0)
+	qFour := getOrAdd(t, uq, "four", 0, false)
 
 	lastUserIndex = confirmOrderForQuerier(t, uq, "querier-1", lastUserIndex, qThree, qFour, qTwo, qThree)
 
@@ -92,7 +93,7 @@ func TestQueuesWithQueriers(t *testing.T) {
 	// Add user queues.
 	for u := 0; u < users; u++ {
 		uid := fmt.Sprintf("user-%d", u)
-		getOrAdd(t, uq, uid, maxQueriersPerUser)
+		getOrAdd(t, uq, uid, maxQueriersPerUser, false)
 
 		// Verify it has maxQueriersPerUser queriers assigned now.
 		qs := uq.userQueues[uid].queriers
@@ -156,9 +157,10 @@ func TestQueuesConsistency(t *testing.T) {
 			conns := map[string]int{}
 
 			for i := 0; i < 10000; i++ {
+				queue := uq.getOrAddQueue(generateTenant(r), 3, false)
 				switch r.Int() % 6 {
 				case 0:
-					assert.NotNil(t, uq.getOrAddQueue(generateTenant(r), 3))
+					assert.NotNil(t, queue)
 				case 1:
 					qid := generateQuerier(r)
 					_, _, luid := uq.getNextQueueForQuerier(lastUserIndexes[qid], qid)
@@ -207,7 +209,7 @@ func TestQueues_ForgetDelay(t *testing.T) {
 	// Add user queues.
 	for i := 0; i < numUsers; i++ {
 		userID := fmt.Sprintf("user-%d", i)
-		getOrAdd(t, uq, userID, maxQueriersPerUser)
+		getOrAdd(t, uq, userID, maxQueriersPerUser, false)
 	}
 
 	// We expect querier-1 to have some users.
@@ -299,7 +301,7 @@ func TestQueues_ForgetDelay_ShouldCorrectlyHandleQuerierReconnectingBeforeForget
 	// Add user queues.
 	for i := 0; i < numUsers; i++ {
 		userID := fmt.Sprintf("user-%d", i)
-		getOrAdd(t, uq, userID, maxQueriersPerUser)
+		getOrAdd(t, uq, userID, maxQueriersPerUser, false)
 	}
 
 	// We expect querier-1 to have some users.
@@ -359,11 +361,11 @@ func generateQuerier(r *rand.Rand) string {
 	return fmt.Sprint("querier-", r.Int()%5)
 }
 
-func getOrAdd(t *testing.T, uq *queues, tenant string, maxQueriers int) chan Request {
-	q := uq.getOrAddQueue(tenant, maxQueriers)
+func getOrAdd(t *testing.T, uq *queues, tenant string, maxQueriers int, isHighPriority bool) chan Request {
+	q := uq.getOrAddQueue(tenant, maxQueriers, isHighPriority)
 	assert.NotNil(t, q)
 	assert.NoError(t, isConsistent(uq))
-	assert.Equal(t, q, uq.getOrAddQueue(tenant, maxQueriers))
+	assert.Equal(t, q, uq.getOrAddQueue(tenant, maxQueriers, isHighPriority))
 	return q
 }
 
@@ -437,6 +439,11 @@ func getUsersByQuerier(queues *queues, querierID string) []string {
 	return userIDs
 }
 
+func enqueueRequest(queue chan Request, request Request, wg *sync.WaitGroup) {
+	defer wg.Done()
+	queue <- request
+}
+
 func TestShuffleQueriers(t *testing.T) {
 	allQueriers := []string{"a", "b", "c", "d", "e"}
 
diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go
index f03def5ea3..602bc0d296 100644
--- a/pkg/scheduler/scheduler.go
+++ b/pkg/scheduler/scheduler.go
@@ -165,6 +165,11 @@ type schedulerRequest struct {
 	parentSpanContext opentracing.SpanContext
 }
 
+func (s schedulerRequest) IsHighPriority() bool {
+	//TODO implement me
+	return true
+}
+
 // FrontendLoop handles connection from frontend.
 func (s *Scheduler) FrontendLoop(frontend schedulerpb.SchedulerForFrontend_FrontendLoopServer) error {
 	frontendAddress, frontendCtx, err := s.frontendConnected(frontend)

From 646853a49579c63b3fb2bf3a30cd5b85d6f06981 Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Tue, 17 Oct 2023 10:12:27 -0700
Subject: [PATCH 02/49] Add ReservedHighPriorityQueriers

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 pkg/scheduler/queue/queue.go            |  2 +-
 pkg/scheduler/queue/queue_test.go       | 39 +++++++++++-
 pkg/scheduler/queue/user_queues.go      | 75 +++++++++++++++++-----
 pkg/scheduler/queue/user_queues_test.go | 84 +++++++++++++++++++++----
 pkg/util/validation/limits.go           |  9 ++-
 5 files changed, 179 insertions(+), 30 deletions(-)

diff --git a/pkg/scheduler/queue/queue.go b/pkg/scheduler/queue/queue.go
index 93095d58f7..0597f2f76e 100644
--- a/pkg/scheduler/queue/queue.go
+++ b/pkg/scheduler/queue/queue.go
@@ -152,7 +152,7 @@ FindQueue:
 	for {
 		queue, userID, idx := q.queues.getNextQueueForQuerier(last.last, querierID)
 		last.last = idx
-		if queue == nil {
+		if queue == nil || len(queue) < 1 {
 			break
 		}
 
diff --git a/pkg/scheduler/queue/queue_test.go b/pkg/scheduler/queue/queue_test.go
index 18edb80989..bb5c86a22c 100644
--- a/pkg/scheduler/queue/queue_test.go
+++ b/pkg/scheduler/queue/queue_test.go
@@ -159,7 +159,7 @@ func TestRequestQueue_GetNextRequestForQuerier_ShouldGetRequestAfterReshardingBe
 	assert.GreaterOrEqual(t, waitTime.Milliseconds(), forgetDelay.Milliseconds())
 }
 
-func TestRequestQueue_HighPriority(t *testing.T) {
+func TestRequestQueue_QueriersShouldGetHighPriorityQueryFirst(t *testing.T) {
 	queue := NewRequestQueue(3, 0,
 		prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user"}),
 		prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user"}),
@@ -196,6 +196,43 @@ func TestRequestQueue_HighPriority(t *testing.T) {
 	assert.Equal(t, normalRequest2, nextRequest)
 }
 
+func TestRequestQueue_ReservedQueriersShouldOnlyGetHighPriorityQueries(t *testing.T) {
+	queue := NewRequestQueue(3, 0,
+		prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user"}),
+		prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user"}),
+		MockLimits{MaxOutstanding: 3, ReservedQueriers: 1},
+		nil,
+	)
+	ctx := context.Background()
+
+	queue.RegisterQuerierConnection("querier-1")
+
+	normalRequest := MockRequest{
+		id:             "normal query",
+		isHighPriority: false,
+	}
+	highPriorityRequest := MockRequest{
+		id:             "high priority query",
+		isHighPriority: true,
+	}
+
+	queue.EnqueueRequest("userID", normalRequest, 1, func() {})
+	queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {})
+
+	nextRequest, _, _ := queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1")
+	assert.Equal(t, highPriorityRequest, nextRequest)
+
+	ctxTimeout, cancel := context.WithTimeout(ctx, 1*time.Second)
+	defer cancel()
+
+	time.AfterFunc(2*time.Second, func() {
+		queue.cond.Broadcast()
+	})
+	nextRequest, _, _ = queue.GetNextRequestForQuerier(ctxTimeout, FirstUser(), "querier-1")
+	assert.Nil(t, nextRequest)
+	assert.Equal(t, 1, queue.queues.getTotalQueueSize("userID"))
+}
+
 type MockRequest struct {
 	id             string
 	isHighPriority bool
diff --git a/pkg/scheduler/queue/user_queues.go b/pkg/scheduler/queue/user_queues.go
index 9ab34f9820..fa6ac4296d 100644
--- a/pkg/scheduler/queue/user_queues.go
+++ b/pkg/scheduler/queue/user_queues.go
@@ -1,6 +1,7 @@
 package queue
 
 import (
+	"math"
 	"math/rand"
 	"sort"
 	"time"
@@ -13,6 +14,13 @@ type Limits interface {
 	// MaxOutstandingPerTenant returns the limit to the maximum number
 	// of outstanding requests per tenant per request queue.
 	MaxOutstandingPerTenant(user string) int
+
+	// ReservedHighPriorityQueriers returns the minimum number of queriers dedicated for high priority
+	// queue per tenant. All queriers still handle priority queue first, but this provides extra protection on
+	// high priority queries from slow normal queries exhausting all queriers.
+	// If ReservedHighPriorityQueriers is capped by MaxQueriersPerUser.
+	// If less than 1, it will be applied as a percentage of MaxQueriersPerUser.
+	ReservedHighPriorityQueriers(user string) float64
 }
 
 // querier holds information about a querier registered in the queue.
@@ -58,8 +66,9 @@ type userQueue struct {
 
 	// If not nil, only these queriers can handle user requests. If nil, all queriers can.
 	// We set this to nil if number of available queriers <= maxQueriers.
-	queriers    map[string]struct{}
-	maxQueriers int
+	queriers         map[string]struct{}
+	reservedQueriers map[string]struct{}
+	maxQueriers      int
 
 	// Seed for shuffle sharding of queriers. This seed is based on userID only and is therefore consistent
 	// between different frontends.
@@ -149,7 +158,7 @@ func (q *queues) getOrAddQueue(userID string, maxQueriers int, isHighPriority bo
 
 	if uq.maxQueriers != maxQueriers {
 		uq.maxQueriers = maxQueriers
-		uq.queriers = shuffleQueriersForUser(uq.seed, maxQueriers, q.sortedQueriers, nil)
+		uq.queriers, uq.reservedQueriers = shuffleQueriersForUser(uq.seed, maxQueriers, q.sortedQueriers, q.limits.ReservedHighPriorityQueriers(userID), nil)
 	}
 
 	if isHighPriority {
@@ -183,20 +192,21 @@ func (q *queues) getNextQueueForQuerier(lastUserIndex int, querierID string) (ch
 			continue
 		}
 
-		q := q.userQueues[u]
+		uq := q.userQueues[u]
 
-		if q.queriers != nil {
-			if _, ok := q.queriers[querierID]; !ok {
+		if uq.queriers != nil {
+			if _, ok := uq.queriers[querierID]; !ok {
 				// This querier is not handling the user.
 				continue
 			}
 		}
 
-		if len(q.highPriorityQueue) > 0 {
-			return q.highPriorityQueue, u, uid
+		_, isReserved := uq.reservedQueriers[querierID]
+		if isReserved || len(uq.highPriorityQueue) > 0 {
+			return uq.highPriorityQueue, u, uid
 		}
 
-		return q.normalQueue, u, uid
+		return uq.normalQueue, u, uid
 	}
 	return nil, "", uid
 }
@@ -304,19 +314,26 @@ func (q *queues) recomputeUserQueriers() {
 	scratchpad := make([]string, 0, len(q.sortedQueriers))
 
 	for _, uq := range q.userQueues {
-		uq.queriers = shuffleQueriersForUser(uq.seed, uq.maxQueriers, q.sortedQueriers, scratchpad)
+		userID := q.users[uq.index]
+		uq.queriers, uq.reservedQueriers = shuffleQueriersForUser(uq.seed, uq.maxQueriers, q.sortedQueriers, q.limits.ReservedHighPriorityQueriers(userID), scratchpad)
 	}
 }
 
 // shuffleQueriersForUser returns nil if queriersToSelect is 0 or there are not enough queriers to select from.
 // In that case *all* queriers should be used.
 // Scratchpad is used for shuffling, to avoid new allocations. If nil, new slice is allocated.
-func shuffleQueriersForUser(userSeed int64, queriersToSelect int, allSortedQueriers []string, scratchpad []string) map[string]struct{} {
+func shuffleQueriersForUser(userSeed int64, queriersToSelect int, allSortedQueriers []string, numOfReservedQueriersFloat float64, scratchpad []string) (map[string]struct{}, map[string]struct{}) {
+	numOfReservedQueriers := getNumOfReservedQueriers(queriersToSelect, len(allSortedQueriers), numOfReservedQueriersFloat)
+	reservedQueriers := make(map[string]struct{}, numOfReservedQueriers)
+
 	if queriersToSelect == 0 || len(allSortedQueriers) <= queriersToSelect {
-		return nil
+		for i := 0; i < numOfReservedQueriers; i++ {
+			reservedQueriers[allSortedQueriers[i]] = struct{}{}
+		}
+		return nil, reservedQueriers
 	}
 
-	result := make(map[string]struct{}, queriersToSelect)
+	queriers := make(map[string]struct{}, queriersToSelect)
 	rnd := rand.New(rand.NewSource(userSeed))
 
 	scratchpad = scratchpad[:0]
@@ -325,20 +342,46 @@ func shuffleQueriersForUser(userSeed int64, queriersToSelect int, allSortedQueri
 	last := len(scratchpad) - 1
 	for i := 0; i < queriersToSelect; i++ {
 		r := rnd.Intn(last + 1)
-		result[scratchpad[r]] = struct{}{}
+		queriers[scratchpad[r]] = struct{}{}
+		if i < numOfReservedQueriers {
+			reservedQueriers[scratchpad[r]] = struct{}{}
+		}
 		// move selected item to the end, it won't be selected anymore.
 		scratchpad[r], scratchpad[last] = scratchpad[last], scratchpad[r]
 		last--
 	}
 
-	return result
+	return queriers, reservedQueriers
+}
+
+func getNumOfReservedQueriers(queriersToSelect int, totalNumOfQueriers int, reservedQueriers float64) int {
+	numOfReservedQueriers := int(reservedQueriers)
+
+	if reservedQueriers < 1 && reservedQueriers > 0 {
+		if queriersToSelect == 0 || queriersToSelect > totalNumOfQueriers {
+			queriersToSelect = totalNumOfQueriers
+		}
+
+		numOfReservedQueriers = int(math.Ceil(float64(queriersToSelect) * reservedQueriers))
+	}
+
+	if numOfReservedQueriers > totalNumOfQueriers {
+		return totalNumOfQueriers
+	}
+
+	return numOfReservedQueriers
 }
 
 // MockLimits implements the Limits interface. Used in tests only.
 type MockLimits struct {
-	MaxOutstanding int
+	MaxOutstanding   int
+	ReservedQueriers float64
 }
 
 func (l MockLimits) MaxOutstandingPerTenant(_ string) int {
 	return l.MaxOutstanding
 }
+
+func (l MockLimits) ReservedHighPriorityQueriers(_ string) float64 {
+	return l.ReservedQueriers
+}
diff --git a/pkg/scheduler/queue/user_queues_test.go b/pkg/scheduler/queue/user_queues_test.go
index 56f05c6f48..2d47dd6c48 100644
--- a/pkg/scheduler/queue/user_queues_test.go
+++ b/pkg/scheduler/queue/user_queues_test.go
@@ -5,7 +5,6 @@ import (
 	"math"
 	"math/rand"
 	"sort"
-	"sync"
 	"testing"
 	"time"
 
@@ -439,22 +438,20 @@ func getUsersByQuerier(queues *queues, querierID string) []string {
 	return userIDs
 }
 
-func enqueueRequest(queue chan Request, request Request, wg *sync.WaitGroup) {
-	defer wg.Done()
-	queue <- request
-}
-
 func TestShuffleQueriers(t *testing.T) {
 	allQueriers := []string{"a", "b", "c", "d", "e"}
 
-	require.Nil(t, shuffleQueriersForUser(12345, 10, allQueriers, nil))
-	require.Nil(t, shuffleQueriersForUser(12345, len(allQueriers), allQueriers, nil))
+	queriers, _ := shuffleQueriersForUser(12345, 10, allQueriers, 0, nil)
+	require.Nil(t, queriers)
+
+	queriers, _ = shuffleQueriersForUser(12345, len(allQueriers), allQueriers, 0, nil)
+	require.Nil(t, queriers)
 
-	r1 := shuffleQueriersForUser(12345, 3, allQueriers, nil)
+	r1, _ := shuffleQueriersForUser(12345, 3, allQueriers, 0, nil)
 	require.Equal(t, 3, len(r1))
 
 	// Same input produces same output.
-	r2 := shuffleQueriersForUser(12345, 3, allQueriers, nil)
+	r2, _ := shuffleQueriersForUser(12345, 3, allQueriers, 0, nil)
 	require.Equal(t, 3, len(r2))
 	require.Equal(t, r1, r2)
 }
@@ -476,7 +473,7 @@ func TestShuffleQueriersCorrectness(t *testing.T) {
 			toSelect = 3
 		}
 
-		selected := shuffleQueriersForUser(r.Int63(), toSelect, allSortedQueriers, nil)
+		selected, _ := shuffleQueriersForUser(r.Int63(), toSelect, allSortedQueriers, 0, nil)
 
 		require.Equal(t, toSelect, len(selected))
 
@@ -491,3 +488,68 @@ func TestShuffleQueriersCorrectness(t *testing.T) {
 		}
 	}
 }
+
+func TestShuffleQueriers_WithReservedQueriers(t *testing.T) {
+	allQueriers := []string{"a", "b", "c", "d", "e"}
+
+	queriers, reservedQueriers := shuffleQueriersForUser(12345, 0, allQueriers, 0, nil)
+	require.Nil(t, queriers)
+	require.Equal(t, 0, len(reservedQueriers))
+
+	queriers, reservedQueriers = shuffleQueriersForUser(12345, 0, allQueriers, 0.5, nil)
+	require.Nil(t, queriers)
+	require.Equal(t, 3, len(reservedQueriers))
+
+	queriers, reservedQueriers = shuffleQueriersForUser(12345, 0, allQueriers, 1, nil)
+	require.Nil(t, queriers)
+	require.Equal(t, 1, len(reservedQueriers))
+
+	queriers, reservedQueriers = shuffleQueriersForUser(12345, 0, allQueriers, 100, nil)
+	require.Nil(t, queriers)
+	require.Equal(t, 5, len(reservedQueriers))
+
+	queriers, reservedQueriers = shuffleQueriersForUser(12345, 3, allQueriers, 0, nil)
+	require.Equal(t, 3, len(queriers))
+	require.Equal(t, 0, len(reservedQueriers))
+
+	queriers, reservedQueriers = shuffleQueriersForUser(12345, 3, allQueriers, 0.5, nil)
+	require.Equal(t, 3, len(queriers))
+	require.Equal(t, 2, len(reservedQueriers))
+
+	queriers, reservedQueriers = shuffleQueriersForUser(12345, 3, allQueriers, 1, nil)
+	require.Equal(t, 3, len(queriers))
+	require.Equal(t, 1, len(reservedQueriers))
+
+	queriers, reservedQueriers = shuffleQueriersForUser(12345, 3, allQueriers, 100, nil)
+	require.Equal(t, 3, len(queriers))
+	require.Equal(t, 3, len(reservedQueriers))
+
+	queriers, reservedQueriers = shuffleQueriersForUser(12345, 100, allQueriers, 0, nil)
+	require.Nil(t, queriers)
+	require.Equal(t, 0, len(reservedQueriers))
+
+	queriers, reservedQueriers = shuffleQueriersForUser(12345, 100, allQueriers, 0.5, nil)
+	require.Nil(t, queriers)
+	require.Equal(t, 3, len(reservedQueriers))
+
+	queriers, reservedQueriers = shuffleQueriersForUser(12345, 100, allQueriers, 1, nil)
+	require.Nil(t, queriers)
+	require.Equal(t, 1, len(reservedQueriers))
+
+	queriers, reservedQueriers = shuffleQueriersForUser(12345, 100, allQueriers, 100, nil)
+	require.Nil(t, queriers)
+	require.Equal(t, 5, len(reservedQueriers))
+}
+
+func TestShuffleQueriers_WithReservedQueriers_Correctness(t *testing.T) {
+	allQueriers := []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n"}
+
+	prevQueriers, prevReservedQueriers := shuffleQueriersForUser(12345, 10, allQueriers, 5, nil)
+	for i := 0; i < 100; i++ {
+		queriers, reservedQueriers := shuffleQueriersForUser(12345, 10, allQueriers, 5, nil)
+		require.Equal(t, prevQueriers, queriers)
+		require.Equal(t, prevReservedQueriers, reservedQueriers)
+		prevQueriers = queriers
+		prevReservedQueriers = reservedQueriers
+	}
+}
diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go
index d60b2ca8e6..20e89f15df 100644
--- a/pkg/util/validation/limits.go
+++ b/pkg/util/validation/limits.go
@@ -101,7 +101,8 @@ type Limits struct {
 	QueryVerticalShardSize       int            `yaml:"query_vertical_shard_size" json:"query_vertical_shard_size" doc:"hidden"`
 
 	// Query Frontend / Scheduler enforced limits.
-	MaxOutstandingPerTenant int `yaml:"max_outstanding_requests_per_tenant" json:"max_outstanding_requests_per_tenant"`
+	MaxOutstandingPerTenant      int     `yaml:"max_outstanding_requests_per_tenant" json:"max_outstanding_requests_per_tenant"`
+	ReservedHighPriorityQueriers float64 `yaml:"reserved_high_priority_queriers" json:"reserved_high_priority_queriers"`
 
 	// Ruler defaults and limits.
 	RulerEvaluationDelay        model.Duration `yaml:"ruler_evaluation_delay_duration" json:"ruler_evaluation_delay_duration"`
@@ -188,6 +189,7 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) {
 	f.IntVar(&l.QueryVerticalShardSize, "frontend.query-vertical-shard-size", 0, "[Experimental] Number of shards to use when distributing shardable PromQL queries.")
 
 	f.IntVar(&l.MaxOutstandingPerTenant, "frontend.max-outstanding-requests-per-tenant", 100, "Maximum number of outstanding requests per tenant per request queue (either query frontend or query scheduler); requests beyond this error with HTTP 429.")
+	f.Float64Var(&l.ReservedHighPriorityQueriers, "frontend.reserved-high-priority-queriers", 0, "Number of reserved queriers to only handle high priority queue (either query frontend or query scheduler).")
 
 	f.Var(&l.RulerEvaluationDelay, "ruler.evaluation-delay-duration", "Duration to delay the evaluation of rules to ensure the underlying metrics have been pushed to Cortex.")
 	f.IntVar(&l.RulerTenantShardSize, "ruler.tenant-shard-size", 0, "The default tenant's shard size when the shuffle-sharding strategy is used by ruler. When this setting is specified in the per-tenant overrides, a value of 0 disables shuffle sharding for the tenant.")
@@ -492,6 +494,11 @@ func (o *Overrides) MaxOutstandingPerTenant(userID string) int {
 	return o.GetOverridesForUser(userID).MaxOutstandingPerTenant
 }
 
+// ReservedHighPriorityQueriers returns the number of reserved queriers that only handle high priority queue for the tenant.
+func (o *Overrides) ReservedHighPriorityQueriers(userID string) float64 {
+	return o.GetOverridesForUser(userID).ReservedHighPriorityQueriers
+}
+
 // EnforceMetricName whether to enforce the presence of a metric name.
 func (o *Overrides) EnforceMetricName(userID string) bool {
 	return o.GetOverridesForUser(userID).EnforceMetricName

From 39639c402fd1703c3d396f68acfebb3a4b5e4d0c Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Tue, 17 Oct 2023 10:16:42 -0700
Subject: [PATCH 03/49] Change default priority for all requests to low

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 pkg/frontend/v1/frontend.go | 2 +-
 pkg/scheduler/scheduler.go  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkg/frontend/v1/frontend.go b/pkg/frontend/v1/frontend.go
index b36a62162e..99e5d8eff1 100644
--- a/pkg/frontend/v1/frontend.go
+++ b/pkg/frontend/v1/frontend.go
@@ -95,7 +95,7 @@ type request struct {
 
 func (r request) IsHighPriority() bool {
 	//TODO implement me
-	return true
+	return false
 }
 
 // New creates a new frontend. Frontend implements service, and must be started and stopped.
diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go
index 602bc0d296..42a7b7413a 100644
--- a/pkg/scheduler/scheduler.go
+++ b/pkg/scheduler/scheduler.go
@@ -167,7 +167,7 @@ type schedulerRequest struct {
 
 func (s schedulerRequest) IsHighPriority() bool {
 	//TODO implement me
-	return true
+	return false
 }
 
 // FrontendLoop handles connection from frontend.

From 38945ac1deedfed55e93713ba046be13f812f221 Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Tue, 17 Oct 2023 11:23:17 -0700
Subject: [PATCH 04/49] Update config description

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 docs/configuration/config-file-reference.md | 6 ++++++
 pkg/util/validation/limits.go               | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md
index c2c07da5e4..3defa44f4c 100644
--- a/docs/configuration/config-file-reference.md
+++ b/docs/configuration/config-file-reference.md
@@ -3041,6 +3041,12 @@ The `limits_config` configures default and per-tenant limits imposed by Cortex s
 # CLI flag: -frontend.max-outstanding-requests-per-tenant
 [max_outstanding_requests_per_tenant: <int> | default = 100]
 
+# Number of reserved queriers to only handle high priority queue (either query
+# frontend or query scheduler). If the value is between 0 and 1, it will be used
+# as a percentage of per-tenant queriers.
+# CLI flag: -frontend.reserved-high-priority-queriers
+[reserved_high_priority_queriers: <float> | default = 0]
+
 # Duration to delay the evaluation of rules to ensure the underlying metrics
 # have been pushed to Cortex.
 # CLI flag: -ruler.evaluation-delay-duration
diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go
index 20e89f15df..94db111b77 100644
--- a/pkg/util/validation/limits.go
+++ b/pkg/util/validation/limits.go
@@ -189,7 +189,7 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) {
 	f.IntVar(&l.QueryVerticalShardSize, "frontend.query-vertical-shard-size", 0, "[Experimental] Number of shards to use when distributing shardable PromQL queries.")
 
 	f.IntVar(&l.MaxOutstandingPerTenant, "frontend.max-outstanding-requests-per-tenant", 100, "Maximum number of outstanding requests per tenant per request queue (either query frontend or query scheduler); requests beyond this error with HTTP 429.")
-	f.Float64Var(&l.ReservedHighPriorityQueriers, "frontend.reserved-high-priority-queriers", 0, "Number of reserved queriers to only handle high priority queue (either query frontend or query scheduler).")
+	f.Float64Var(&l.ReservedHighPriorityQueriers, "frontend.reserved-high-priority-queriers", 0, "Number of reserved queriers to only handle high priority queue (either query frontend or query scheduler). If the value is between 0 and 1, it will be used as a percentage of per-tenant queriers.")
 
 	f.Var(&l.RulerEvaluationDelay, "ruler.evaluation-delay-duration", "Duration to delay the evaluation of rules to ensure the underlying metrics have been pushed to Cortex.")
 	f.IntVar(&l.RulerTenantShardSize, "ruler.tenant-shard-size", 0, "The default tenant's shard size when the shuffle-sharding strategy is used by ruler. When this setting is specified in the per-tenant overrides, a value of 0 disables shuffle sharding for the tenant.")

From d0f7e741d923eb3805e3973a923729ac5e524b99 Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Tue, 17 Oct 2023 11:27:33 -0700
Subject: [PATCH 05/49] Lint

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 pkg/scheduler/queue/queue.go      | 16 +++++++---------
 pkg/scheduler/queue/queue_test.go | 10 +++++-----
 2 files changed, 12 insertions(+), 14 deletions(-)

diff --git a/pkg/scheduler/queue/queue.go b/pkg/scheduler/queue/queue.go
index 0597f2f76e..0931b25f3b 100644
--- a/pkg/scheduler/queue/queue.go
+++ b/pkg/scheduler/queue/queue.go
@@ -113,16 +113,14 @@ func (q *RequestQueue) EnqueueRequest(userID string, req Request, maxQueriers fl
 		return ErrTooManyRequests
 	}
 
-	select {
-	case queue <- req:
-		q.queueLength.WithLabelValues(userID).Inc()
-		q.cond.Broadcast()
-		// Call this function while holding a lock. This guarantees that no querier can fetch the request before function returns.
-		if successFn != nil {
-			successFn()
-		}
-		return nil
+	queue <- req
+	q.queueLength.WithLabelValues(userID).Inc()
+	q.cond.Broadcast()
+	// Call this function while holding a lock. This guarantees that no querier can fetch the request before function returns.
+	if successFn != nil {
+		successFn()
 	}
+	return nil
 }
 
 // GetNextRequestForQuerier find next user queue and takes the next request off of it. Will block if there are no requests.
diff --git a/pkg/scheduler/queue/queue_test.go b/pkg/scheduler/queue/queue_test.go
index bb5c86a22c..426af2d520 100644
--- a/pkg/scheduler/queue/queue_test.go
+++ b/pkg/scheduler/queue/queue_test.go
@@ -182,9 +182,9 @@ func TestRequestQueue_QueriersShouldGetHighPriorityQueryFirst(t *testing.T) {
 		isHighPriority: true,
 	}
 
-	queue.EnqueueRequest("userID", normalRequest1, 1, func() {})
-	queue.EnqueueRequest("userID", normalRequest2, 1, func() {})
-	queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {})
+	assert.NotNil(t, queue.EnqueueRequest("userID", normalRequest1, 1, func() {}))
+	assert.NotNil(t, queue.EnqueueRequest("userID", normalRequest2, 1, func() {}))
+	assert.NotNil(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {}))
 
 	assert.Error(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {})) // should fail due to maxOutstandingPerTenant = 3
 	assert.Equal(t, 3, queue.queues.getTotalQueueSize("userID"))
@@ -216,8 +216,8 @@ func TestRequestQueue_ReservedQueriersShouldOnlyGetHighPriorityQueries(t *testin
 		isHighPriority: true,
 	}
 
-	queue.EnqueueRequest("userID", normalRequest, 1, func() {})
-	queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {})
+	assert.NotNil(t, queue.EnqueueRequest("userID", normalRequest, 1, func() {}))
+	assert.NotNil(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {}))
 
 	nextRequest, _, _ := queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1")
 	assert.Equal(t, highPriorityRequest, nextRequest)

From fec4a902fed2d8a193f4d9b124a5c3e13096c3bb Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Tue, 17 Oct 2023 13:10:58 -0700
Subject: [PATCH 06/49] Fix test

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 pkg/scheduler/queue/queue.go      |  2 ++
 pkg/scheduler/queue/queue_test.go | 10 +++++-----
 pkg/scheduler/scheduler_test.go   |  2 +-
 3 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/pkg/scheduler/queue/queue.go b/pkg/scheduler/queue/queue.go
index 0931b25f3b..27cc6f540c 100644
--- a/pkg/scheduler/queue/queue.go
+++ b/pkg/scheduler/queue/queue.go
@@ -2,6 +2,7 @@ package queue
 
 import (
 	"context"
+	"fmt"
 	"sync"
 	"time"
 
@@ -108,6 +109,7 @@ func (q *RequestQueue) EnqueueRequest(userID string, req Request, maxQueriers fl
 
 	q.totalRequests.WithLabelValues(userID).Inc()
 
+	fmt.Println("EnqueueRequest", q.queues.getTotalQueueSize(userID), maxOutstandingRequests)
 	if q.queues.getTotalQueueSize(userID) >= maxOutstandingRequests {
 		q.discardedRequests.WithLabelValues(userID).Inc()
 		return ErrTooManyRequests
diff --git a/pkg/scheduler/queue/queue_test.go b/pkg/scheduler/queue/queue_test.go
index 426af2d520..fca29e24df 100644
--- a/pkg/scheduler/queue/queue_test.go
+++ b/pkg/scheduler/queue/queue_test.go
@@ -160,7 +160,7 @@ func TestRequestQueue_GetNextRequestForQuerier_ShouldGetRequestAfterReshardingBe
 }
 
 func TestRequestQueue_QueriersShouldGetHighPriorityQueryFirst(t *testing.T) {
-	queue := NewRequestQueue(3, 0,
+	queue := NewRequestQueue(100, 0,
 		prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user"}),
 		prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user"}),
 		MockLimits{MaxOutstanding: 3},
@@ -182,9 +182,9 @@ func TestRequestQueue_QueriersShouldGetHighPriorityQueryFirst(t *testing.T) {
 		isHighPriority: true,
 	}
 
-	assert.NotNil(t, queue.EnqueueRequest("userID", normalRequest1, 1, func() {}))
-	assert.NotNil(t, queue.EnqueueRequest("userID", normalRequest2, 1, func() {}))
-	assert.NotNil(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {}))
+	assert.Nil(t, queue.EnqueueRequest("userID", normalRequest1, 1, func() {}))
+	assert.Nil(t, queue.EnqueueRequest("userID", normalRequest2, 1, func() {}))
+	assert.Nil(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {}))
 
 	assert.Error(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {})) // should fail due to maxOutstandingPerTenant = 3
 	assert.Equal(t, 3, queue.queues.getTotalQueueSize("userID"))
@@ -197,7 +197,7 @@ func TestRequestQueue_QueriersShouldGetHighPriorityQueryFirst(t *testing.T) {
 }
 
 func TestRequestQueue_ReservedQueriersShouldOnlyGetHighPriorityQueries(t *testing.T) {
-	queue := NewRequestQueue(3, 0,
+	queue := NewRequestQueue(100, 0,
 		prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user"}),
 		prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user"}),
 		MockLimits{MaxOutstanding: 3, ReservedQueriers: 1},
diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go
index b2ef66cc65..c309ff195e 100644
--- a/pkg/scheduler/scheduler_test.go
+++ b/pkg/scheduler/scheduler_test.go
@@ -37,7 +37,7 @@ func setupScheduler(t *testing.T, reg prometheus.Registerer) (*Scheduler, schedu
 	flagext.DefaultValues(&cfg)
 	cfg.MaxOutstandingPerTenant = testMaxOutstandingPerTenant
 
-	s, err := NewScheduler(cfg, frontendv1.MockLimits{Queriers: 2, MockLimits: queue.MockLimits{MaxOutstanding: 100}}, log.NewNopLogger(), reg)
+	s, err := NewScheduler(cfg, frontendv1.MockLimits{Queriers: 2, MockLimits: queue.MockLimits{MaxOutstanding: testMaxOutstandingPerTenant}}, log.NewNopLogger(), reg)
 	require.NoError(t, err)
 
 	server := grpc.NewServer()

From ad046d0b6f623b6fcf860158398a912d5b80ba2f Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Tue, 17 Oct 2023 13:40:36 -0700
Subject: [PATCH 07/49] More test fix

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 pkg/scheduler/queue/queue.go      |  2 --
 pkg/scheduler/queue/queue_test.go | 14 +++++++-------
 2 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/pkg/scheduler/queue/queue.go b/pkg/scheduler/queue/queue.go
index 27cc6f540c..0931b25f3b 100644
--- a/pkg/scheduler/queue/queue.go
+++ b/pkg/scheduler/queue/queue.go
@@ -2,7 +2,6 @@ package queue
 
 import (
 	"context"
-	"fmt"
 	"sync"
 	"time"
 
@@ -109,7 +108,6 @@ func (q *RequestQueue) EnqueueRequest(userID string, req Request, maxQueriers fl
 
 	q.totalRequests.WithLabelValues(userID).Inc()
 
-	fmt.Println("EnqueueRequest", q.queues.getTotalQueueSize(userID), maxOutstandingRequests)
 	if q.queues.getTotalQueueSize(userID) >= maxOutstandingRequests {
 		q.discardedRequests.WithLabelValues(userID).Inc()
 		return ErrTooManyRequests
diff --git a/pkg/scheduler/queue/queue_test.go b/pkg/scheduler/queue/queue_test.go
index fca29e24df..5e260323ec 100644
--- a/pkg/scheduler/queue/queue_test.go
+++ b/pkg/scheduler/queue/queue_test.go
@@ -160,7 +160,7 @@ func TestRequestQueue_GetNextRequestForQuerier_ShouldGetRequestAfterReshardingBe
 }
 
 func TestRequestQueue_QueriersShouldGetHighPriorityQueryFirst(t *testing.T) {
-	queue := NewRequestQueue(100, 0,
+	queue := NewRequestQueue(0, 0,
 		prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user"}),
 		prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user"}),
 		MockLimits{MaxOutstanding: 3},
@@ -182,9 +182,9 @@ func TestRequestQueue_QueriersShouldGetHighPriorityQueryFirst(t *testing.T) {
 		isHighPriority: true,
 	}
 
-	assert.Nil(t, queue.EnqueueRequest("userID", normalRequest1, 1, func() {}))
-	assert.Nil(t, queue.EnqueueRequest("userID", normalRequest2, 1, func() {}))
-	assert.Nil(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {}))
+	assert.NoError(t, queue.EnqueueRequest("userID", normalRequest1, 1, func() {}))
+	assert.NoError(t, queue.EnqueueRequest("userID", normalRequest2, 1, func() {}))
+	assert.NoError(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {}))
 
 	assert.Error(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {})) // should fail due to maxOutstandingPerTenant = 3
 	assert.Equal(t, 3, queue.queues.getTotalQueueSize("userID"))
@@ -197,7 +197,7 @@ func TestRequestQueue_QueriersShouldGetHighPriorityQueryFirst(t *testing.T) {
 }
 
 func TestRequestQueue_ReservedQueriersShouldOnlyGetHighPriorityQueries(t *testing.T) {
-	queue := NewRequestQueue(100, 0,
+	queue := NewRequestQueue(0, 0,
 		prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user"}),
 		prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user"}),
 		MockLimits{MaxOutstanding: 3, ReservedQueriers: 1},
@@ -216,8 +216,8 @@ func TestRequestQueue_ReservedQueriersShouldOnlyGetHighPriorityQueries(t *testin
 		isHighPriority: true,
 	}
 
-	assert.NotNil(t, queue.EnqueueRequest("userID", normalRequest, 1, func() {}))
-	assert.NotNil(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {}))
+	assert.NoError(t, queue.EnqueueRequest("userID", normalRequest, 1, func() {}))
+	assert.NoError(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {}))
 
 	nextRequest, _, _ := queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1")
 	assert.Equal(t, highPriorityRequest, nextRequest)

From a88ad90c78372dc35fb604b9b6819243d1a88453 Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Tue, 17 Oct 2023 15:15:12 -0700
Subject: [PATCH 08/49] Add HighPriorityQueries confing + placeholder for
 IsHighPriorityQuery func

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 docs/configuration/config-file-reference.md  |  19 +++
 pkg/frontend/v1/frontend.go                  |  11 +-
 pkg/frontend/v2/frontend.go                  |  19 +--
 pkg/frontend/v2/frontend_scheduler_worker.go |   1 +
 pkg/scheduler/scheduler.go                   |   5 +-
 pkg/scheduler/schedulerpb/scheduler.pb.go    | 139 +++++++++++++------
 pkg/scheduler/schedulerpb/scheduler.proto    |   1 +
 pkg/util/query_priority.go                   |   9 ++
 pkg/util/validation/limits.go                |  16 ++-
 tools/doc-generator/parser.go                |  10 +-
 10 files changed, 166 insertions(+), 64 deletions(-)
 create mode 100644 pkg/util/query_priority.go

diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md
index 3defa44f4c..78adee77b4 100644
--- a/docs/configuration/config-file-reference.md
+++ b/docs/configuration/config-file-reference.md
@@ -3047,6 +3047,9 @@ The `limits_config` configures default and per-tenant limits imposed by Cortex s
 # CLI flag: -frontend.reserved-high-priority-queriers
 [reserved_high_priority_queriers: <float> | default = 0]
 
+# List of query definitions to be handled with a high priority.
+[high_priority_queries: <list of HighPriorityQuery> | default = []]
+
 # Duration to delay the evaluation of rules to ensure the underlying metrics
 # have been pushed to Cortex.
 # CLI flag: -ruler.evaluation-delay-duration
@@ -5038,6 +5041,22 @@ otel:
     [tls_insecure_skip_verify: <boolean> | default = false]
 ```
 
+### `HighPriorityQuery`
+
+```yaml
+# Query string regex. If evaluated true (on top of meeting all other criteria),
+# query is handled with a high priority.
+[regex: <string> | default = ""]
+
+# If query range falls between the start_time and end_time (on top of meeting
+# all other criteria), query is handled with a high priority.
+[start_time: <duration> | default = 1h]
+
+# If query range falls between the start_time and end_time (on top of meeting
+# all other criteria), query is handled with a high priority.
+[end_time: <duration> | default = 0s]
+```
+
 ### `DisabledRuleGroup`
 
 ```yaml
diff --git a/pkg/frontend/v1/frontend.go b/pkg/frontend/v1/frontend.go
index 99e5d8eff1..5a0ad4e52e 100644
--- a/pkg/frontend/v1/frontend.go
+++ b/pkg/frontend/v1/frontend.go
@@ -88,14 +88,14 @@ type request struct {
 	queueSpan   opentracing.Span
 	originalCtx context.Context
 
-	request  *httpgrpc.HTTPRequest
-	err      chan error
-	response chan *httpgrpc.HTTPResponse
+	request        *httpgrpc.HTTPRequest
+	err            chan error
+	response       chan *httpgrpc.HTTPResponse
+	isHighPriority bool
 }
 
 func (r request) IsHighPriority() bool {
-	//TODO implement me
-	return false
+	return r.isHighPriority
 }
 
 // New creates a new frontend. Frontend implements service, and must be started and stopped.
@@ -346,6 +346,7 @@ func (f *Frontend) queueRequest(ctx context.Context, req *request) error {
 	now := time.Now()
 	req.enqueueTime = now
 	req.queueSpan, _ = opentracing.StartSpanFromContext(ctx, "queued")
+	req.isHighPriority = util.IsHighPriorityQuery()
 
 	// aggregate the max queriers limit in the case of a multi tenant query
 	maxQueriers := validation.SmallestPositiveNonZeroFloat64PerTenant(tenantIDs, f.limits.MaxQueriersPerUser)
diff --git a/pkg/frontend/v2/frontend.go b/pkg/frontend/v2/frontend.go
index dea15faeaf..dfed325fc4 100644
--- a/pkg/frontend/v2/frontend.go
+++ b/pkg/frontend/v2/frontend.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"github.com/cortexproject/cortex/pkg/util"
 	"math/rand"
 	"net/http"
 	"sync"
@@ -82,10 +83,11 @@ type Frontend struct {
 }
 
 type frontendRequest struct {
-	queryID      uint64
-	request      *httpgrpc.HTTPRequest
-	userID       string
-	statsEnabled bool
+	queryID        uint64
+	request        *httpgrpc.HTTPRequest
+	userID         string
+	statsEnabled   bool
+	isHighPriority bool
 
 	cancel context.CancelFunc
 
@@ -190,10 +192,11 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest)
 
 	return f.retry.Do(ctx, func() (*httpgrpc.HTTPResponse, error) {
 		freq := &frontendRequest{
-			queryID:      f.lastQueryID.Inc(),
-			request:      req,
-			userID:       userID,
-			statsEnabled: stats.IsEnabled(ctx),
+			queryID:        f.lastQueryID.Inc(),
+			request:        req,
+			userID:         userID,
+			statsEnabled:   stats.IsEnabled(ctx),
+			isHighPriority: util.IsHighPriorityQuery(),
 
 			cancel: cancel,
 
diff --git a/pkg/frontend/v2/frontend_scheduler_worker.go b/pkg/frontend/v2/frontend_scheduler_worker.go
index bbe1e2ed74..5e5b22eb13 100644
--- a/pkg/frontend/v2/frontend_scheduler_worker.go
+++ b/pkg/frontend/v2/frontend_scheduler_worker.go
@@ -263,6 +263,7 @@ func (w *frontendSchedulerWorker) schedulerLoop(loop schedulerpb.SchedulerForFro
 				HttpRequest:     req.request,
 				FrontendAddress: w.frontendAddr,
 				StatsEnabled:    req.statsEnabled,
+				IsHighPriority:  req.isHighPriority,
 			})
 
 			if err != nil {
diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go
index 42a7b7413a..12b5575197 100644
--- a/pkg/scheduler/scheduler.go
+++ b/pkg/scheduler/scheduler.go
@@ -154,6 +154,7 @@ type schedulerRequest struct {
 	queryID         uint64
 	request         *httpgrpc.HTTPRequest
 	statsEnabled    bool
+	isHighPriority  bool
 
 	enqueueTime time.Time
 
@@ -166,8 +167,7 @@ type schedulerRequest struct {
 }
 
 func (s schedulerRequest) IsHighPriority() bool {
-	//TODO implement me
-	return false
+	return s.isHighPriority
 }
 
 // FrontendLoop handles connection from frontend.
@@ -298,6 +298,7 @@ func (s *Scheduler) enqueueRequest(frontendContext context.Context, frontendAddr
 		queryID:         msg.QueryID,
 		request:         msg.HttpRequest,
 		statsEnabled:    msg.StatsEnabled,
+		isHighPriority:  msg.IsHighPriority,
 	}
 
 	now := time.Now()
diff --git a/pkg/scheduler/schedulerpb/scheduler.pb.go b/pkg/scheduler/schedulerpb/scheduler.pb.go
index d3288f95b3..0d5c95d68e 100644
--- a/pkg/scheduler/schedulerpb/scheduler.pb.go
+++ b/pkg/scheduler/schedulerpb/scheduler.pb.go
@@ -216,9 +216,10 @@ type FrontendToScheduler struct {
 	// Each frontend manages its own queryIDs. Different frontends may use same set of query IDs.
 	QueryID uint64 `protobuf:"varint,3,opt,name=queryID,proto3" json:"queryID,omitempty"`
 	// Following are used by ENQUEUE only.
-	UserID       string                `protobuf:"bytes,4,opt,name=userID,proto3" json:"userID,omitempty"`
-	HttpRequest  *httpgrpc.HTTPRequest `protobuf:"bytes,5,opt,name=httpRequest,proto3" json:"httpRequest,omitempty"`
-	StatsEnabled bool                  `protobuf:"varint,6,opt,name=statsEnabled,proto3" json:"statsEnabled,omitempty"`
+	UserID         string                `protobuf:"bytes,4,opt,name=userID,proto3" json:"userID,omitempty"`
+	HttpRequest    *httpgrpc.HTTPRequest `protobuf:"bytes,5,opt,name=httpRequest,proto3" json:"httpRequest,omitempty"`
+	StatsEnabled   bool                  `protobuf:"varint,6,opt,name=statsEnabled,proto3" json:"statsEnabled,omitempty"`
+	IsHighPriority bool                  `protobuf:"varint,7,opt,name=isHighPriority,proto3" json:"isHighPriority,omitempty"`
 }
 
 func (m *FrontendToScheduler) Reset()      { *m = FrontendToScheduler{} }
@@ -295,6 +296,13 @@ func (m *FrontendToScheduler) GetStatsEnabled() bool {
 	return false
 }
 
+func (m *FrontendToScheduler) GetIsHighPriority() bool {
+	if m != nil {
+		return m.IsHighPriority
+	}
+	return false
+}
+
 type SchedulerToFrontend struct {
 	Status SchedulerToFrontendStatus `protobuf:"varint,1,opt,name=status,proto3,enum=schedulerpb.SchedulerToFrontendStatus" json:"status,omitempty"`
 	Error  string                    `protobuf:"bytes,2,opt,name=error,proto3" json:"error,omitempty"`
@@ -438,48 +446,49 @@ func init() {
 func init() { proto.RegisterFile("scheduler.proto", fileDescriptor_2b3fc28395a6d9c5) }
 
 var fileDescriptor_2b3fc28395a6d9c5 = []byte{
-	// 644 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0x4f, 0x4f, 0xdb, 0x4e,
-	0x10, 0xf5, 0x86, 0x24, 0xc0, 0x84, 0xdf, 0x0f, 0x77, 0x81, 0x36, 0x8d, 0xe8, 0x12, 0x45, 0x55,
-	0x95, 0x72, 0x48, 0xaa, 0xb4, 0x52, 0x7b, 0x40, 0x95, 0x52, 0x30, 0x25, 0x2a, 0x75, 0x60, 0xb3,
-	0x51, 0xff, 0x5c, 0x22, 0x92, 0x2c, 0x09, 0x02, 0xbc, 0x66, 0x6d, 0x17, 0xe5, 0xd6, 0x63, 0x8f,
-	0xfd, 0x18, 0xfd, 0x28, 0xbd, 0x54, 0xe2, 0xc8, 0xa1, 0x87, 0x62, 0x2e, 0x3d, 0xf2, 0x11, 0xaa,
-	0x38, 0x76, 0xea, 0xa4, 0x0e, 0x70, 0x9b, 0x1d, 0xbf, 0xe7, 0x9d, 0xf7, 0x66, 0x66, 0x61, 0xde,
-	0x6a, 0x75, 0x79, 0xdb, 0x39, 0xe2, 0xb2, 0x60, 0x4a, 0x61, 0x0b, 0x9c, 0x1a, 0x26, 0xcc, 0x66,
-	0x66, 0xb1, 0x23, 0x3a, 0xc2, 0xcb, 0x17, 0xfb, 0xd1, 0x00, 0x92, 0x79, 0xd6, 0x39, 0xb0, 0xbb,
-	0x4e, 0xb3, 0xd0, 0x12, 0xc7, 0xc5, 0x53, 0xbe, 0xf7, 0x89, 0x9f, 0x0a, 0x79, 0x68, 0x15, 0x5b,
-	0xe2, 0xf8, 0x58, 0x18, 0xc5, 0xae, 0x6d, 0x9b, 0x1d, 0x69, 0xb6, 0x86, 0xc1, 0x80, 0x95, 0x2b,
-	0x01, 0xde, 0x75, 0xb8, 0x3c, 0xe0, 0x92, 0x89, 0x5a, 0x70, 0x07, 0x5e, 0x86, 0xd9, 0x93, 0x41,
-	0xb6, 0xb2, 0x91, 0x46, 0x59, 0x94, 0x9f, 0xa5, 0x7f, 0x13, 0xb9, 0x1f, 0x08, 0xf0, 0x10, 0xcb,
-	0x84, 0xcf, 0xc7, 0x69, 0x98, 0xee, 0x63, 0x7a, 0x3e, 0x25, 0x4e, 0x83, 0x23, 0x7e, 0x0e, 0xa9,
-	0xfe, 0xb5, 0x94, 0x9f, 0x38, 0xdc, 0xb2, 0xd3, 0xb1, 0x2c, 0xca, 0xa7, 0x4a, 0x4b, 0x85, 0x61,
-	0x29, 0x5b, 0x8c, 0xed, 0xf8, 0x1f, 0x69, 0x18, 0x89, 0xf3, 0x30, 0xbf, 0x2f, 0x85, 0x61, 0x73,
-	0xa3, 0x5d, 0x6e, 0xb7, 0x25, 0xb7, 0xac, 0xf4, 0x94, 0x57, 0xcd, 0x78, 0x1a, 0xdf, 0x85, 0xa4,
-	0x63, 0x79, 0xe5, 0xc6, 0x3d, 0x80, 0x7f, 0xc2, 0x39, 0x98, 0xb3, 0xec, 0x3d, 0xdb, 0xd2, 0x8c,
-	0xbd, 0xe6, 0x11, 0x6f, 0xa7, 0x13, 0x59, 0x94, 0x9f, 0xa1, 0x23, 0xb9, 0xdc, 0x97, 0x18, 0x2c,
-	0x6c, 0xfa, 0xff, 0x0b, 0xbb, 0xf0, 0x02, 0xe2, 0x76, 0xcf, 0xe4, 0x9e, 0x9a, 0xff, 0x4b, 0x0f,
-	0x0b, 0xa1, 0x1e, 0x14, 0x22, 0xf0, 0xac, 0x67, 0x72, 0xea, 0x31, 0xa2, 0xea, 0x8e, 0x45, 0xd7,
-	0x1d, 0x32, 0x6d, 0x6a, 0xd4, 0xb4, 0x49, 0x8a, 0xc6, 0xcc, 0x4c, 0xdc, 0xda, 0xcc, 0x71, 0x2b,
-	0x92, 0x11, 0x56, 0x1c, 0xc2, 0x42, 0xa8, 0xb3, 0x81, 0x48, 0xfc, 0x12, 0x92, 0x7d, 0x98, 0x63,
-	0xf9, 0x5e, 0x3c, 0x1a, 0xf1, 0x22, 0x82, 0x51, 0xf3, 0xd0, 0xd4, 0x67, 0xe1, 0x45, 0x48, 0x70,
-	0x29, 0x85, 0xf4, 0x5d, 0x18, 0x1c, 0x72, 0x6b, 0xb0, 0xac, 0x0b, 0xfb, 0x60, 0xbf, 0xe7, 0x4f,
-	0x50, 0xad, 0xeb, 0xd8, 0x6d, 0x71, 0x6a, 0x04, 0x05, 0x5f, 0x3f, 0x85, 0x2b, 0xf0, 0x60, 0x02,
-	0xdb, 0x32, 0x85, 0x61, 0xf1, 0xd5, 0x35, 0xb8, 0x37, 0xa1, 0x4b, 0x78, 0x06, 0xe2, 0x15, 0xbd,
-	0xc2, 0x54, 0x05, 0xa7, 0x60, 0x5a, 0xd3, 0x77, 0xeb, 0x5a, 0x5d, 0x53, 0x11, 0x06, 0x48, 0xae,
-	0x97, 0xf5, 0x75, 0x6d, 0x5b, 0x8d, 0xad, 0xb6, 0xe0, 0xfe, 0x44, 0x5d, 0x38, 0x09, 0xb1, 0xea,
-	0x1b, 0x55, 0xc1, 0x59, 0x58, 0x66, 0xd5, 0x6a, 0xe3, 0x6d, 0x59, 0xff, 0xd0, 0xa0, 0xda, 0x6e,
-	0x5d, 0xab, 0xb1, 0x5a, 0x63, 0x47, 0xa3, 0x0d, 0xa6, 0xe9, 0x65, 0x9d, 0xa9, 0x08, 0xcf, 0x42,
-	0x42, 0xa3, 0xb4, 0x4a, 0xd5, 0x18, 0xbe, 0x03, 0xff, 0xd5, 0xb6, 0xea, 0x8c, 0x55, 0xf4, 0xd7,
-	0x8d, 0x8d, 0xea, 0x3b, 0x5d, 0x9d, 0x2a, 0xfd, 0x44, 0x21, 0xbf, 0x37, 0x85, 0x0c, 0x56, 0xa9,
-	0x0e, 0x29, 0x3f, 0xdc, 0x16, 0xc2, 0xc4, 0x2b, 0x23, 0x76, 0xff, 0xbb, 0xaf, 0x99, 0x95, 0x49,
-	0xfd, 0xf0, 0xb1, 0x39, 0x25, 0x8f, 0x9e, 0x20, 0x6c, 0xc0, 0x52, 0xa4, 0x65, 0xf8, 0xf1, 0x08,
-	0xff, 0xba, 0xa6, 0x64, 0x56, 0x6f, 0x03, 0x1d, 0x74, 0xa0, 0x64, 0xc2, 0x62, 0x58, 0xdd, 0x70,
-	0x9c, 0xde, 0xc3, 0x5c, 0x10, 0x7b, 0xfa, 0xb2, 0x37, 0xad, 0x56, 0x26, 0x7b, 0xd3, 0xc0, 0x0d,
-	0x14, 0xbe, 0x2a, 0x9f, 0x5d, 0x10, 0xe5, 0xfc, 0x82, 0x28, 0x57, 0x17, 0x04, 0x7d, 0x76, 0x09,
-	0xfa, 0xe6, 0x12, 0xf4, 0xdd, 0x25, 0xe8, 0xcc, 0x25, 0xe8, 0x97, 0x4b, 0xd0, 0x6f, 0x97, 0x28,
-	0x57, 0x2e, 0x41, 0x5f, 0x2f, 0x89, 0x72, 0x76, 0x49, 0x94, 0xf3, 0x4b, 0xa2, 0x7c, 0x0c, 0xbf,
-	0xae, 0xcd, 0xa4, 0xf7, 0x30, 0x3e, 0xfd, 0x13, 0x00, 0x00, 0xff, 0xff, 0x88, 0x0c, 0xfe, 0x56,
-	0x84, 0x05, 0x00, 0x00,
+	// 666 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x94, 0xcf, 0x4e, 0xdb, 0x40,
+	0x10, 0xc6, 0xbd, 0x21, 0x09, 0x30, 0xa1, 0xe0, 0x2e, 0xd0, 0xa6, 0x11, 0x5d, 0x22, 0xab, 0x42,
+	0x29, 0x87, 0xa4, 0x4a, 0x2b, 0xb5, 0x07, 0x54, 0x29, 0x05, 0x53, 0xa2, 0x52, 0x07, 0x1c, 0x47,
+	0xfd, 0x73, 0x89, 0x48, 0xb2, 0x24, 0x16, 0xe0, 0x35, 0xeb, 0x75, 0x51, 0x6e, 0x7d, 0x84, 0x3e,
+	0x46, 0x8f, 0x7d, 0x8c, 0x5e, 0x2a, 0x71, 0xe4, 0xd0, 0x43, 0x71, 0x2e, 0x3d, 0xf2, 0x08, 0x55,
+	0x1c, 0x27, 0x75, 0x42, 0x02, 0xdc, 0x66, 0xc7, 0xdf, 0x67, 0xcf, 0xfc, 0x66, 0xd6, 0xb0, 0xe0,
+	0xd4, 0x5b, 0xb4, 0xe1, 0x1e, 0x53, 0x9e, 0xb5, 0x39, 0x13, 0x0c, 0x27, 0x06, 0x09, 0xbb, 0x96,
+	0x5a, 0x6a, 0xb2, 0x26, 0xf3, 0xf3, 0xb9, 0x6e, 0xd4, 0x93, 0xa4, 0x5e, 0x34, 0x4d, 0xd1, 0x72,
+	0x6b, 0xd9, 0x3a, 0x3b, 0xc9, 0x9d, 0xd1, 0x83, 0x2f, 0xf4, 0x8c, 0xf1, 0x23, 0x27, 0x57, 0x67,
+	0x27, 0x27, 0xcc, 0xca, 0xb5, 0x84, 0xb0, 0x9b, 0xdc, 0xae, 0x0f, 0x82, 0x9e, 0x4b, 0xc9, 0x03,
+	0xde, 0x77, 0x29, 0x37, 0x29, 0x37, 0x58, 0xb9, 0xff, 0x0d, 0xbc, 0x02, 0xb3, 0xa7, 0xbd, 0x6c,
+	0x71, 0x2b, 0x89, 0xd2, 0x28, 0x33, 0xab, 0xff, 0x4f, 0x28, 0xbf, 0x10, 0xe0, 0x81, 0xd6, 0x60,
+	0x81, 0x1f, 0x27, 0x61, 0xba, 0xab, 0x69, 0x07, 0x96, 0xa8, 0xde, 0x3f, 0xe2, 0x97, 0x90, 0xe8,
+	0x7e, 0x56, 0xa7, 0xa7, 0x2e, 0x75, 0x44, 0x32, 0x92, 0x46, 0x99, 0x44, 0x7e, 0x39, 0x3b, 0x28,
+	0x65, 0xc7, 0x30, 0xf6, 0x82, 0x87, 0x7a, 0x58, 0x89, 0x33, 0xb0, 0x70, 0xc8, 0x99, 0x25, 0xa8,
+	0xd5, 0x28, 0x34, 0x1a, 0x9c, 0x3a, 0x4e, 0x72, 0xca, 0xaf, 0x66, 0x34, 0x8d, 0x1f, 0x40, 0xdc,
+	0x75, 0xfc, 0x72, 0xa3, 0xbe, 0x20, 0x38, 0x61, 0x05, 0xe6, 0x1c, 0x71, 0x20, 0x1c, 0xd5, 0x3a,
+	0xa8, 0x1d, 0xd3, 0x46, 0x32, 0x96, 0x46, 0x99, 0x19, 0x7d, 0x28, 0xa7, 0xfc, 0x88, 0xc0, 0xe2,
+	0x76, 0xf0, 0xbe, 0x30, 0x85, 0x57, 0x10, 0x15, 0x6d, 0x9b, 0xfa, 0xdd, 0xcc, 0xe7, 0x9f, 0x64,
+	0x43, 0x33, 0xc8, 0x8e, 0xd1, 0x1b, 0x6d, 0x9b, 0xea, 0xbe, 0x63, 0x5c, 0xdd, 0x91, 0xf1, 0x75,
+	0x87, 0xa0, 0x4d, 0x0d, 0x43, 0x9b, 0xd4, 0xd1, 0x08, 0xcc, 0xd8, 0x9d, 0x61, 0x8e, 0xa2, 0x88,
+	0x5f, 0x47, 0x81, 0xd7, 0x60, 0xde, 0x74, 0x76, 0xcc, 0x66, 0x6b, 0x8f, 0x9b, 0x8c, 0x9b, 0xa2,
+	0x9d, 0x9c, 0xf6, 0x55, 0x23, 0x59, 0xe5, 0x08, 0x16, 0x43, 0x1b, 0xd0, 0x87, 0x81, 0x5f, 0x43,
+	0xbc, 0xfb, 0x3a, 0xd7, 0x09, 0x98, 0xad, 0x0d, 0x31, 0x1b, 0xe3, 0x28, 0xfb, 0x6a, 0x3d, 0x70,
+	0xe1, 0x25, 0x88, 0x51, 0xce, 0x19, 0x0f, 0x68, 0xf5, 0x0e, 0xca, 0x06, 0xac, 0x68, 0x4c, 0x98,
+	0x87, 0xed, 0x60, 0xd3, 0xca, 0x2d, 0x57, 0x34, 0xd8, 0x99, 0xd5, 0x6f, 0xec, 0xe6, 0x6d, 0x5d,
+	0x85, 0xc7, 0x13, 0xdc, 0x8e, 0xcd, 0x2c, 0x87, 0xae, 0x6f, 0xc0, 0xc3, 0x09, 0xd3, 0xc4, 0x33,
+	0x10, 0x2d, 0x6a, 0x45, 0x43, 0x96, 0x70, 0x02, 0xa6, 0x55, 0x6d, 0xbf, 0xa2, 0x56, 0x54, 0x19,
+	0x61, 0x80, 0xf8, 0x66, 0x41, 0xdb, 0x54, 0x77, 0xe5, 0xc8, 0x7a, 0x1d, 0x1e, 0x4d, 0xec, 0x0b,
+	0xc7, 0x21, 0x52, 0x7a, 0x27, 0x4b, 0x38, 0x0d, 0x2b, 0x46, 0xa9, 0x54, 0x7d, 0x5f, 0xd0, 0x3e,
+	0x55, 0x75, 0x75, 0xbf, 0xa2, 0x96, 0x8d, 0x72, 0x75, 0x4f, 0xd5, 0xab, 0x86, 0xaa, 0x15, 0x34,
+	0x43, 0x46, 0x78, 0x16, 0x62, 0xaa, 0xae, 0x97, 0x74, 0x39, 0x82, 0xef, 0xc3, 0xbd, 0xf2, 0x4e,
+	0xc5, 0x30, 0x8a, 0xda, 0xdb, 0xea, 0x56, 0xe9, 0x83, 0x26, 0x4f, 0xe5, 0x7f, 0xa3, 0x10, 0xef,
+	0x6d, 0xc6, 0xfb, 0x57, 0xae, 0x02, 0x89, 0x20, 0xdc, 0x65, 0xcc, 0xc6, 0xab, 0x43, 0xb8, 0xaf,
+	0xdf, 0xeb, 0xd4, 0xea, 0xa4, 0x79, 0x04, 0x5a, 0x45, 0xca, 0xa0, 0x67, 0x08, 0x5b, 0xb0, 0x3c,
+	0x16, 0x19, 0x7e, 0x3a, 0xe4, 0xbf, 0x69, 0x28, 0xa9, 0xf5, 0xbb, 0x48, 0x7b, 0x13, 0xc8, 0xdb,
+	0xb0, 0x14, 0xee, 0x6e, 0xb0, 0x4e, 0x1f, 0x61, 0xae, 0x1f, 0xfb, 0xfd, 0xa5, 0x6f, 0xbb, 0x82,
+	0xa9, 0xf4, 0x6d, 0x0b, 0xd7, 0xeb, 0xf0, 0x4d, 0xe1, 0xfc, 0x92, 0x48, 0x17, 0x97, 0x44, 0xba,
+	0xba, 0x24, 0xe8, 0xab, 0x47, 0xd0, 0x77, 0x8f, 0xa0, 0x9f, 0x1e, 0x41, 0xe7, 0x1e, 0x41, 0x7f,
+	0x3c, 0x82, 0xfe, 0x7a, 0x44, 0xba, 0xf2, 0x08, 0xfa, 0xd6, 0x21, 0xd2, 0x79, 0x87, 0x48, 0x17,
+	0x1d, 0x22, 0x7d, 0x0e, 0xff, 0x85, 0x6b, 0x71, 0xff, 0x07, 0xfa, 0xfc, 0x5f, 0x00, 0x00, 0x00,
+	0xff, 0xff, 0x13, 0x54, 0x85, 0x09, 0xac, 0x05, 0x00, 0x00,
 }
 
 func (x FrontendToSchedulerType) String() string {
@@ -593,6 +602,9 @@ func (this *FrontendToScheduler) Equal(that interface{}) bool {
 	if this.StatsEnabled != that1.StatsEnabled {
 		return false
 	}
+	if this.IsHighPriority != that1.IsHighPriority {
+		return false
+	}
 	return true
 }
 func (this *SchedulerToFrontend) Equal(that interface{}) bool {
@@ -697,7 +709,7 @@ func (this *FrontendToScheduler) GoString() string {
 	if this == nil {
 		return "nil"
 	}
-	s := make([]string, 0, 10)
+	s := make([]string, 0, 11)
 	s = append(s, "&schedulerpb.FrontendToScheduler{")
 	s = append(s, "Type: "+fmt.Sprintf("%#v", this.Type)+",\n")
 	s = append(s, "FrontendAddress: "+fmt.Sprintf("%#v", this.FrontendAddress)+",\n")
@@ -707,6 +719,7 @@ func (this *FrontendToScheduler) GoString() string {
 		s = append(s, "HttpRequest: "+fmt.Sprintf("%#v", this.HttpRequest)+",\n")
 	}
 	s = append(s, "StatsEnabled: "+fmt.Sprintf("%#v", this.StatsEnabled)+",\n")
+	s = append(s, "IsHighPriority: "+fmt.Sprintf("%#v", this.IsHighPriority)+",\n")
 	s = append(s, "}")
 	return strings.Join(s, "")
 }
@@ -1142,6 +1155,16 @@ func (m *FrontendToScheduler) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 	_ = i
 	var l int
 	_ = l
+	if m.IsHighPriority {
+		i--
+		if m.IsHighPriority {
+			dAtA[i] = 1
+		} else {
+			dAtA[i] = 0
+		}
+		i--
+		dAtA[i] = 0x38
+	}
 	if m.StatsEnabled {
 		i--
 		if m.StatsEnabled {
@@ -1357,6 +1380,9 @@ func (m *FrontendToScheduler) Size() (n int) {
 	if m.StatsEnabled {
 		n += 2
 	}
+	if m.IsHighPriority {
+		n += 2
+	}
 	return n
 }
 
@@ -1439,6 +1465,7 @@ func (this *FrontendToScheduler) String() string {
 		`UserID:` + fmt.Sprintf("%v", this.UserID) + `,`,
 		`HttpRequest:` + strings.Replace(fmt.Sprintf("%v", this.HttpRequest), "HTTPRequest", "httpgrpc.HTTPRequest", 1) + `,`,
 		`StatsEnabled:` + fmt.Sprintf("%v", this.StatsEnabled) + `,`,
+		`IsHighPriority:` + fmt.Sprintf("%v", this.IsHighPriority) + `,`,
 		`}`,
 	}, "")
 	return s
@@ -1945,6 +1972,26 @@ func (m *FrontendToScheduler) Unmarshal(dAtA []byte) error {
 				}
 			}
 			m.StatsEnabled = bool(v != 0)
+		case 7:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field IsHighPriority", wireType)
+			}
+			var v int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowScheduler
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				v |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			m.IsHighPriority = bool(v != 0)
 		default:
 			iNdEx = preIndex
 			skippy, err := skipScheduler(dAtA[iNdEx:])
diff --git a/pkg/scheduler/schedulerpb/scheduler.proto b/pkg/scheduler/schedulerpb/scheduler.proto
index eea28717b8..5e71946008 100644
--- a/pkg/scheduler/schedulerpb/scheduler.proto
+++ b/pkg/scheduler/schedulerpb/scheduler.proto
@@ -78,6 +78,7 @@ message FrontendToScheduler {
   string userID = 4;
   httpgrpc.HTTPRequest httpRequest = 5;
   bool statsEnabled = 6;
+  bool isHighPriority = 7;
 }
 
 enum SchedulerToFrontendStatus {
diff --git a/pkg/util/query_priority.go b/pkg/util/query_priority.go
new file mode 100644
index 0000000000..035bccb0df
--- /dev/null
+++ b/pkg/util/query_priority.go
@@ -0,0 +1,9 @@
+package util
+
+func IsHighPriorityQuery() bool {
+	// TODO: Implement
+	// query string <> regex
+	// query param <> start_time, end_time
+
+	return false
+}
diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go
index 94db111b77..6e1298249a 100644
--- a/pkg/util/validation/limits.go
+++ b/pkg/util/validation/limits.go
@@ -46,6 +46,12 @@ type DisabledRuleGroup struct {
 
 type DisabledRuleGroups []DisabledRuleGroup
 
+type HighPriorityQuery struct {
+	Regex     string        `yaml:"regex" doc:"nocli|description=Query string regex. If evaluated true (on top of meeting all other criteria), query is handled with a high priority."`
+	StartTime time.Duration `yaml:"start_time" doc:"nocli|description=If query range falls between the start_time and end_time (on top of meeting all other criteria), query is handled with a high priority.|default=1h"`
+	EndTime   time.Duration `yaml:"end_time" doc:"nocli|description=If query range falls between the start_time and end_time (on top of meeting all other criteria), query is handled with a high priority.|default=0s"`
+}
+
 // Limits describe all the limits for users; can be used to describe global default
 // limits via flags, or per-user limits via yaml config.
 type Limits struct {
@@ -101,8 +107,9 @@ type Limits struct {
 	QueryVerticalShardSize       int            `yaml:"query_vertical_shard_size" json:"query_vertical_shard_size" doc:"hidden"`
 
 	// Query Frontend / Scheduler enforced limits.
-	MaxOutstandingPerTenant      int     `yaml:"max_outstanding_requests_per_tenant" json:"max_outstanding_requests_per_tenant"`
-	ReservedHighPriorityQueriers float64 `yaml:"reserved_high_priority_queriers" json:"reserved_high_priority_queriers"`
+	MaxOutstandingPerTenant      int                 `yaml:"max_outstanding_requests_per_tenant" json:"max_outstanding_requests_per_tenant"`
+	ReservedHighPriorityQueriers float64             `yaml:"reserved_high_priority_queriers" json:"reserved_high_priority_queriers"`
+	HighPriorityQueries          []HighPriorityQuery `yaml:"high_priority_queries" json:"high_priority_queries" doc:"nocli|description=List of query definitions to be handled with a high priority."`
 
 	// Ruler defaults and limits.
 	RulerEvaluationDelay        model.Duration `yaml:"ruler_evaluation_delay_duration" json:"ruler_evaluation_delay_duration"`
@@ -499,6 +506,11 @@ func (o *Overrides) ReservedHighPriorityQueriers(userID string) float64 {
 	return o.GetOverridesForUser(userID).ReservedHighPriorityQueriers
 }
 
+// HighPriorityQueries returns list of definitions for high priority query.
+func (o *Overrides) HighPriorityQueries(userID string) []HighPriorityQuery {
+	return o.GetOverridesForUser(userID).HighPriorityQueries
+}
+
 // EnforceMetricName whether to enforce the presence of a metric name.
 func (o *Overrides) EnforceMetricName(userID string) bool {
 	return o.GetOverridesForUser(userID).EnforceMetricName
diff --git a/tools/doc-generator/parser.go b/tools/doc-generator/parser.go
index 3ebb313368..7a6dbfebfb 100644
--- a/tools/doc-generator/parser.go
+++ b/tools/doc-generator/parser.go
@@ -227,7 +227,7 @@ func parseConfig(block *configBlock, cfg interface{}, flags map[uintptr]*flag.Fl
 				required:     isFieldRequired(field),
 				fieldDesc:    getFieldDescription(field, ""),
 				fieldType:    fieldType,
-				fieldDefault: fieldDefault,
+				fieldDefault: getFieldDefault(field, fieldDefault),
 			})
 			continue
 		}
@@ -470,6 +470,14 @@ func getFieldDescription(f reflect.StructField, fallback string) string {
 	return fallback
 }
 
+func getFieldDefault(f reflect.StructField, fieldDefault string) string {
+	if defaultValue := getDocTagValue(f, "default"); defaultValue != "" {
+		return defaultValue
+	}
+
+	return fieldDefault
+}
+
 func isRootBlock(t reflect.Type) (string, string, bool) {
 	for _, rootBlock := range rootBlocks {
 		if t == rootBlock.structType {

From b050ac1eaefda92eadbac6395ee1c0986a90e26e Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Tue, 17 Oct 2023 15:38:59 -0700
Subject: [PATCH 09/49] Nit

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 docs/configuration/config-file-reference.md | 8 ++++----
 pkg/frontend/v1/frontend.go                 | 3 ++-
 pkg/frontend/v2/frontend.go                 | 4 ++--
 pkg/util/{ => query}/query_priority.go      | 4 ++--
 pkg/util/validation/limits.go               | 8 ++++----
 5 files changed, 14 insertions(+), 13 deletions(-)
 rename pkg/util/{ => query}/query_priority.go (68%)

diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md
index 78adee77b4..d8cffa118f 100644
--- a/docs/configuration/config-file-reference.md
+++ b/docs/configuration/config-file-reference.md
@@ -3047,7 +3047,7 @@ The `limits_config` configures default and per-tenant limits imposed by Cortex s
 # CLI flag: -frontend.reserved-high-priority-queriers
 [reserved_high_priority_queriers: <float> | default = 0]
 
-# List of query definitions to be handled with a high priority.
+# List of query definitions to be treated as a high priority.
 [high_priority_queries: <list of HighPriorityQuery> | default = []]
 
 # Duration to delay the evaluation of rules to ensure the underlying metrics
@@ -5045,15 +5045,15 @@ otel:
 
 ```yaml
 # Query string regex. If evaluated true (on top of meeting all other criteria),
-# query is handled with a high priority.
+# query is treated as a high priority.
 [regex: <string> | default = ""]
 
 # If query range falls between the start_time and end_time (on top of meeting
-# all other criteria), query is handled with a high priority.
+# all other criteria), query is treated as a high priority.
 [start_time: <duration> | default = 1h]
 
 # If query range falls between the start_time and end_time (on top of meeting
-# all other criteria), query is handled with a high priority.
+# all other criteria), query is treated as a high priority.
 [end_time: <duration> | default = 0s]
 ```
 
diff --git a/pkg/frontend/v1/frontend.go b/pkg/frontend/v1/frontend.go
index 5a0ad4e52e..6bd37274c5 100644
--- a/pkg/frontend/v1/frontend.go
+++ b/pkg/frontend/v1/frontend.go
@@ -22,6 +22,7 @@ import (
 	"github.com/cortexproject/cortex/pkg/tenant"
 	"github.com/cortexproject/cortex/pkg/util"
 	"github.com/cortexproject/cortex/pkg/util/httpgrpcutil"
+	"github.com/cortexproject/cortex/pkg/util/query"
 	"github.com/cortexproject/cortex/pkg/util/services"
 	"github.com/cortexproject/cortex/pkg/util/validation"
 )
@@ -346,7 +347,7 @@ func (f *Frontend) queueRequest(ctx context.Context, req *request) error {
 	now := time.Now()
 	req.enqueueTime = now
 	req.queueSpan, _ = opentracing.StartSpanFromContext(ctx, "queued")
-	req.isHighPriority = util.IsHighPriorityQuery()
+	req.isHighPriority = query.IsHighPriority()
 
 	// aggregate the max queriers limit in the case of a multi tenant query
 	maxQueriers := validation.SmallestPositiveNonZeroFloat64PerTenant(tenantIDs, f.limits.MaxQueriersPerUser)
diff --git a/pkg/frontend/v2/frontend.go b/pkg/frontend/v2/frontend.go
index dfed325fc4..202c2d3042 100644
--- a/pkg/frontend/v2/frontend.go
+++ b/pkg/frontend/v2/frontend.go
@@ -4,7 +4,6 @@ import (
 	"context"
 	"flag"
 	"fmt"
-	"github.com/cortexproject/cortex/pkg/util"
 	"math/rand"
 	"net/http"
 	"sync"
@@ -27,6 +26,7 @@ import (
 	"github.com/cortexproject/cortex/pkg/util/grpcclient"
 	"github.com/cortexproject/cortex/pkg/util/httpgrpcutil"
 	util_log "github.com/cortexproject/cortex/pkg/util/log"
+	"github.com/cortexproject/cortex/pkg/util/query"
 	"github.com/cortexproject/cortex/pkg/util/services"
 )
 
@@ -196,7 +196,7 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest)
 			request:        req,
 			userID:         userID,
 			statsEnabled:   stats.IsEnabled(ctx),
-			isHighPriority: util.IsHighPriorityQuery(),
+			isHighPriority: query.IsHighPriority(),
 
 			cancel: cancel,
 
diff --git a/pkg/util/query_priority.go b/pkg/util/query/query_priority.go
similarity index 68%
rename from pkg/util/query_priority.go
rename to pkg/util/query/query_priority.go
index 035bccb0df..6e18fe61de 100644
--- a/pkg/util/query_priority.go
+++ b/pkg/util/query/query_priority.go
@@ -1,6 +1,6 @@
-package util
+package query
 
-func IsHighPriorityQuery() bool {
+func IsHighPriority() bool {
 	// TODO: Implement
 	// query string <> regex
 	// query param <> start_time, end_time
diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go
index 6e1298249a..145ad317ae 100644
--- a/pkg/util/validation/limits.go
+++ b/pkg/util/validation/limits.go
@@ -47,9 +47,9 @@ type DisabledRuleGroup struct {
 type DisabledRuleGroups []DisabledRuleGroup
 
 type HighPriorityQuery struct {
-	Regex     string        `yaml:"regex" doc:"nocli|description=Query string regex. If evaluated true (on top of meeting all other criteria), query is handled with a high priority."`
-	StartTime time.Duration `yaml:"start_time" doc:"nocli|description=If query range falls between the start_time and end_time (on top of meeting all other criteria), query is handled with a high priority.|default=1h"`
-	EndTime   time.Duration `yaml:"end_time" doc:"nocli|description=If query range falls between the start_time and end_time (on top of meeting all other criteria), query is handled with a high priority.|default=0s"`
+	Regex     string        `yaml:"regex" doc:"nocli|description=Query string regex. If evaluated true (on top of meeting all other criteria), query is treated as a high priority."`
+	StartTime time.Duration `yaml:"start_time" doc:"nocli|description=If query range falls between the start_time and end_time (on top of meeting all other criteria), query is treated as a high priority.|default=1h"`
+	EndTime   time.Duration `yaml:"end_time" doc:"nocli|description=If query range falls between the start_time and end_time (on top of meeting all other criteria), query is treated as a high priority.|default=0s"`
 }
 
 // Limits describe all the limits for users; can be used to describe global default
@@ -109,7 +109,7 @@ type Limits struct {
 	// Query Frontend / Scheduler enforced limits.
 	MaxOutstandingPerTenant      int                 `yaml:"max_outstanding_requests_per_tenant" json:"max_outstanding_requests_per_tenant"`
 	ReservedHighPriorityQueriers float64             `yaml:"reserved_high_priority_queriers" json:"reserved_high_priority_queriers"`
-	HighPriorityQueries          []HighPriorityQuery `yaml:"high_priority_queries" json:"high_priority_queries" doc:"nocli|description=List of query definitions to be handled with a high priority."`
+	HighPriorityQueries          []HighPriorityQuery `yaml:"high_priority_queries" json:"high_priority_queries" doc:"nocli|description=List of query definitions to be treated as a high priority."`
 
 	// Ruler defaults and limits.
 	RulerEvaluationDelay        model.Duration `yaml:"ruler_evaluation_delay_duration" json:"ruler_evaluation_delay_duration"`

From f76761490bb12c544a9b95dbeeb39f00b5043580 Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Tue, 17 Oct 2023 15:48:26 -0700
Subject: [PATCH 10/49] Lint

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 pkg/frontend/v1/frontend.go | 4 ++--
 pkg/frontend/v2/frontend.go | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/pkg/frontend/v1/frontend.go b/pkg/frontend/v1/frontend.go
index 6bd37274c5..9eae59255b 100644
--- a/pkg/frontend/v1/frontend.go
+++ b/pkg/frontend/v1/frontend.go
@@ -22,7 +22,7 @@ import (
 	"github.com/cortexproject/cortex/pkg/tenant"
 	"github.com/cortexproject/cortex/pkg/util"
 	"github.com/cortexproject/cortex/pkg/util/httpgrpcutil"
-	"github.com/cortexproject/cortex/pkg/util/query"
+	util_query "github.com/cortexproject/cortex/pkg/util/query"
 	"github.com/cortexproject/cortex/pkg/util/services"
 	"github.com/cortexproject/cortex/pkg/util/validation"
 )
@@ -347,7 +347,7 @@ func (f *Frontend) queueRequest(ctx context.Context, req *request) error {
 	now := time.Now()
 	req.enqueueTime = now
 	req.queueSpan, _ = opentracing.StartSpanFromContext(ctx, "queued")
-	req.isHighPriority = query.IsHighPriority()
+	req.isHighPriority = util_query.IsHighPriority()
 
 	// aggregate the max queriers limit in the case of a multi tenant query
 	maxQueriers := validation.SmallestPositiveNonZeroFloat64PerTenant(tenantIDs, f.limits.MaxQueriersPerUser)
diff --git a/pkg/frontend/v2/frontend.go b/pkg/frontend/v2/frontend.go
index 202c2d3042..e01b9562e8 100644
--- a/pkg/frontend/v2/frontend.go
+++ b/pkg/frontend/v2/frontend.go
@@ -26,7 +26,7 @@ import (
 	"github.com/cortexproject/cortex/pkg/util/grpcclient"
 	"github.com/cortexproject/cortex/pkg/util/httpgrpcutil"
 	util_log "github.com/cortexproject/cortex/pkg/util/log"
-	"github.com/cortexproject/cortex/pkg/util/query"
+	util_query "github.com/cortexproject/cortex/pkg/util/query"
 	"github.com/cortexproject/cortex/pkg/util/services"
 )
 
@@ -196,7 +196,7 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest)
 			request:        req,
 			userID:         userID,
 			statsEnabled:   stats.IsEnabled(ctx),
-			isHighPriority: query.IsHighPriority(),
+			isHighPriority: util_query.IsHighPriority(),
 
 			cancel: cancel,
 

From de06e37fa7d3579bbc61bad801aad6cecb871949 Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Wed, 18 Oct 2023 15:33:23 -0700
Subject: [PATCH 11/49] Implement IsPriorityQuery

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 docs/configuration/config-file-reference.md |   2 +-
 pkg/frontend/config.go                      |   2 +-
 pkg/frontend/transport/roundtripper.go      |   5 +-
 pkg/frontend/v1/frontend.go                 |  20 ++-
 pkg/frontend/v2/frontend.go                 |  18 ++-
 pkg/frontend/v2/frontend_test.go            |  17 ++-
 pkg/scheduler/queue/queue_test.go           |   2 +-
 pkg/scheduler/queue/user_queues.go          |  21 ++-
 pkg/util/query/priority.go                  |  49 +++++++
 pkg/util/query/priority_test.go             | 154 ++++++++++++++++++++
 pkg/util/query/query_priority.go            |   9 --
 pkg/util/validation/limits.go               |   2 +-
 12 files changed, 265 insertions(+), 36 deletions(-)
 create mode 100644 pkg/util/query/priority.go
 create mode 100644 pkg/util/query/priority_test.go
 delete mode 100644 pkg/util/query/query_priority.go

diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md
index d8cffa118f..9ae1ddb8d5 100644
--- a/docs/configuration/config-file-reference.md
+++ b/docs/configuration/config-file-reference.md
@@ -5050,7 +5050,7 @@ otel:
 
 # If query range falls between the start_time and end_time (on top of meeting
 # all other criteria), query is treated as a high priority.
-[start_time: <duration> | default = 1h]
+[start_time: <duration> | default = 0s]
 
 # If query range falls between the start_time and end_time (on top of meeting
 # all other criteria), query is treated as a high priority.
diff --git a/pkg/frontend/config.go b/pkg/frontend/config.go
index 8ef8fa3603..03dff13980 100644
--- a/pkg/frontend/config.go
+++ b/pkg/frontend/config.go
@@ -59,7 +59,7 @@ func InitFrontend(cfg CombinedFrontendConfig, limits v1.Limits, grpcListenPort i
 			cfg.FrontendV2.Port = grpcListenPort
 		}
 
-		fr, err := v2.NewFrontend(cfg.FrontendV2, log, reg, retry)
+		fr, err := v2.NewFrontend(cfg.FrontendV2, limits, log, reg, retry)
 		return transport.AdaptGrpcRoundTripperToHTTPRoundTripper(fr), nil, fr, err
 
 	default:
diff --git a/pkg/frontend/transport/roundtripper.go b/pkg/frontend/transport/roundtripper.go
index 583fc22d04..e996d8abb3 100644
--- a/pkg/frontend/transport/roundtripper.go
+++ b/pkg/frontend/transport/roundtripper.go
@@ -5,6 +5,7 @@ import (
 	"context"
 	"io"
 	"net/http"
+	"net/url"
 
 	"github.com/weaveworks/common/httpgrpc"
 	"github.com/weaveworks/common/httpgrpc/server"
@@ -12,7 +13,7 @@ import (
 
 // GrpcRoundTripper is similar to http.RoundTripper, but works with HTTP requests converted to protobuf messages.
 type GrpcRoundTripper interface {
-	RoundTripGRPC(context.Context, *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error)
+	RoundTripGRPC(context.Context, url.Values, *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error)
 }
 
 func AdaptGrpcRoundTripperToHTTPRoundTripper(r GrpcRoundTripper) http.RoundTripper {
@@ -39,7 +40,7 @@ func (a *grpcRoundTripperAdapter) RoundTrip(r *http.Request) (*http.Response, er
 		return nil, err
 	}
 
-	resp, err := a.roundTripper.RoundTripGRPC(r.Context(), req)
+	resp, err := a.roundTripper.RoundTripGRPC(r.Context(), r.Form, req)
 	if err != nil {
 		return nil, err
 	}
diff --git a/pkg/frontend/v1/frontend.go b/pkg/frontend/v1/frontend.go
index 9eae59255b..64e19f35cb 100644
--- a/pkg/frontend/v1/frontend.go
+++ b/pkg/frontend/v1/frontend.go
@@ -5,6 +5,7 @@ import (
 	"flag"
 	"fmt"
 	"net/http"
+	"net/url"
 	"time"
 
 	"github.com/go-kit/log"
@@ -53,6 +54,7 @@ type Limits interface {
 // MockLimits implements the Limits interface. Used in tests only.
 type MockLimits struct {
 	Queriers float64
+	Queries  []validation.HighPriorityQuery
 	queue.MockLimits
 }
 
@@ -60,6 +62,10 @@ func (l MockLimits) MaxQueriersPerUser(_ string) float64 {
 	return l.Queriers
 }
 
+func (l MockLimits) HighPriorityQueries(_ string) []validation.HighPriorityQuery {
+	return l.Queries
+}
+
 // Frontend queues HTTP requests, dispatches them to backends, and handles retries
 // for requests which failed.
 type Frontend struct {
@@ -171,7 +177,7 @@ func (f *Frontend) cleanupInactiveUserMetrics(user string) {
 }
 
 // RoundTripGRPC round trips a proto (instead of a HTTP request).
-func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error) {
+func (f *Frontend) RoundTripGRPC(ctx context.Context, requestParams url.Values, req *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error) {
 	// Propagate trace context in gRPC too - this will be ignored if using HTTP.
 	tracer, span := opentracing.GlobalTracer(), opentracing.SpanFromContext(ctx)
 	if tracer != nil && span != nil {
@@ -182,10 +188,17 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest)
 		}
 	}
 
+	tenantIDs, err := tenant.TenantIDs(ctx)
+	if err != nil {
+		return nil, err
+	}
+	userID := tenant.JoinTenantIDs(tenantIDs)
+
 	return f.retry.Do(ctx, func() (*httpgrpc.HTTPResponse, error) {
 		request := request{
-			request:     req,
-			originalCtx: ctx,
+			request:        req,
+			originalCtx:    ctx,
+			isHighPriority: util_query.IsHighPriority(requestParams, f.limits.HighPriorityQueries(userID)),
 
 			// Buffer of 1 to ensure response can be written by the server side
 			// of the Process stream, even if this goroutine goes away due to
@@ -347,7 +360,6 @@ func (f *Frontend) queueRequest(ctx context.Context, req *request) error {
 	now := time.Now()
 	req.enqueueTime = now
 	req.queueSpan, _ = opentracing.StartSpanFromContext(ctx, "queued")
-	req.isHighPriority = util_query.IsHighPriority()
 
 	// aggregate the max queriers limit in the case of a multi tenant query
 	maxQueriers := validation.SmallestPositiveNonZeroFloat64PerTenant(tenantIDs, f.limits.MaxQueriersPerUser)
diff --git a/pkg/frontend/v2/frontend.go b/pkg/frontend/v2/frontend.go
index e01b9562e8..3f0b8d3372 100644
--- a/pkg/frontend/v2/frontend.go
+++ b/pkg/frontend/v2/frontend.go
@@ -6,9 +6,12 @@ import (
 	"fmt"
 	"math/rand"
 	"net/http"
+	"net/url"
 	"sync"
 	"time"
 
+	"github.com/cortexproject/cortex/pkg/scheduler"
+
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 	"github.com/opentracing/opentracing-go"
@@ -65,10 +68,10 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
 type Frontend struct {
 	services.Service
 
-	cfg Config
-	log log.Logger
-
-	retry *transport.Retry
+	cfg    Config
+	log    log.Logger
+	limits scheduler.Limits
+	retry  *transport.Retry
 
 	lastQueryID atomic.Uint64
 
@@ -114,7 +117,7 @@ type enqueueResult struct {
 }
 
 // NewFrontend creates a new frontend.
-func NewFrontend(cfg Config, log log.Logger, reg prometheus.Registerer, retry *transport.Retry) (*Frontend, error) {
+func NewFrontend(cfg Config, limits scheduler.Limits, log log.Logger, reg prometheus.Registerer, retry *transport.Retry) (*Frontend, error) {
 	requestsCh := make(chan *frontendRequest)
 
 	schedulerWorkers, err := newFrontendSchedulerWorkers(cfg, fmt.Sprintf("%s:%d", cfg.Addr, cfg.Port), requestsCh, log)
@@ -124,6 +127,7 @@ func NewFrontend(cfg Config, log log.Logger, reg prometheus.Registerer, retry *t
 
 	f := &Frontend{
 		cfg:              cfg,
+		limits:           limits,
 		log:              log,
 		requestsCh:       requestsCh,
 		schedulerWorkers: schedulerWorkers,
@@ -167,7 +171,7 @@ func (f *Frontend) stopping(_ error) error {
 }
 
 // RoundTripGRPC round trips a proto (instead of a HTTP request).
-func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error) {
+func (f *Frontend) RoundTripGRPC(ctx context.Context, requestParams url.Values, req *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error) {
 	if s := f.State(); s != services.Running {
 		return nil, fmt.Errorf("frontend not running: %v", s)
 	}
@@ -196,7 +200,7 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest)
 			request:        req,
 			userID:         userID,
 			statsEnabled:   stats.IsEnabled(ctx),
-			isHighPriority: util_query.IsHighPriority(),
+			isHighPriority: util_query.IsHighPriority(requestParams, f.limits.HighPriorityQueries(userID)),
 
 			cancel: cancel,
 
diff --git a/pkg/frontend/v2/frontend_test.go b/pkg/frontend/v2/frontend_test.go
index 59729e1757..68023f9f57 100644
--- a/pkg/frontend/v2/frontend_test.go
+++ b/pkg/frontend/v2/frontend_test.go
@@ -3,12 +3,15 @@ package v2
 import (
 	"context"
 	"net"
+	"net/url"
 	"strconv"
 	"strings"
 	"sync"
 	"testing"
 	"time"
 
+	"github.com/cortexproject/cortex/pkg/scheduler/queue"
+
 	"github.com/go-kit/log"
 	"github.com/stretchr/testify/require"
 	"github.com/weaveworks/common/httpgrpc"
@@ -48,7 +51,7 @@ func setupFrontend(t *testing.T, schedulerReplyFunc func(f *Frontend, msg *sched
 
 	//logger := log.NewLogfmtLogger(os.Stdout)
 	logger := log.NewNopLogger()
-	f, err := NewFrontend(cfg, logger, nil, transport.NewRetry(maxRetries, nil))
+	f, err := NewFrontend(cfg, queue.MockLimits{}, logger, nil, transport.NewRetry(maxRetries, nil))
 	require.NoError(t, err)
 
 	frontendv2pb.RegisterFrontendForQuerierServer(server, f)
@@ -110,7 +113,7 @@ func TestFrontendBasicWorkflow(t *testing.T) {
 		return &schedulerpb.SchedulerToFrontend{Status: schedulerpb.OK}
 	}, 0)
 
-	resp, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), &httpgrpc.HTTPRequest{})
+	resp, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), url.Values{}, &httpgrpc.HTTPRequest{})
 	require.NoError(t, err)
 	require.Equal(t, int32(200), resp.Code)
 	require.Equal(t, []byte(body), resp.Body)
@@ -140,7 +143,7 @@ func TestFrontendRetryRequest(t *testing.T) {
 		return &schedulerpb.SchedulerToFrontend{Status: schedulerpb.OK}
 	}, 3)
 
-	res, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), &httpgrpc.HTTPRequest{})
+	res, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), url.Values{}, &httpgrpc.HTTPRequest{})
 	require.NoError(t, err)
 	require.Equal(t, int32(200), res.Code)
 }
@@ -167,7 +170,7 @@ func TestFrontendRetryEnqueue(t *testing.T) {
 		return &schedulerpb.SchedulerToFrontend{Status: schedulerpb.OK}
 	}, 0)
 
-	_, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), &httpgrpc.HTTPRequest{})
+	_, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), url.Values{}, &httpgrpc.HTTPRequest{})
 	require.NoError(t, err)
 }
 
@@ -176,7 +179,7 @@ func TestFrontendEnqueueFailure(t *testing.T) {
 		return &schedulerpb.SchedulerToFrontend{Status: schedulerpb.SHUTTING_DOWN}
 	}, 0)
 
-	_, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), "test"), &httpgrpc.HTTPRequest{})
+	_, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), "test"), url.Values{}, &httpgrpc.HTTPRequest{})
 	require.Error(t, err)
 	require.True(t, strings.Contains(err.Error(), "failed to enqueue request"))
 }
@@ -187,7 +190,7 @@ func TestFrontendCancellation(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
 	defer cancel()
 
-	resp, err := f.RoundTripGRPC(user.InjectOrgID(ctx, "test"), &httpgrpc.HTTPRequest{})
+	resp, err := f.RoundTripGRPC(user.InjectOrgID(ctx, "test"), url.Values{}, &httpgrpc.HTTPRequest{})
 	require.EqualError(t, err, context.DeadlineExceeded.Error())
 	require.Nil(t, resp)
 
@@ -236,7 +239,7 @@ func TestFrontendFailedCancellation(t *testing.T) {
 	}()
 
 	// send request
-	resp, err := f.RoundTripGRPC(user.InjectOrgID(ctx, "test"), &httpgrpc.HTTPRequest{})
+	resp, err := f.RoundTripGRPC(user.InjectOrgID(ctx, "test"), url.Values{}, &httpgrpc.HTTPRequest{})
 	require.EqualError(t, err, context.Canceled.Error())
 	require.Nil(t, resp)
 
diff --git a/pkg/scheduler/queue/queue_test.go b/pkg/scheduler/queue/queue_test.go
index 5e260323ec..283eb9df4f 100644
--- a/pkg/scheduler/queue/queue_test.go
+++ b/pkg/scheduler/queue/queue_test.go
@@ -200,7 +200,7 @@ func TestRequestQueue_ReservedQueriersShouldOnlyGetHighPriorityQueries(t *testin
 	queue := NewRequestQueue(0, 0,
 		prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user"}),
 		prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user"}),
-		MockLimits{MaxOutstanding: 3, ReservedQueriers: 1},
+		MockLimits{MaxOutstanding: 3, reservedHighPriorityQueriers: 1},
 		nil,
 	)
 	ctx := context.Background()
diff --git a/pkg/scheduler/queue/user_queues.go b/pkg/scheduler/queue/user_queues.go
index fa6ac4296d..29194b613b 100644
--- a/pkg/scheduler/queue/user_queues.go
+++ b/pkg/scheduler/queue/user_queues.go
@@ -6,6 +6,8 @@ import (
 	"sort"
 	"time"
 
+	"github.com/cortexproject/cortex/pkg/util/validation"
+
 	"github.com/cortexproject/cortex/pkg/util"
 )
 
@@ -21,6 +23,9 @@ type Limits interface {
 	// If ReservedHighPriorityQueriers is capped by MaxQueriersPerUser.
 	// If less than 1, it will be applied as a percentage of MaxQueriersPerUser.
 	ReservedHighPriorityQueriers(user string) float64
+
+	// HighPriorityQueries returns list of definitions for high priority query.
+	HighPriorityQueries(user string) []validation.HighPriorityQuery
 }
 
 // querier holds information about a querier registered in the queue.
@@ -374,8 +379,14 @@ func getNumOfReservedQueriers(queriersToSelect int, totalNumOfQueriers int, rese
 
 // MockLimits implements the Limits interface. Used in tests only.
 type MockLimits struct {
-	MaxOutstanding   int
-	ReservedQueriers float64
+	MaxOutstanding               int
+	maxQueriersPerUser           float64
+	reservedHighPriorityQueriers float64
+	highPriorityQueries          []validation.HighPriorityQuery
+}
+
+func (l MockLimits) MaxQueriersPerUser(user string) float64 {
+	return l.maxQueriersPerUser
 }
 
 func (l MockLimits) MaxOutstandingPerTenant(_ string) int {
@@ -383,5 +394,9 @@ func (l MockLimits) MaxOutstandingPerTenant(_ string) int {
 }
 
 func (l MockLimits) ReservedHighPriorityQueriers(_ string) float64 {
-	return l.ReservedQueriers
+	return l.reservedHighPriorityQueriers
+}
+
+func (l MockLimits) HighPriorityQueries(_ string) []validation.HighPriorityQuery {
+	return l.highPriorityQueries
 }
diff --git a/pkg/util/query/priority.go b/pkg/util/query/priority.go
new file mode 100644
index 0000000000..7347ce04d1
--- /dev/null
+++ b/pkg/util/query/priority.go
@@ -0,0 +1,49 @@
+package query
+
+import (
+	"net/url"
+	"regexp"
+	"strconv"
+	"time"
+
+	"github.com/cortexproject/cortex/pkg/util/validation"
+)
+
+func IsHighPriority(requestParams url.Values, highPriorityQueries []validation.HighPriorityQuery) bool {
+	queryParam := requestParams.Get("query")
+	timeParam := requestParams.Get("time")
+	startParam := requestParams.Get("start")
+	endParam := requestParams.Get("end")
+
+	for _, highPriorityQuery := range highPriorityQueries {
+		regex := highPriorityQuery.Regex
+
+		if match, err := regexp.MatchString(regex, queryParam); !match || err != nil {
+			continue
+		}
+
+		now := time.Now()
+		startTimeThreshold := now.Add(-1 * highPriorityQuery.StartTime).UnixMilli()
+		endTimeThreshold := now.Add(-1 * highPriorityQuery.EndTime).UnixMilli()
+
+		if time, err := strconv.ParseInt(timeParam, 10, 64); err == nil {
+			if isBetweenThresholds(time, time, startTimeThreshold, endTimeThreshold) {
+				return true
+			}
+		}
+
+		if startTime, err := strconv.ParseInt(startParam, 10, 64); err == nil {
+			if endTime, err := strconv.ParseInt(endParam, 10, 64); err == nil {
+				if isBetweenThresholds(startTime, endTime, startTimeThreshold, endTimeThreshold) {
+					return true
+				}
+			}
+		}
+	}
+
+	return false
+}
+
+func isBetweenThresholds(start, end, startThreshold, endThreshold int64) bool {
+	return start >= startThreshold && end <= endThreshold
+}
diff --git a/pkg/util/query/priority_test.go b/pkg/util/query/priority_test.go
new file mode 100644
index 0000000000..2e7a414487
--- /dev/null
+++ b/pkg/util/query/priority_test.go
@@ -0,0 +1,154 @@
+package query
+
+import (
+	"net/url"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/cortexproject/cortex/pkg/util/validation"
+)
+
+func Test_IsHighPriority_DefaultValues(t *testing.T) {
+	now := time.Now()
+	config := []validation.HighPriorityQuery{
+		{}, // By default, it should match all queries happened at "now"
+	}
+
+	assert.True(t, IsHighPriority(url.Values{
+		"query": []string{"count(up)"},
+		"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
+	}, config))
+	assert.False(t, IsHighPriority(url.Values{
+		"query": []string{"count(up)"},
+		"time":  []string{strconv.FormatInt(now.Add(-1*time.Second).UnixMilli(), 10)},
+	}, config))
+}
+
+func Test_IsHighPriority_ShouldMatchRegex(t *testing.T) {
+	now := time.Now()
+	config := []validation.HighPriorityQuery{
+		{
+			Regex: "sum",
+		},
+	}
+
+	assert.True(t, IsHighPriority(url.Values{
+		"query": []string{"sum(up)"},
+		"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
+	}, config))
+	assert.False(t, IsHighPriority(url.Values{
+		"query": []string{"count(up)"},
+		"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
+	}, config))
+
+	config = []validation.HighPriorityQuery{
+		{
+			Regex: "up",
+		},
+	}
+
+	assert.True(t, IsHighPriority(url.Values{
+		"query": []string{"sum(up)"},
+		"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
+	}, config))
+	assert.True(t, IsHighPriority(url.Values{
+		"query": []string{"count(up)"},
+		"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
+	}, config))
+
+	config = []validation.HighPriorityQuery{
+		{
+			Regex: "sum",
+		},
+		{
+			Regex: "c(.+)t",
+		},
+	}
+
+	assert.True(t, IsHighPriority(url.Values{
+		"query": []string{"sum(up)"},
+		"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
+	}, config))
+	assert.True(t, IsHighPriority(url.Values{
+		"query": []string{"count(up)"},
+		"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
+	}, config))
+
+	config = []validation.HighPriorityQuery{
+		{
+			Regex: "doesnotexist",
+		},
+		{
+			Regex: "^sum$",
+		},
+	}
+
+	assert.False(t, IsHighPriority(url.Values{
+		"query": []string{"sum(up)"},
+		"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
+	}, config))
+	assert.False(t, IsHighPriority(url.Values{
+		"query": []string{"count(up)"},
+		"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
+	}, config))
+
+	config = []validation.HighPriorityQuery{
+		{
+			Regex: ".*",
+		},
+	}
+
+	assert.True(t, IsHighPriority(url.Values{
+		"query": []string{"sum(up)"},
+		"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
+	}, config))
+	assert.True(t, IsHighPriority(url.Values{
+		"query": []string{"count(up)"},
+		"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
+	}, config))
+}
+
+func Test_IsHighPriority_ShouldBeBetweenStartAndEndTime(t *testing.T) {
+	now := time.Now()
+	config := []validation.HighPriorityQuery{
+		{
+			StartTime: 1 * time.Hour,
+			EndTime:   30 * time.Minute,
+		},
+	}
+
+	assert.False(t, IsHighPriority(url.Values{
+		"query": []string{"sum(up)"},
+		"time":  []string{strconv.FormatInt(now.Add(-2*time.Hour).UnixMilli(), 10)},
+	}, config))
+	assert.True(t, IsHighPriority(url.Values{
+		"query": []string{"sum(up)"},
+		"time":  []string{strconv.FormatInt(now.Add(-1*time.Hour).UnixMilli(), 10)},
+	}, config))
+	assert.True(t, IsHighPriority(url.Values{
+		"query": []string{"sum(up)"},
+		"time":  []string{strconv.FormatInt(now.Add(-30*time.Minute).UnixMilli(), 10)},
+	}, config))
+	assert.False(t, IsHighPriority(url.Values{
+		"query": []string{"sum(up)"},
+		"time":  []string{strconv.FormatInt(now.Add(-1*time.Minute).UnixMilli(), 10)},
+	}, config))
+	assert.False(t, IsHighPriority(url.Values{
+		"query": []string{"sum(up)"},
+		"start": []string{strconv.FormatInt(now.Add(-2*time.Hour).UnixMilli(), 10)},
+		"end":   []string{strconv.FormatInt(now.Add(-30*time.Minute).UnixMilli(), 10)},
+	}, config))
+	assert.True(t, IsHighPriority(url.Values{
+		"query": []string{"sum(up)"},
+		"start": []string{strconv.FormatInt(now.Add(-1*time.Hour).UnixMilli(), 10)},
+		"end":   []string{strconv.FormatInt(now.Add(-30*time.Minute).UnixMilli(), 10)},
+	}, config))
+	assert.False(t, IsHighPriority(url.Values{
+		"query": []string{"sum(up)"},
+		"start": []string{strconv.FormatInt(now.Add(-1*time.Hour).UnixMilli(), 10)},
+		"end":   []string{strconv.FormatInt(now.UnixMilli(), 10)},
+	}, config))
+}
diff --git a/pkg/util/query/query_priority.go b/pkg/util/query/query_priority.go
deleted file mode 100644
index 6e18fe61de..0000000000
--- a/pkg/util/query/query_priority.go
+++ /dev/null
@@ -1,9 +0,0 @@
-package query
-
-func IsHighPriority() bool {
-	// TODO: Implement
-	// query string <> regex
-	// query param <> start_time, end_time
-
-	return false
-}
diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go
index 145ad317ae..301e945ea8 100644
--- a/pkg/util/validation/limits.go
+++ b/pkg/util/validation/limits.go
@@ -48,7 +48,7 @@ type DisabledRuleGroups []DisabledRuleGroup
 
 type HighPriorityQuery struct {
 	Regex     string        `yaml:"regex" doc:"nocli|description=Query string regex. If evaluated true (on top of meeting all other criteria), query is treated as a high priority."`
-	StartTime time.Duration `yaml:"start_time" doc:"nocli|description=If query range falls between the start_time and end_time (on top of meeting all other criteria), query is treated as a high priority.|default=1h"`
+	StartTime time.Duration `yaml:"start_time" doc:"nocli|description=If query range falls between the start_time and end_time (on top of meeting all other criteria), query is treated as a high priority.|default=0s"`
 	EndTime   time.Duration `yaml:"end_time" doc:"nocli|description=If query range falls between the start_time and end_time (on top of meeting all other criteria), query is treated as a high priority.|default=0s"`
 }
 

From 8b31b1cb25143b8fe8baf38de05687a1c60a0f8e Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Wed, 18 Oct 2023 15:37:51 -0700
Subject: [PATCH 12/49] Add changelog

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index efcf471713..7a8a00c5d0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,7 @@
 * [CHANGE] Store Gateway: Add a new fastcache based inmemory index cache. #5619
 * [CHANGE] Index Cache: Multi level cache backfilling operation becomes async. Added `-blocks-storage.bucket-store.index-cache.multilevel.max-async-concurrency` and `-blocks-storage.bucket-store.index-cache.multilevel.max-async-buffer-size` configs and metric `cortex_store_multilevel_index_cache_backfill_dropped_items_total` for number of dropped items. #5661
 * [FEATURE] Ingester: Add per-tenant new metric `cortex_ingester_tsdb_data_replay_duration_seconds`. #5477
+* [FEATURE] Query Frontend/Scheduler: Add query priority support. #5605
 * [ENHANCEMENT] Store Gateway: Added `-store-gateway.enabled-tenants` and `-store-gateway.disabled-tenants` to explicitly enable or disable store-gateway for specific tenants. #5638
 
 ## 1.16.0 2023-11-20

From d28aa335dcc66e535fa4afa5918601ad3559662e Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Wed, 18 Oct 2023 16:37:00 -0700
Subject: [PATCH 13/49] Pass timestamp as param

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 pkg/frontend/transport/roundtripper.go |  5 +-
 pkg/frontend/v1/frontend.go            |  4 +-
 pkg/frontend/v2/frontend.go            |  4 +-
 pkg/frontend/v2/frontend_test.go       | 12 ++---
 pkg/util/query/priority.go             | 11 ++---
 pkg/util/query/priority_test.go        | 65 +++++++++++++-------------
 6 files changed, 50 insertions(+), 51 deletions(-)

diff --git a/pkg/frontend/transport/roundtripper.go b/pkg/frontend/transport/roundtripper.go
index e996d8abb3..ccbd8d4cc5 100644
--- a/pkg/frontend/transport/roundtripper.go
+++ b/pkg/frontend/transport/roundtripper.go
@@ -6,6 +6,7 @@ import (
 	"io"
 	"net/http"
 	"net/url"
+	"time"
 
 	"github.com/weaveworks/common/httpgrpc"
 	"github.com/weaveworks/common/httpgrpc/server"
@@ -13,7 +14,7 @@ import (
 
 // GrpcRoundTripper is similar to http.RoundTripper, but works with HTTP requests converted to protobuf messages.
 type GrpcRoundTripper interface {
-	RoundTripGRPC(context.Context, url.Values, *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error)
+	RoundTripGRPC(context.Context, url.Values, time.Time, *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error)
 }
 
 func AdaptGrpcRoundTripperToHTTPRoundTripper(r GrpcRoundTripper) http.RoundTripper {
@@ -40,7 +41,7 @@ func (a *grpcRoundTripperAdapter) RoundTrip(r *http.Request) (*http.Response, er
 		return nil, err
 	}
 
-	resp, err := a.roundTripper.RoundTripGRPC(r.Context(), r.Form, req)
+	resp, err := a.roundTripper.RoundTripGRPC(r.Context(), r.Form, time.Now(), req)
 	if err != nil {
 		return nil, err
 	}
diff --git a/pkg/frontend/v1/frontend.go b/pkg/frontend/v1/frontend.go
index 64e19f35cb..499790f3af 100644
--- a/pkg/frontend/v1/frontend.go
+++ b/pkg/frontend/v1/frontend.go
@@ -177,7 +177,7 @@ func (f *Frontend) cleanupInactiveUserMetrics(user string) {
 }
 
 // RoundTripGRPC round trips a proto (instead of a HTTP request).
-func (f *Frontend) RoundTripGRPC(ctx context.Context, requestParams url.Values, req *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error) {
+func (f *Frontend) RoundTripGRPC(ctx context.Context, requestParams url.Values, timestamp time.Time, req *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error) {
 	// Propagate trace context in gRPC too - this will be ignored if using HTTP.
 	tracer, span := opentracing.GlobalTracer(), opentracing.SpanFromContext(ctx)
 	if tracer != nil && span != nil {
@@ -198,7 +198,7 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, requestParams url.Values,
 		request := request{
 			request:        req,
 			originalCtx:    ctx,
-			isHighPriority: util_query.IsHighPriority(requestParams, f.limits.HighPriorityQueries(userID)),
+			isHighPriority: util_query.IsHighPriority(requestParams, timestamp, f.limits.HighPriorityQueries(userID)),
 
 			// Buffer of 1 to ensure response can be written by the server side
 			// of the Process stream, even if this goroutine goes away due to
diff --git a/pkg/frontend/v2/frontend.go b/pkg/frontend/v2/frontend.go
index 3f0b8d3372..447f797a43 100644
--- a/pkg/frontend/v2/frontend.go
+++ b/pkg/frontend/v2/frontend.go
@@ -171,7 +171,7 @@ func (f *Frontend) stopping(_ error) error {
 }
 
 // RoundTripGRPC round trips a proto (instead of a HTTP request).
-func (f *Frontend) RoundTripGRPC(ctx context.Context, requestParams url.Values, req *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error) {
+func (f *Frontend) RoundTripGRPC(ctx context.Context, requestParams url.Values, timestamp time.Time, req *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error) {
 	if s := f.State(); s != services.Running {
 		return nil, fmt.Errorf("frontend not running: %v", s)
 	}
@@ -200,7 +200,7 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, requestParams url.Values,
 			request:        req,
 			userID:         userID,
 			statsEnabled:   stats.IsEnabled(ctx),
-			isHighPriority: util_query.IsHighPriority(requestParams, f.limits.HighPriorityQueries(userID)),
+			isHighPriority: util_query.IsHighPriority(requestParams, timestamp, f.limits.HighPriorityQueries(userID)),
 
 			cancel: cancel,
 
diff --git a/pkg/frontend/v2/frontend_test.go b/pkg/frontend/v2/frontend_test.go
index 68023f9f57..3b7b9337eb 100644
--- a/pkg/frontend/v2/frontend_test.go
+++ b/pkg/frontend/v2/frontend_test.go
@@ -113,7 +113,7 @@ func TestFrontendBasicWorkflow(t *testing.T) {
 		return &schedulerpb.SchedulerToFrontend{Status: schedulerpb.OK}
 	}, 0)
 
-	resp, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), url.Values{}, &httpgrpc.HTTPRequest{})
+	resp, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), url.Values{}, time.Now(), &httpgrpc.HTTPRequest{})
 	require.NoError(t, err)
 	require.Equal(t, int32(200), resp.Code)
 	require.Equal(t, []byte(body), resp.Body)
@@ -143,7 +143,7 @@ func TestFrontendRetryRequest(t *testing.T) {
 		return &schedulerpb.SchedulerToFrontend{Status: schedulerpb.OK}
 	}, 3)
 
-	res, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), url.Values{}, &httpgrpc.HTTPRequest{})
+	res, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), url.Values{}, time.Now(), &httpgrpc.HTTPRequest{})
 	require.NoError(t, err)
 	require.Equal(t, int32(200), res.Code)
 }
@@ -170,7 +170,7 @@ func TestFrontendRetryEnqueue(t *testing.T) {
 		return &schedulerpb.SchedulerToFrontend{Status: schedulerpb.OK}
 	}, 0)
 
-	_, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), url.Values{}, &httpgrpc.HTTPRequest{})
+	_, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), url.Values{}, time.Now(), &httpgrpc.HTTPRequest{})
 	require.NoError(t, err)
 }
 
@@ -179,7 +179,7 @@ func TestFrontendEnqueueFailure(t *testing.T) {
 		return &schedulerpb.SchedulerToFrontend{Status: schedulerpb.SHUTTING_DOWN}
 	}, 0)
 
-	_, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), "test"), url.Values{}, &httpgrpc.HTTPRequest{})
+	_, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), "test"), url.Values{}, time.Now(), &httpgrpc.HTTPRequest{})
 	require.Error(t, err)
 	require.True(t, strings.Contains(err.Error(), "failed to enqueue request"))
 }
@@ -190,7 +190,7 @@ func TestFrontendCancellation(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
 	defer cancel()
 
-	resp, err := f.RoundTripGRPC(user.InjectOrgID(ctx, "test"), url.Values{}, &httpgrpc.HTTPRequest{})
+	resp, err := f.RoundTripGRPC(user.InjectOrgID(ctx, "test"), url.Values{}, time.Now(), &httpgrpc.HTTPRequest{})
 	require.EqualError(t, err, context.DeadlineExceeded.Error())
 	require.Nil(t, resp)
 
@@ -239,7 +239,7 @@ func TestFrontendFailedCancellation(t *testing.T) {
 	}()
 
 	// send request
-	resp, err := f.RoundTripGRPC(user.InjectOrgID(ctx, "test"), url.Values{}, &httpgrpc.HTTPRequest{})
+	resp, err := f.RoundTripGRPC(user.InjectOrgID(ctx, "test"), url.Values{}, time.Now(), &httpgrpc.HTTPRequest{})
 	require.EqualError(t, err, context.Canceled.Error())
 	require.Nil(t, resp)
 
diff --git a/pkg/util/query/priority.go b/pkg/util/query/priority.go
index 7347ce04d1..45d36f4270 100644
--- a/pkg/util/query/priority.go
+++ b/pkg/util/query/priority.go
@@ -9,7 +9,7 @@ import (
 	"github.com/cortexproject/cortex/pkg/util/validation"
 )
 
-func IsHighPriority(requestParams url.Values, highPriorityQueries []validation.HighPriorityQuery) bool {
+func IsHighPriority(requestParams url.Values, timestamp time.Time, highPriorityQueries []validation.HighPriorityQuery) bool {
 	queryParam := requestParams.Get("query")
 	timeParam := requestParams.Get("time")
 	startParam := requestParams.Get("start")
@@ -22,12 +22,11 @@ func IsHighPriority(requestParams url.Values, highPriorityQueries []validation.H
 			continue
 		}
 
-		now := time.Now()
-		startTimeThreshold := now.Add(-1 * highPriorityQuery.StartTime).UnixMilli()
-		endTimeThreshold := now.Add(-1 * highPriorityQuery.EndTime).UnixMilli()
+		startTimeThreshold := timestamp.Add(-1 * highPriorityQuery.StartTime.Abs()).UnixMilli()
+		endTimeThreshold := timestamp.Add(-1 * highPriorityQuery.EndTime.Abs()).UnixMilli()
 
-		if time, err := strconv.ParseInt(timeParam, 10, 64); err == nil {
-			if isBetweenThresholds(time, time, startTimeThreshold, endTimeThreshold) {
+		if instantTime, err := strconv.ParseInt(timeParam, 10, 64); err == nil {
+			if isBetweenThresholds(instantTime, instantTime, startTimeThreshold, endTimeThreshold) {
 				return true
 			}
 		}
diff --git a/pkg/util/query/priority_test.go b/pkg/util/query/priority_test.go
index 2e7a414487..dd1ac9c78b 100644
--- a/pkg/util/query/priority_test.go
+++ b/pkg/util/query/priority_test.go
@@ -11,22 +11,6 @@ import (
 	"github.com/cortexproject/cortex/pkg/util/validation"
 )
 
-func Test_IsHighPriority_DefaultValues(t *testing.T) {
-	now := time.Now()
-	config := []validation.HighPriorityQuery{
-		{}, // By default, it should match all queries happened at "now"
-	}
-
-	assert.True(t, IsHighPriority(url.Values{
-		"query": []string{"count(up)"},
-		"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
-	}, config))
-	assert.False(t, IsHighPriority(url.Values{
-		"query": []string{"count(up)"},
-		"time":  []string{strconv.FormatInt(now.Add(-1*time.Second).UnixMilli(), 10)},
-	}, config))
-}
-
 func Test_IsHighPriority_ShouldMatchRegex(t *testing.T) {
 	now := time.Now()
 	config := []validation.HighPriorityQuery{
@@ -38,11 +22,11 @@ func Test_IsHighPriority_ShouldMatchRegex(t *testing.T) {
 	assert.True(t, IsHighPriority(url.Values{
 		"query": []string{"sum(up)"},
 		"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
-	}, config))
+	}, now, config))
 	assert.False(t, IsHighPriority(url.Values{
 		"query": []string{"count(up)"},
 		"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
-	}, config))
+	}, now, config))
 
 	config = []validation.HighPriorityQuery{
 		{
@@ -53,11 +37,11 @@ func Test_IsHighPriority_ShouldMatchRegex(t *testing.T) {
 	assert.True(t, IsHighPriority(url.Values{
 		"query": []string{"sum(up)"},
 		"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
-	}, config))
+	}, now, config))
 	assert.True(t, IsHighPriority(url.Values{
 		"query": []string{"count(up)"},
 		"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
-	}, config))
+	}, now, config))
 
 	config = []validation.HighPriorityQuery{
 		{
@@ -71,11 +55,11 @@ func Test_IsHighPriority_ShouldMatchRegex(t *testing.T) {
 	assert.True(t, IsHighPriority(url.Values{
 		"query": []string{"sum(up)"},
 		"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
-	}, config))
+	}, now, config))
 	assert.True(t, IsHighPriority(url.Values{
 		"query": []string{"count(up)"},
 		"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
-	}, config))
+	}, now, config))
 
 	config = []validation.HighPriorityQuery{
 		{
@@ -89,11 +73,11 @@ func Test_IsHighPriority_ShouldMatchRegex(t *testing.T) {
 	assert.False(t, IsHighPriority(url.Values{
 		"query": []string{"sum(up)"},
 		"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
-	}, config))
+	}, now, config))
 	assert.False(t, IsHighPriority(url.Values{
 		"query": []string{"count(up)"},
 		"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
-	}, config))
+	}, now, config))
 
 	config = []validation.HighPriorityQuery{
 		{
@@ -104,11 +88,26 @@ func Test_IsHighPriority_ShouldMatchRegex(t *testing.T) {
 	assert.True(t, IsHighPriority(url.Values{
 		"query": []string{"sum(up)"},
 		"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
-	}, config))
+	}, now, config))
+	assert.True(t, IsHighPriority(url.Values{
+		"query": []string{"count(up)"},
+		"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
+	}, now, config))
+
+	config = []validation.HighPriorityQuery{
+		{
+			Regex: "",
+		},
+	}
+
+	assert.True(t, IsHighPriority(url.Values{
+		"query": []string{"sum(up)"},
+		"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
+	}, now, config))
 	assert.True(t, IsHighPriority(url.Values{
 		"query": []string{"count(up)"},
 		"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
-	}, config))
+	}, now, config))
 }
 
 func Test_IsHighPriority_ShouldBeBetweenStartAndEndTime(t *testing.T) {
@@ -123,32 +122,32 @@ func Test_IsHighPriority_ShouldBeBetweenStartAndEndTime(t *testing.T) {
 	assert.False(t, IsHighPriority(url.Values{
 		"query": []string{"sum(up)"},
 		"time":  []string{strconv.FormatInt(now.Add(-2*time.Hour).UnixMilli(), 10)},
-	}, config))
+	}, now, config))
 	assert.True(t, IsHighPriority(url.Values{
 		"query": []string{"sum(up)"},
 		"time":  []string{strconv.FormatInt(now.Add(-1*time.Hour).UnixMilli(), 10)},
-	}, config))
+	}, now, config))
 	assert.True(t, IsHighPriority(url.Values{
 		"query": []string{"sum(up)"},
 		"time":  []string{strconv.FormatInt(now.Add(-30*time.Minute).UnixMilli(), 10)},
-	}, config))
+	}, now, config))
 	assert.False(t, IsHighPriority(url.Values{
 		"query": []string{"sum(up)"},
 		"time":  []string{strconv.FormatInt(now.Add(-1*time.Minute).UnixMilli(), 10)},
-	}, config))
+	}, now, config))
 	assert.False(t, IsHighPriority(url.Values{
 		"query": []string{"sum(up)"},
 		"start": []string{strconv.FormatInt(now.Add(-2*time.Hour).UnixMilli(), 10)},
 		"end":   []string{strconv.FormatInt(now.Add(-30*time.Minute).UnixMilli(), 10)},
-	}, config))
+	}, now, config))
 	assert.True(t, IsHighPriority(url.Values{
 		"query": []string{"sum(up)"},
 		"start": []string{strconv.FormatInt(now.Add(-1*time.Hour).UnixMilli(), 10)},
 		"end":   []string{strconv.FormatInt(now.Add(-30*time.Minute).UnixMilli(), 10)},
-	}, config))
+	}, now, config))
 	assert.False(t, IsHighPriority(url.Values{
 		"query": []string{"sum(up)"},
 		"start": []string{strconv.FormatInt(now.Add(-1*time.Hour).UnixMilli(), 10)},
 		"end":   []string{strconv.FormatInt(now.UnixMilli(), 10)},
-	}, config))
+	}, now, config))
 }

From daf269bfbb6ea44fd49de66e49589cfd7907f647 Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Wed, 25 Oct 2023 14:11:18 -0700
Subject: [PATCH 14/49] Parse form so that range query parameters are passed to
 the roundtrip

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 pkg/frontend/transport/roundtripper.go |  4 +++
 pkg/util/query/priority.go             | 38 ++++++++++++++++++++------
 2 files changed, 34 insertions(+), 8 deletions(-)

diff --git a/pkg/frontend/transport/roundtripper.go b/pkg/frontend/transport/roundtripper.go
index ccbd8d4cc5..cc653de3b6 100644
--- a/pkg/frontend/transport/roundtripper.go
+++ b/pkg/frontend/transport/roundtripper.go
@@ -41,6 +41,10 @@ func (a *grpcRoundTripperAdapter) RoundTrip(r *http.Request) (*http.Response, er
 		return nil, err
 	}
 
+	if err = r.ParseForm(); err != nil {
+		return nil, err
+	}
+
 	resp, err := a.roundTripper.RoundTripGRPC(r.Context(), r.Form, time.Now(), req)
 	if err != nil {
 		return nil, err
diff --git a/pkg/util/query/priority.go b/pkg/util/query/priority.go
index 45d36f4270..84c7dfa262 100644
--- a/pkg/util/query/priority.go
+++ b/pkg/util/query/priority.go
@@ -1,20 +1,27 @@
 package query
 
 import (
+	"math"
 	"net/url"
 	"regexp"
 	"strconv"
 	"time"
 
+	"github.com/pkg/errors"
+
 	"github.com/cortexproject/cortex/pkg/util/validation"
 )
 
-func IsHighPriority(requestParams url.Values, timestamp time.Time, highPriorityQueries []validation.HighPriorityQuery) bool {
+func IsHighPriority(requestParams url.Values, now time.Time, highPriorityQueries []validation.HighPriorityQuery) bool {
 	queryParam := requestParams.Get("query")
 	timeParam := requestParams.Get("time")
 	startParam := requestParams.Get("start")
 	endParam := requestParams.Get("end")
 
+	if queryParam == "" {
+		return false
+	}
+
 	for _, highPriorityQuery := range highPriorityQueries {
 		regex := highPriorityQuery.Regex
 
@@ -22,17 +29,17 @@ func IsHighPriority(requestParams url.Values, timestamp time.Time, highPriorityQ
 			continue
 		}
 
-		startTimeThreshold := timestamp.Add(-1 * highPriorityQuery.StartTime.Abs()).UnixMilli()
-		endTimeThreshold := timestamp.Add(-1 * highPriorityQuery.EndTime.Abs()).UnixMilli()
+		startTimeThreshold := now.Add(-1 * highPriorityQuery.StartTime.Abs())
+		endTimeThreshold := now.Add(-1 * highPriorityQuery.EndTime.Abs())
 
-		if instantTime, err := strconv.ParseInt(timeParam, 10, 64); err == nil {
+		if instantTime, err := parseTime(timeParam); err == nil {
 			if isBetweenThresholds(instantTime, instantTime, startTimeThreshold, endTimeThreshold) {
 				return true
 			}
 		}
 
-		if startTime, err := strconv.ParseInt(startParam, 10, 64); err == nil {
-			if endTime, err := strconv.ParseInt(endParam, 10, 64); err == nil {
+		if startTime, err := parseTime(startParam); err == nil {
+			if endTime, err := parseTime(endParam); err == nil {
 				if isBetweenThresholds(startTime, endTime, startTimeThreshold, endTimeThreshold) {
 					return true
 				}
@@ -43,6 +50,21 @@ func IsHighPriority(requestParams url.Values, timestamp time.Time, highPriorityQ
 	return false
 }
 
-func isBetweenThresholds(start, end, startThreshold, endThreshold int64) bool {
-	return start >= startThreshold && end <= endThreshold
+func parseTime(s string) (time.Time, error) {
+	if s != "" {
+		if t, err := strconv.ParseFloat(s, 64); err == nil {
+			s, ns := math.Modf(t)
+			ns = math.Round(ns*1000) / 1000
+			return time.Unix(int64(s), int64(ns*float64(time.Second))).UTC(), nil
+		}
+		if t, err := time.Parse(time.RFC3339Nano, s); err == nil {
+			return t, nil
+		}
+	}
+
+	return time.Time{}, errors.Errorf("cannot parse %q to a valid timestamp", s)
+}
+
+func isBetweenThresholds(start, end, startThreshold, endThreshold time.Time) bool {
+	return start.After(startThreshold) && end.Before(endThreshold)
 }

From 3b603ba8d15b26b96cd54a4e0cf6d1baba27c081 Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Sun, 29 Oct 2023 14:33:12 -0700
Subject: [PATCH 15/49] Address comments

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 pkg/frontend/transport/roundtripper.go       |  22 +++-
 pkg/frontend/v1/frontend.go                  |  21 ++--
 pkg/frontend/v2/frontend.go                  |  28 ++---
 pkg/frontend/v2/frontend_scheduler_worker.go |   2 +-
 pkg/scheduler/queue/queue_test.go            |   2 +-
 pkg/scheduler/queue/user_queues.go           |   6 +-
 pkg/scheduler/scheduler.go                   |   6 +-
 pkg/scheduler/schedulerpb/scheduler.pb.go    | 114 +++++++++----------
 pkg/scheduler/schedulerpb/scheduler.proto    |   2 +-
 pkg/util/query/priority_test.go              |   4 +-
 pkg/util/validation/limits.go                |   8 +-
 11 files changed, 118 insertions(+), 97 deletions(-)

diff --git a/pkg/frontend/transport/roundtripper.go b/pkg/frontend/transport/roundtripper.go
index cc653de3b6..aaa6f306d1 100644
--- a/pkg/frontend/transport/roundtripper.go
+++ b/pkg/frontend/transport/roundtripper.go
@@ -3,9 +3,12 @@ package transport
 import (
 	"bytes"
 	"context"
+	"fmt"
 	"io"
 	"net/http"
 	"net/url"
+	"regexp"
+	"strings"
 	"time"
 
 	"github.com/weaveworks/common/httpgrpc"
@@ -14,7 +17,7 @@ import (
 
 // GrpcRoundTripper is similar to http.RoundTripper, but works with HTTP requests converted to protobuf messages.
 type GrpcRoundTripper interface {
-	RoundTripGRPC(context.Context, url.Values, time.Time, *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error)
+	RoundTripGRPC(context.Context, *httpgrpc.HTTPRequest, url.Values, time.Time) (*httpgrpc.HTTPResponse, error)
 }
 
 func AdaptGrpcRoundTripperToHTTPRoundTripper(r GrpcRoundTripper) http.RoundTripper {
@@ -36,16 +39,27 @@ func (b *buffer) Bytes() []byte {
 }
 
 func (a *grpcRoundTripperAdapter) RoundTrip(r *http.Request) (*http.Response, error) {
+	regexp.MustCompile("str")
 	req, err := server.HTTPRequest(r)
 	if err != nil {
 		return nil, err
 	}
+	fmt.Println(r.URL.Path)
 
-	if err = r.ParseForm(); err != nil {
-		return nil, err
+	var (
+		resp      *httpgrpc.HTTPResponse
+		reqValues url.Values
+		ts        time.Time
+	)
+
+	if strings.HasSuffix(r.URL.Path, "/query") || strings.HasSuffix(r.URL.Path, "/query_range") {
+		if err = r.ParseForm(); err == nil {
+			reqValues = r.Form
+			ts = time.Now()
+		}
 	}
 
-	resp, err := a.roundTripper.RoundTripGRPC(r.Context(), r.Form, time.Now(), req)
+	resp, err = a.roundTripper.RoundTripGRPC(r.Context(), req, reqValues, ts)
 	if err != nil {
 		return nil, err
 	}
diff --git a/pkg/frontend/v1/frontend.go b/pkg/frontend/v1/frontend.go
index 499790f3af..8f2fae830f 100644
--- a/pkg/frontend/v1/frontend.go
+++ b/pkg/frontend/v1/frontend.go
@@ -95,14 +95,14 @@ type request struct {
 	queueSpan   opentracing.Span
 	originalCtx context.Context
 
-	request        *httpgrpc.HTTPRequest
-	err            chan error
-	response       chan *httpgrpc.HTTPResponse
-	isHighPriority bool
+	request      *httpgrpc.HTTPRequest
+	err          chan error
+	response     chan *httpgrpc.HTTPResponse
+	highPriority bool
 }
 
 func (r request) IsHighPriority() bool {
-	return r.isHighPriority
+	return r.highPriority
 }
 
 // New creates a new frontend. Frontend implements service, and must be started and stopped.
@@ -177,7 +177,7 @@ func (f *Frontend) cleanupInactiveUserMetrics(user string) {
 }
 
 // RoundTripGRPC round trips a proto (instead of a HTTP request).
-func (f *Frontend) RoundTripGRPC(ctx context.Context, requestParams url.Values, timestamp time.Time, req *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error) {
+func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest, reqParams url.Values, ts time.Time) (*httpgrpc.HTTPResponse, error) {
 	// Propagate trace context in gRPC too - this will be ignored if using HTTP.
 	tracer, span := opentracing.GlobalTracer(), opentracing.SpanFromContext(ctx)
 	if tracer != nil && span != nil {
@@ -196,9 +196,8 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, requestParams url.Values,
 
 	return f.retry.Do(ctx, func() (*httpgrpc.HTTPResponse, error) {
 		request := request{
-			request:        req,
-			originalCtx:    ctx,
-			isHighPriority: util_query.IsHighPriority(requestParams, timestamp, f.limits.HighPriorityQueries(userID)),
+			request:     req,
+			originalCtx: ctx,
 
 			// Buffer of 1 to ensure response can be written by the server side
 			// of the Process stream, even if this goroutine goes away due to
@@ -207,6 +206,10 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, requestParams url.Values,
 			response: make(chan *httpgrpc.HTTPResponse, 1),
 		}
 
+		if reqParams != nil {
+			request.highPriority = util_query.IsHighPriority(reqParams, ts, f.limits.HighPriorityQueries(userID))
+		}
+
 		if err := f.queueRequest(ctx, &request); err != nil {
 			return nil, err
 		}
diff --git a/pkg/frontend/v2/frontend.go b/pkg/frontend/v2/frontend.go
index 447f797a43..3cb00bdbca 100644
--- a/pkg/frontend/v2/frontend.go
+++ b/pkg/frontend/v2/frontend.go
@@ -10,8 +10,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/cortexproject/cortex/pkg/scheduler"
-
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 	"github.com/opentracing/opentracing-go"
@@ -24,6 +22,7 @@ import (
 	"github.com/cortexproject/cortex/pkg/frontend/transport"
 	"github.com/cortexproject/cortex/pkg/frontend/v2/frontendv2pb"
 	"github.com/cortexproject/cortex/pkg/querier/stats"
+	"github.com/cortexproject/cortex/pkg/scheduler"
 	"github.com/cortexproject/cortex/pkg/tenant"
 	"github.com/cortexproject/cortex/pkg/util/flagext"
 	"github.com/cortexproject/cortex/pkg/util/grpcclient"
@@ -86,11 +85,11 @@ type Frontend struct {
 }
 
 type frontendRequest struct {
-	queryID        uint64
-	request        *httpgrpc.HTTPRequest
-	userID         string
-	statsEnabled   bool
-	isHighPriority bool
+	queryID      uint64
+	request      *httpgrpc.HTTPRequest
+	userID       string
+	statsEnabled bool
+	highPriority bool
 
 	cancel context.CancelFunc
 
@@ -171,7 +170,7 @@ func (f *Frontend) stopping(_ error) error {
 }
 
 // RoundTripGRPC round trips a proto (instead of a HTTP request).
-func (f *Frontend) RoundTripGRPC(ctx context.Context, requestParams url.Values, timestamp time.Time, req *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error) {
+func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest, reqParams url.Values, ts time.Time) (*httpgrpc.HTTPResponse, error) {
 	if s := f.State(); s != services.Running {
 		return nil, fmt.Errorf("frontend not running: %v", s)
 	}
@@ -196,11 +195,10 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, requestParams url.Values,
 
 	return f.retry.Do(ctx, func() (*httpgrpc.HTTPResponse, error) {
 		freq := &frontendRequest{
-			queryID:        f.lastQueryID.Inc(),
-			request:        req,
-			userID:         userID,
-			statsEnabled:   stats.IsEnabled(ctx),
-			isHighPriority: util_query.IsHighPriority(requestParams, timestamp, f.limits.HighPriorityQueries(userID)),
+			queryID:      f.lastQueryID.Inc(),
+			request:      req,
+			userID:       userID,
+			statsEnabled: stats.IsEnabled(ctx),
 
 			cancel: cancel,
 
@@ -212,6 +210,10 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, requestParams url.Values,
 			retryOnTooManyOutstandingRequests: f.cfg.RetryOnTooManyOutstandingRequests && f.schedulerWorkers.getWorkersCount() > 1,
 		}
 
+		if reqParams != nil {
+			freq.highPriority = util_query.IsHighPriority(reqParams, ts, f.limits.HighPriorityQueries(userID))
+		}
+
 		f.requests.put(freq)
 		defer f.requests.delete(freq.queryID)
 
diff --git a/pkg/frontend/v2/frontend_scheduler_worker.go b/pkg/frontend/v2/frontend_scheduler_worker.go
index 5e5b22eb13..557457fdc0 100644
--- a/pkg/frontend/v2/frontend_scheduler_worker.go
+++ b/pkg/frontend/v2/frontend_scheduler_worker.go
@@ -263,7 +263,7 @@ func (w *frontendSchedulerWorker) schedulerLoop(loop schedulerpb.SchedulerForFro
 				HttpRequest:     req.request,
 				FrontendAddress: w.frontendAddr,
 				StatsEnabled:    req.statsEnabled,
-				IsHighPriority:  req.isHighPriority,
+				HighPriority:    req.highPriority,
 			})
 
 			if err != nil {
diff --git a/pkg/scheduler/queue/queue_test.go b/pkg/scheduler/queue/queue_test.go
index 283eb9df4f..9dd1ee78eb 100644
--- a/pkg/scheduler/queue/queue_test.go
+++ b/pkg/scheduler/queue/queue_test.go
@@ -174,7 +174,7 @@ func TestRequestQueue_QueriersShouldGetHighPriorityQueryFirst(t *testing.T) {
 		isHighPriority: false,
 	}
 	normalRequest2 := MockRequest{
-		id:             "normal query 1",
+		id:             "normal query 2",
 		isHighPriority: false,
 	}
 	highPriorityRequest := MockRequest{
diff --git a/pkg/scheduler/queue/user_queues.go b/pkg/scheduler/queue/user_queues.go
index 29194b613b..a437afce83 100644
--- a/pkg/scheduler/queue/user_queues.go
+++ b/pkg/scheduler/queue/user_queues.go
@@ -20,7 +20,7 @@ type Limits interface {
 	// ReservedHighPriorityQueriers returns the minimum number of queriers dedicated for high priority
 	// queue per tenant. All queriers still handle priority queue first, but this provides extra protection on
 	// high priority queries from slow normal queries exhausting all queriers.
-	// If ReservedHighPriorityQueriers is capped by MaxQueriersPerUser.
+	// ReservedHighPriorityQueriers is capped by MaxQueriersPerUser.
 	// If less than 1, it will be applied as a percentage of MaxQueriersPerUser.
 	ReservedHighPriorityQueriers(user string) float64
 
@@ -118,7 +118,7 @@ func (q *queues) deleteQueue(userID string) {
 // MaxQueriers is used to compute which queriers should handle requests for this user.
 // If maxQueriers is <= 0, all queriers can handle this user's requests.
 // If maxQueriers has changed since the last call, queriers for this are recomputed.
-func (q *queues) getOrAddQueue(userID string, maxQueriers int, isHighPriority bool) chan Request {
+func (q *queues) getOrAddQueue(userID string, maxQueriers int, highPriority bool) chan Request {
 	// Empty user is not allowed, as that would break our users list ("" is used for free spot).
 	if userID == "" {
 		return nil
@@ -166,7 +166,7 @@ func (q *queues) getOrAddQueue(userID string, maxQueriers int, isHighPriority bo
 		uq.queriers, uq.reservedQueriers = shuffleQueriersForUser(uq.seed, maxQueriers, q.sortedQueriers, q.limits.ReservedHighPriorityQueriers(userID), nil)
 	}
 
-	if isHighPriority {
+	if highPriority {
 		return uq.highPriorityQueue
 	}
 
diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go
index 12b5575197..cbd673badf 100644
--- a/pkg/scheduler/scheduler.go
+++ b/pkg/scheduler/scheduler.go
@@ -154,7 +154,7 @@ type schedulerRequest struct {
 	queryID         uint64
 	request         *httpgrpc.HTTPRequest
 	statsEnabled    bool
-	isHighPriority  bool
+	highPriority    bool
 
 	enqueueTime time.Time
 
@@ -167,7 +167,7 @@ type schedulerRequest struct {
 }
 
 func (s schedulerRequest) IsHighPriority() bool {
-	return s.isHighPriority
+	return s.highPriority
 }
 
 // FrontendLoop handles connection from frontend.
@@ -298,7 +298,7 @@ func (s *Scheduler) enqueueRequest(frontendContext context.Context, frontendAddr
 		queryID:         msg.QueryID,
 		request:         msg.HttpRequest,
 		statsEnabled:    msg.StatsEnabled,
-		isHighPriority:  msg.IsHighPriority,
+		highPriority:    msg.HighPriority,
 	}
 
 	now := time.Now()
diff --git a/pkg/scheduler/schedulerpb/scheduler.pb.go b/pkg/scheduler/schedulerpb/scheduler.pb.go
index 0d5c95d68e..5933f21c77 100644
--- a/pkg/scheduler/schedulerpb/scheduler.pb.go
+++ b/pkg/scheduler/schedulerpb/scheduler.pb.go
@@ -216,10 +216,10 @@ type FrontendToScheduler struct {
 	// Each frontend manages its own queryIDs. Different frontends may use same set of query IDs.
 	QueryID uint64 `protobuf:"varint,3,opt,name=queryID,proto3" json:"queryID,omitempty"`
 	// Following are used by ENQUEUE only.
-	UserID         string                `protobuf:"bytes,4,opt,name=userID,proto3" json:"userID,omitempty"`
-	HttpRequest    *httpgrpc.HTTPRequest `protobuf:"bytes,5,opt,name=httpRequest,proto3" json:"httpRequest,omitempty"`
-	StatsEnabled   bool                  `protobuf:"varint,6,opt,name=statsEnabled,proto3" json:"statsEnabled,omitempty"`
-	IsHighPriority bool                  `protobuf:"varint,7,opt,name=isHighPriority,proto3" json:"isHighPriority,omitempty"`
+	UserID       string                `protobuf:"bytes,4,opt,name=userID,proto3" json:"userID,omitempty"`
+	HttpRequest  *httpgrpc.HTTPRequest `protobuf:"bytes,5,opt,name=httpRequest,proto3" json:"httpRequest,omitempty"`
+	StatsEnabled bool                  `protobuf:"varint,6,opt,name=statsEnabled,proto3" json:"statsEnabled,omitempty"`
+	HighPriority bool                  `protobuf:"varint,7,opt,name=highPriority,proto3" json:"highPriority,omitempty"`
 }
 
 func (m *FrontendToScheduler) Reset()      { *m = FrontendToScheduler{} }
@@ -296,9 +296,9 @@ func (m *FrontendToScheduler) GetStatsEnabled() bool {
 	return false
 }
 
-func (m *FrontendToScheduler) GetIsHighPriority() bool {
+func (m *FrontendToScheduler) GetHighPriority() bool {
 	if m != nil {
-		return m.IsHighPriority
+		return m.HighPriority
 	}
 	return false
 }
@@ -446,49 +446,49 @@ func init() {
 func init() { proto.RegisterFile("scheduler.proto", fileDescriptor_2b3fc28395a6d9c5) }
 
 var fileDescriptor_2b3fc28395a6d9c5 = []byte{
-	// 666 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x94, 0xcf, 0x4e, 0xdb, 0x40,
-	0x10, 0xc6, 0xbd, 0x21, 0x09, 0x30, 0xa1, 0xe0, 0x2e, 0xd0, 0xa6, 0x11, 0x5d, 0x22, 0xab, 0x42,
-	0x29, 0x87, 0xa4, 0x4a, 0x2b, 0xb5, 0x07, 0x54, 0x29, 0x05, 0x53, 0xa2, 0x52, 0x07, 0x1c, 0x47,
-	0xfd, 0x73, 0x89, 0x48, 0xb2, 0x24, 0x16, 0xe0, 0x35, 0xeb, 0x75, 0x51, 0x6e, 0x7d, 0x84, 0x3e,
-	0x46, 0x8f, 0x7d, 0x8c, 0x5e, 0x2a, 0x71, 0xe4, 0xd0, 0x43, 0x71, 0x2e, 0x3d, 0xf2, 0x08, 0x55,
-	0x1c, 0x27, 0x75, 0x42, 0x02, 0xdc, 0x66, 0xc7, 0xdf, 0x67, 0xcf, 0xfc, 0x66, 0xd6, 0xb0, 0xe0,
-	0xd4, 0x5b, 0xb4, 0xe1, 0x1e, 0x53, 0x9e, 0xb5, 0x39, 0x13, 0x0c, 0x27, 0x06, 0x09, 0xbb, 0x96,
-	0x5a, 0x6a, 0xb2, 0x26, 0xf3, 0xf3, 0xb9, 0x6e, 0xd4, 0x93, 0xa4, 0x5e, 0x34, 0x4d, 0xd1, 0x72,
-	0x6b, 0xd9, 0x3a, 0x3b, 0xc9, 0x9d, 0xd1, 0x83, 0x2f, 0xf4, 0x8c, 0xf1, 0x23, 0x27, 0x57, 0x67,
-	0x27, 0x27, 0xcc, 0xca, 0xb5, 0x84, 0xb0, 0x9b, 0xdc, 0xae, 0x0f, 0x82, 0x9e, 0x4b, 0xc9, 0x03,
-	0xde, 0x77, 0x29, 0x37, 0x29, 0x37, 0x58, 0xb9, 0xff, 0x0d, 0xbc, 0x02, 0xb3, 0xa7, 0xbd, 0x6c,
-	0x71, 0x2b, 0x89, 0xd2, 0x28, 0x33, 0xab, 0xff, 0x4f, 0x28, 0xbf, 0x10, 0xe0, 0x81, 0xd6, 0x60,
-	0x81, 0x1f, 0x27, 0x61, 0xba, 0xab, 0x69, 0x07, 0x96, 0xa8, 0xde, 0x3f, 0xe2, 0x97, 0x90, 0xe8,
-	0x7e, 0x56, 0xa7, 0xa7, 0x2e, 0x75, 0x44, 0x32, 0x92, 0x46, 0x99, 0x44, 0x7e, 0x39, 0x3b, 0x28,
-	0x65, 0xc7, 0x30, 0xf6, 0x82, 0x87, 0x7a, 0x58, 0x89, 0x33, 0xb0, 0x70, 0xc8, 0x99, 0x25, 0xa8,
-	0xd5, 0x28, 0x34, 0x1a, 0x9c, 0x3a, 0x4e, 0x72, 0xca, 0xaf, 0x66, 0x34, 0x8d, 0x1f, 0x40, 0xdc,
-	0x75, 0xfc, 0x72, 0xa3, 0xbe, 0x20, 0x38, 0x61, 0x05, 0xe6, 0x1c, 0x71, 0x20, 0x1c, 0xd5, 0x3a,
-	0xa8, 0x1d, 0xd3, 0x46, 0x32, 0x96, 0x46, 0x99, 0x19, 0x7d, 0x28, 0xa7, 0xfc, 0x88, 0xc0, 0xe2,
-	0x76, 0xf0, 0xbe, 0x30, 0x85, 0x57, 0x10, 0x15, 0x6d, 0x9b, 0xfa, 0xdd, 0xcc, 0xe7, 0x9f, 0x64,
-	0x43, 0x33, 0xc8, 0x8e, 0xd1, 0x1b, 0x6d, 0x9b, 0xea, 0xbe, 0x63, 0x5c, 0xdd, 0x91, 0xf1, 0x75,
-	0x87, 0xa0, 0x4d, 0x0d, 0x43, 0x9b, 0xd4, 0xd1, 0x08, 0xcc, 0xd8, 0x9d, 0x61, 0x8e, 0xa2, 0x88,
-	0x5f, 0x47, 0x81, 0xd7, 0x60, 0xde, 0x74, 0x76, 0xcc, 0x66, 0x6b, 0x8f, 0x9b, 0x8c, 0x9b, 0xa2,
-	0x9d, 0x9c, 0xf6, 0x55, 0x23, 0x59, 0xe5, 0x08, 0x16, 0x43, 0x1b, 0xd0, 0x87, 0x81, 0x5f, 0x43,
-	0xbc, 0xfb, 0x3a, 0xd7, 0x09, 0x98, 0xad, 0x0d, 0x31, 0x1b, 0xe3, 0x28, 0xfb, 0x6a, 0x3d, 0x70,
-	0xe1, 0x25, 0x88, 0x51, 0xce, 0x19, 0x0f, 0x68, 0xf5, 0x0e, 0xca, 0x06, 0xac, 0x68, 0x4c, 0x98,
-	0x87, 0xed, 0x60, 0xd3, 0xca, 0x2d, 0x57, 0x34, 0xd8, 0x99, 0xd5, 0x6f, 0xec, 0xe6, 0x6d, 0x5d,
-	0x85, 0xc7, 0x13, 0xdc, 0x8e, 0xcd, 0x2c, 0x87, 0xae, 0x6f, 0xc0, 0xc3, 0x09, 0xd3, 0xc4, 0x33,
-	0x10, 0x2d, 0x6a, 0x45, 0x43, 0x96, 0x70, 0x02, 0xa6, 0x55, 0x6d, 0xbf, 0xa2, 0x56, 0x54, 0x19,
-	0x61, 0x80, 0xf8, 0x66, 0x41, 0xdb, 0x54, 0x77, 0xe5, 0xc8, 0x7a, 0x1d, 0x1e, 0x4d, 0xec, 0x0b,
-	0xc7, 0x21, 0x52, 0x7a, 0x27, 0x4b, 0x38, 0x0d, 0x2b, 0x46, 0xa9, 0x54, 0x7d, 0x5f, 0xd0, 0x3e,
-	0x55, 0x75, 0x75, 0xbf, 0xa2, 0x96, 0x8d, 0x72, 0x75, 0x4f, 0xd5, 0xab, 0x86, 0xaa, 0x15, 0x34,
-	0x43, 0x46, 0x78, 0x16, 0x62, 0xaa, 0xae, 0x97, 0x74, 0x39, 0x82, 0xef, 0xc3, 0xbd, 0xf2, 0x4e,
-	0xc5, 0x30, 0x8a, 0xda, 0xdb, 0xea, 0x56, 0xe9, 0x83, 0x26, 0x4f, 0xe5, 0x7f, 0xa3, 0x10, 0xef,
-	0x6d, 0xc6, 0xfb, 0x57, 0xae, 0x02, 0x89, 0x20, 0xdc, 0x65, 0xcc, 0xc6, 0xab, 0x43, 0xb8, 0xaf,
-	0xdf, 0xeb, 0xd4, 0xea, 0xa4, 0x79, 0x04, 0x5a, 0x45, 0xca, 0xa0, 0x67, 0x08, 0x5b, 0xb0, 0x3c,
-	0x16, 0x19, 0x7e, 0x3a, 0xe4, 0xbf, 0x69, 0x28, 0xa9, 0xf5, 0xbb, 0x48, 0x7b, 0x13, 0xc8, 0xdb,
-	0xb0, 0x14, 0xee, 0x6e, 0xb0, 0x4e, 0x1f, 0x61, 0xae, 0x1f, 0xfb, 0xfd, 0xa5, 0x6f, 0xbb, 0x82,
-	0xa9, 0xf4, 0x6d, 0x0b, 0xd7, 0xeb, 0xf0, 0x4d, 0xe1, 0xfc, 0x92, 0x48, 0x17, 0x97, 0x44, 0xba,
-	0xba, 0x24, 0xe8, 0xab, 0x47, 0xd0, 0x77, 0x8f, 0xa0, 0x9f, 0x1e, 0x41, 0xe7, 0x1e, 0x41, 0x7f,
-	0x3c, 0x82, 0xfe, 0x7a, 0x44, 0xba, 0xf2, 0x08, 0xfa, 0xd6, 0x21, 0xd2, 0x79, 0x87, 0x48, 0x17,
-	0x1d, 0x22, 0x7d, 0x0e, 0xff, 0x85, 0x6b, 0x71, 0xff, 0x07, 0xfa, 0xfc, 0x5f, 0x00, 0x00, 0x00,
-	0xff, 0xff, 0x13, 0x54, 0x85, 0x09, 0xac, 0x05, 0x00, 0x00,
+	// 662 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x94, 0xcd, 0x4e, 0xdb, 0x40,
+	0x10, 0xc7, 0xbd, 0x21, 0x09, 0x30, 0xa1, 0xc5, 0x5d, 0xa0, 0x4d, 0x23, 0xba, 0x44, 0x56, 0x55,
+	0xa5, 0x1c, 0x92, 0x2a, 0xad, 0xd4, 0x1e, 0x50, 0xa5, 0x14, 0x4c, 0x89, 0x4a, 0x1d, 0x70, 0x1c,
+	0xf5, 0xe3, 0x12, 0x91, 0x64, 0x49, 0x22, 0xc0, 0x6b, 0xd6, 0xeb, 0xa2, 0xdc, 0xfa, 0x08, 0x7d,
+	0x8c, 0x1e, 0xfa, 0x20, 0xbd, 0x54, 0xe2, 0xc8, 0xa1, 0x87, 0x62, 0x2e, 0x3d, 0xf2, 0x08, 0x55,
+	0xfc, 0x91, 0x3a, 0x21, 0x01, 0x6e, 0xb3, 0xe3, 0xff, 0xdf, 0x9e, 0xf9, 0xcd, 0xac, 0x61, 0xde,
+	0x6e, 0x76, 0x68, 0xcb, 0x39, 0xa4, 0x3c, 0x6f, 0x71, 0x26, 0x18, 0x4e, 0x0d, 0x12, 0x56, 0x23,
+	0xb3, 0xd8, 0x66, 0x6d, 0xe6, 0xe5, 0x0b, 0xfd, 0xc8, 0x97, 0x64, 0x5e, 0xb4, 0xbb, 0xa2, 0xe3,
+	0x34, 0xf2, 0x4d, 0x76, 0x54, 0x38, 0xa1, 0x7b, 0x5f, 0xe8, 0x09, 0xe3, 0x07, 0x76, 0xa1, 0xc9,
+	0x8e, 0x8e, 0x98, 0x59, 0xe8, 0x08, 0x61, 0xb5, 0xb9, 0xd5, 0x1c, 0x04, 0xbe, 0x4b, 0x29, 0x02,
+	0xde, 0x75, 0x28, 0xef, 0x52, 0x6e, 0xb0, 0x6a, 0xf8, 0x0d, 0xbc, 0x0c, 0xb3, 0xc7, 0x7e, 0xb6,
+	0xbc, 0x91, 0x46, 0x59, 0x94, 0x9b, 0xd5, 0xff, 0x27, 0x94, 0x5f, 0x08, 0xf0, 0x40, 0x6b, 0xb0,
+	0xc0, 0x8f, 0xd3, 0x30, 0xdd, 0xd7, 0xf4, 0x02, 0x4b, 0x5c, 0x0f, 0x8f, 0xf8, 0x25, 0xa4, 0xfa,
+	0x9f, 0xd5, 0xe9, 0xb1, 0x43, 0x6d, 0x91, 0x8e, 0x65, 0x51, 0x2e, 0x55, 0x5c, 0xca, 0x0f, 0x4a,
+	0xd9, 0x32, 0x8c, 0x9d, 0xe0, 0xa1, 0x1e, 0x55, 0xe2, 0x1c, 0xcc, 0xef, 0x73, 0x66, 0x0a, 0x6a,
+	0xb6, 0x4a, 0xad, 0x16, 0xa7, 0xb6, 0x9d, 0x9e, 0xf2, 0xaa, 0x19, 0x4d, 0xe3, 0xfb, 0x90, 0x74,
+	0x6c, 0xaf, 0xdc, 0xb8, 0x27, 0x08, 0x4e, 0x58, 0x81, 0x39, 0x5b, 0xec, 0x09, 0x5b, 0x35, 0xf7,
+	0x1a, 0x87, 0xb4, 0x95, 0x4e, 0x64, 0x51, 0x6e, 0x46, 0x1f, 0xca, 0x29, 0x3f, 0x62, 0xb0, 0xb0,
+	0x19, 0xbc, 0x2f, 0x4a, 0xe1, 0x15, 0xc4, 0x45, 0xcf, 0xa2, 0x5e, 0x37, 0x77, 0x8b, 0x8f, 0xf3,
+	0x91, 0x19, 0xe4, 0xc7, 0xe8, 0x8d, 0x9e, 0x45, 0x75, 0xcf, 0x31, 0xae, 0xee, 0xd8, 0xf8, 0xba,
+	0x23, 0xd0, 0xa6, 0x86, 0xa1, 0x4d, 0xea, 0x68, 0x04, 0x66, 0xe2, 0xd6, 0x30, 0x47, 0x51, 0x24,
+	0xaf, 0xa2, 0xe8, 0x6b, 0x3a, 0xdd, 0x76, 0x67, 0x87, 0x77, 0x19, 0xef, 0x8a, 0x5e, 0x7a, 0xda,
+	0xd7, 0x44, 0x73, 0xca, 0x01, 0x2c, 0x44, 0xa6, 0x1f, 0x82, 0xc0, 0xaf, 0x21, 0xd9, 0x7f, 0x95,
+	0x63, 0x07, 0xbc, 0x9e, 0x0c, 0xf1, 0x1a, 0xe3, 0xa8, 0x7a, 0x6a, 0x3d, 0x70, 0xe1, 0x45, 0x48,
+	0x50, 0xce, 0x19, 0x0f, 0x48, 0xf9, 0x07, 0x65, 0x0d, 0x96, 0x35, 0x26, 0xba, 0xfb, 0xbd, 0x60,
+	0xcb, 0xaa, 0x1d, 0x47, 0xb4, 0xd8, 0x89, 0x19, 0x36, 0x75, 0xfd, 0xa6, 0xae, 0xc0, 0xa3, 0x09,
+	0x6e, 0xdb, 0x62, 0xa6, 0x4d, 0x57, 0xd7, 0xe0, 0xc1, 0x84, 0x49, 0xe2, 0x19, 0x88, 0x97, 0xb5,
+	0xb2, 0x21, 0x4b, 0x38, 0x05, 0xd3, 0xaa, 0xb6, 0x5b, 0x53, 0x6b, 0xaa, 0x8c, 0x30, 0x40, 0x72,
+	0xbd, 0xa4, 0xad, 0xab, 0xdb, 0x72, 0x6c, 0xb5, 0x09, 0x0f, 0x27, 0xf6, 0x85, 0x93, 0x10, 0xab,
+	0xbc, 0x93, 0x25, 0x9c, 0x85, 0x65, 0xa3, 0x52, 0xa9, 0xbf, 0x2f, 0x69, 0x9f, 0xea, 0xba, 0xba,
+	0x5b, 0x53, 0xab, 0x46, 0xb5, 0xbe, 0xa3, 0xea, 0x75, 0x43, 0xd5, 0x4a, 0x9a, 0x21, 0x23, 0x3c,
+	0x0b, 0x09, 0x55, 0xd7, 0x2b, 0xba, 0x1c, 0xc3, 0xf7, 0xe0, 0x4e, 0x75, 0xab, 0x66, 0x18, 0x65,
+	0xed, 0x6d, 0x7d, 0xa3, 0xf2, 0x41, 0x93, 0xa7, 0x8a, 0xbf, 0x51, 0x84, 0xf7, 0x26, 0xe3, 0xe1,
+	0x75, 0xab, 0x41, 0x2a, 0x08, 0xb7, 0x19, 0xb3, 0xf0, 0xca, 0x10, 0xee, 0xab, 0x77, 0x3a, 0xb3,
+	0x32, 0x69, 0x1e, 0x81, 0x56, 0x91, 0x72, 0xe8, 0x19, 0xc2, 0x26, 0x2c, 0x8d, 0x45, 0x86, 0x9f,
+	0x0e, 0xf9, 0xaf, 0x1b, 0x4a, 0x66, 0xf5, 0x36, 0x52, 0x7f, 0x02, 0x45, 0x0b, 0x16, 0xa3, 0xdd,
+	0x0d, 0xd6, 0xe9, 0x23, 0xcc, 0x85, 0xb1, 0xd7, 0x5f, 0xf6, 0xa6, 0xeb, 0x97, 0xc9, 0xde, 0xb4,
+	0x70, 0x7e, 0x87, 0x6f, 0x4a, 0xa7, 0xe7, 0x44, 0x3a, 0x3b, 0x27, 0xd2, 0xe5, 0x39, 0x41, 0x5f,
+	0x5d, 0x82, 0xbe, 0xbb, 0x04, 0xfd, 0x74, 0x09, 0x3a, 0x75, 0x09, 0xfa, 0xe3, 0x12, 0xf4, 0xd7,
+	0x25, 0xd2, 0xa5, 0x4b, 0xd0, 0xb7, 0x0b, 0x22, 0x9d, 0x5e, 0x10, 0xe9, 0xec, 0x82, 0x48, 0x9f,
+	0xa3, 0x7f, 0xe0, 0x46, 0xd2, 0xfb, 0x79, 0x3e, 0xff, 0x17, 0x00, 0x00, 0xff, 0xff, 0x38, 0x3c,
+	0xc5, 0x8f, 0xa8, 0x05, 0x00, 0x00,
 }
 
 func (x FrontendToSchedulerType) String() string {
@@ -602,7 +602,7 @@ func (this *FrontendToScheduler) Equal(that interface{}) bool {
 	if this.StatsEnabled != that1.StatsEnabled {
 		return false
 	}
-	if this.IsHighPriority != that1.IsHighPriority {
+	if this.HighPriority != that1.HighPriority {
 		return false
 	}
 	return true
@@ -719,7 +719,7 @@ func (this *FrontendToScheduler) GoString() string {
 		s = append(s, "HttpRequest: "+fmt.Sprintf("%#v", this.HttpRequest)+",\n")
 	}
 	s = append(s, "StatsEnabled: "+fmt.Sprintf("%#v", this.StatsEnabled)+",\n")
-	s = append(s, "IsHighPriority: "+fmt.Sprintf("%#v", this.IsHighPriority)+",\n")
+	s = append(s, "HighPriority: "+fmt.Sprintf("%#v", this.HighPriority)+",\n")
 	s = append(s, "}")
 	return strings.Join(s, "")
 }
@@ -1155,9 +1155,9 @@ func (m *FrontendToScheduler) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 	_ = i
 	var l int
 	_ = l
-	if m.IsHighPriority {
+	if m.HighPriority {
 		i--
-		if m.IsHighPriority {
+		if m.HighPriority {
 			dAtA[i] = 1
 		} else {
 			dAtA[i] = 0
@@ -1380,7 +1380,7 @@ func (m *FrontendToScheduler) Size() (n int) {
 	if m.StatsEnabled {
 		n += 2
 	}
-	if m.IsHighPriority {
+	if m.HighPriority {
 		n += 2
 	}
 	return n
@@ -1465,7 +1465,7 @@ func (this *FrontendToScheduler) String() string {
 		`UserID:` + fmt.Sprintf("%v", this.UserID) + `,`,
 		`HttpRequest:` + strings.Replace(fmt.Sprintf("%v", this.HttpRequest), "HTTPRequest", "httpgrpc.HTTPRequest", 1) + `,`,
 		`StatsEnabled:` + fmt.Sprintf("%v", this.StatsEnabled) + `,`,
-		`IsHighPriority:` + fmt.Sprintf("%v", this.IsHighPriority) + `,`,
+		`HighPriority:` + fmt.Sprintf("%v", this.HighPriority) + `,`,
 		`}`,
 	}, "")
 	return s
@@ -1974,7 +1974,7 @@ func (m *FrontendToScheduler) Unmarshal(dAtA []byte) error {
 			m.StatsEnabled = bool(v != 0)
 		case 7:
 			if wireType != 0 {
-				return fmt.Errorf("proto: wrong wireType = %d for field IsHighPriority", wireType)
+				return fmt.Errorf("proto: wrong wireType = %d for field HighPriority", wireType)
 			}
 			var v int
 			for shift := uint(0); ; shift += 7 {
@@ -1991,7 +1991,7 @@ func (m *FrontendToScheduler) Unmarshal(dAtA []byte) error {
 					break
 				}
 			}
-			m.IsHighPriority = bool(v != 0)
+			m.HighPriority = bool(v != 0)
 		default:
 			iNdEx = preIndex
 			skippy, err := skipScheduler(dAtA[iNdEx:])
diff --git a/pkg/scheduler/schedulerpb/scheduler.proto b/pkg/scheduler/schedulerpb/scheduler.proto
index 5e71946008..1d246f73a2 100644
--- a/pkg/scheduler/schedulerpb/scheduler.proto
+++ b/pkg/scheduler/schedulerpb/scheduler.proto
@@ -78,7 +78,7 @@ message FrontendToScheduler {
   string userID = 4;
   httpgrpc.HTTPRequest httpRequest = 5;
   bool statsEnabled = 6;
-  bool isHighPriority = 7;
+  bool highPriority = 7;
 }
 
 enum SchedulerToFrontendStatus {
diff --git a/pkg/util/query/priority_test.go b/pkg/util/query/priority_test.go
index dd1ac9c78b..5085eebf60 100644
--- a/pkg/util/query/priority_test.go
+++ b/pkg/util/query/priority_test.go
@@ -11,7 +11,7 @@ import (
 	"github.com/cortexproject/cortex/pkg/util/validation"
 )
 
-func Test_IsHighPriority_ShouldMatchRegex(t *testing.T) {
+func Test_IsHighPriorityShouldMatchRegex(t *testing.T) {
 	now := time.Now()
 	config := []validation.HighPriorityQuery{
 		{
@@ -110,7 +110,7 @@ func Test_IsHighPriority_ShouldMatchRegex(t *testing.T) {
 	}, now, config))
 }
 
-func Test_IsHighPriority_ShouldBeBetweenStartAndEndTime(t *testing.T) {
+func Test_IsHighPriorityShouldBeBetweenStartAndEndTime(t *testing.T) {
 	now := time.Now()
 	config := []validation.HighPriorityQuery{
 		{
diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go
index 301e945ea8..94907a1905 100644
--- a/pkg/util/validation/limits.go
+++ b/pkg/util/validation/limits.go
@@ -6,6 +6,7 @@ import (
 	"errors"
 	"flag"
 	"math"
+	"regexp"
 	"strings"
 	"time"
 
@@ -47,9 +48,10 @@ type DisabledRuleGroup struct {
 type DisabledRuleGroups []DisabledRuleGroup
 
 type HighPriorityQuery struct {
-	Regex     string        `yaml:"regex" doc:"nocli|description=Query string regex. If evaluated true (on top of meeting all other criteria), query is treated as a high priority."`
-	StartTime time.Duration `yaml:"start_time" doc:"nocli|description=If query range falls between the start_time and end_time (on top of meeting all other criteria), query is treated as a high priority.|default=0s"`
-	EndTime   time.Duration `yaml:"end_time" doc:"nocli|description=If query range falls between the start_time and end_time (on top of meeting all other criteria), query is treated as a high priority.|default=0s"`
+	Regex         string         `yaml:"regex" doc:"nocli|description=Query string regex. If evaluated true (on top of meeting all other criteria), query is treated as a high priority."`
+	CompiledRegex *regexp.Regexp `yaml:"-" doc:"nocli"`
+	StartTime     time.Duration  `yaml:"start_time" doc:"nocli|description=If query range falls between the start_time and end_time (on top of meeting all other criteria), query is treated as a high priority.|default=0s"`
+	EndTime       time.Duration  `yaml:"end_time" doc:"nocli|description=If query range falls between the start_time and end_time (on top of meeting all other criteria), query is treated as a high priority.|default=0s"`
 }
 
 // Limits describe all the limits for users; can be used to describe global default

From 393d9f00a99405ede730f942945b402a291c43f5 Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Mon, 30 Oct 2023 22:19:06 -0700
Subject: [PATCH 16/49] Add CompiledRegex to HighPriorityQuery

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 pkg/util/query/priority.go    | 5 ++---
 pkg/util/validation/limits.go | 7 +++++++
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/pkg/util/query/priority.go b/pkg/util/query/priority.go
index 84c7dfa262..ea345074df 100644
--- a/pkg/util/query/priority.go
+++ b/pkg/util/query/priority.go
@@ -3,7 +3,6 @@ package query
 import (
 	"math"
 	"net/url"
-	"regexp"
 	"strconv"
 	"time"
 
@@ -23,9 +22,9 @@ func IsHighPriority(requestParams url.Values, now time.Time, highPriorityQueries
 	}
 
 	for _, highPriorityQuery := range highPriorityQueries {
-		regex := highPriorityQuery.Regex
+		compiledRegex := highPriorityQuery.CompiledRegex
 
-		if match, err := regexp.MatchString(regex, queryParam); !match || err != nil {
+		if compiledRegex == nil || !compiledRegex.MatchString(queryParam) {
 			continue
 		}
 
diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go
index 94907a1905..45f3cceb8e 100644
--- a/pkg/util/validation/limits.go
+++ b/pkg/util/validation/limits.go
@@ -510,6 +510,13 @@ func (o *Overrides) ReservedHighPriorityQueriers(userID string) float64 {
 
 // HighPriorityQueries returns list of definitions for high priority query.
 func (o *Overrides) HighPriorityQueries(userID string) []HighPriorityQuery {
+	highPriorityQueries := o.GetOverridesForUser(userID).HighPriorityQueries
+	for index, query := range highPriorityQueries {
+		if query.CompiledRegex == nil {
+			// no need to handle error, as we will use the CompiledRegex only if it's not nil
+			o.GetOverridesForUser(userID).HighPriorityQueries[index].CompiledRegex, _ = regexp.Compile(query.Regex)
+		}
+	}
 	return o.GetOverridesForUser(userID).HighPriorityQueries
 }
 

From ebc2c1e9acade23eb0dc0bdbcb2254e56c6083b7 Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Wed, 1 Nov 2023 08:40:24 -0700
Subject: [PATCH 17/49] Introduce numbered priority + change config structure

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 pkg/frontend/v1/frontend.go                  |  22 ++--
 pkg/frontend/v2/frontend.go                  |   4 +-
 pkg/frontend/v2/frontend_scheduler_worker.go |   2 +-
 pkg/scheduler/queue/queue.go                 |   4 +-
 pkg/scheduler/queue/user_queues.go           |  67 +++++------
 pkg/scheduler/scheduler.go                   |   8 +-
 pkg/scheduler/schedulerpb/scheduler.pb.go    | 120 +++++++++----------
 pkg/scheduler/schedulerpb/scheduler.proto    |   2 +-
 pkg/util/query/priority.go                   |  40 ++++---
 pkg/util/validation/limits.go                |  45 ++++---
 10 files changed, 154 insertions(+), 160 deletions(-)

diff --git a/pkg/frontend/v1/frontend.go b/pkg/frontend/v1/frontend.go
index 8f2fae830f..5b874fa865 100644
--- a/pkg/frontend/v1/frontend.go
+++ b/pkg/frontend/v1/frontend.go
@@ -53,8 +53,8 @@ type Limits interface {
 
 // MockLimits implements the Limits interface. Used in tests only.
 type MockLimits struct {
-	Queriers float64
-	Queries  []validation.HighPriorityQuery
+	Queriers      float64
+	QueryPriority validation.QueryPriority
 	queue.MockLimits
 }
 
@@ -62,8 +62,8 @@ func (l MockLimits) MaxQueriersPerUser(_ string) float64 {
 	return l.Queriers
 }
 
-func (l MockLimits) HighPriorityQueries(_ string) []validation.HighPriorityQuery {
-	return l.Queries
+func (l MockLimits) HighPriorityQueries(_ string) validation.QueryPriority {
+	return l.QueryPriority
 }
 
 // Frontend queues HTTP requests, dispatches them to backends, and handles retries
@@ -95,14 +95,14 @@ type request struct {
 	queueSpan   opentracing.Span
 	originalCtx context.Context
 
-	request      *httpgrpc.HTTPRequest
-	err          chan error
-	response     chan *httpgrpc.HTTPResponse
-	highPriority bool
+	request  *httpgrpc.HTTPRequest
+	err      chan error
+	response chan *httpgrpc.HTTPResponse
+	priority int64
 }
 
-func (r request) IsHighPriority() bool {
-	return r.highPriority
+func (r request) GetPriority() int64 {
+	return r.priority
 }
 
 // New creates a new frontend. Frontend implements service, and must be started and stopped.
@@ -207,7 +207,7 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest,
 		}
 
 		if reqParams != nil {
-			request.highPriority = util_query.IsHighPriority(reqParams, ts, f.limits.HighPriorityQueries(userID))
+			request.priority = util_query.GetPriority(reqParams, ts, f.limits.QueryPriority(userID))
 		}
 
 		if err := f.queueRequest(ctx, &request); err != nil {
diff --git a/pkg/frontend/v2/frontend.go b/pkg/frontend/v2/frontend.go
index 3cb00bdbca..81d0e8c802 100644
--- a/pkg/frontend/v2/frontend.go
+++ b/pkg/frontend/v2/frontend.go
@@ -89,7 +89,7 @@ type frontendRequest struct {
 	request      *httpgrpc.HTTPRequest
 	userID       string
 	statsEnabled bool
-	highPriority bool
+	priority     int64
 
 	cancel context.CancelFunc
 
@@ -211,7 +211,7 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest,
 		}
 
 		if reqParams != nil {
-			freq.highPriority = util_query.IsHighPriority(reqParams, ts, f.limits.HighPriorityQueries(userID))
+			freq.priority = util_query.GetPriority(reqParams, ts, f.limits.QueryPriority(userID))
 		}
 
 		f.requests.put(freq)
diff --git a/pkg/frontend/v2/frontend_scheduler_worker.go b/pkg/frontend/v2/frontend_scheduler_worker.go
index 557457fdc0..2abff24bf6 100644
--- a/pkg/frontend/v2/frontend_scheduler_worker.go
+++ b/pkg/frontend/v2/frontend_scheduler_worker.go
@@ -263,7 +263,7 @@ func (w *frontendSchedulerWorker) schedulerLoop(loop schedulerpb.SchedulerForFro
 				HttpRequest:     req.request,
 				FrontendAddress: w.frontendAddr,
 				StatsEnabled:    req.statsEnabled,
-				HighPriority:    req.highPriority,
+				Priority:        req.priority,
 			})
 
 			if err != nil {
diff --git a/pkg/scheduler/queue/queue.go b/pkg/scheduler/queue/queue.go
index 0931b25f3b..cf20a3573a 100644
--- a/pkg/scheduler/queue/queue.go
+++ b/pkg/scheduler/queue/queue.go
@@ -45,7 +45,7 @@ func FirstUser() UserIndex {
 
 // Request stored into the queue.
 type Request interface {
-	IsHighPriority() bool
+	GetPriority() int64
 }
 
 // RequestQueue holds incoming requests in per-user queues. It also assigns each user specified number of queriers,
@@ -98,7 +98,7 @@ func (q *RequestQueue) EnqueueRequest(userID string, req Request, maxQueriers fl
 	}
 
 	shardSize := util.DynamicShardSize(maxQueriers, len(q.queues.queriers))
-	queue := q.queues.getOrAddQueue(userID, shardSize, req.IsHighPriority())
+	queue := q.queues.getOrAddQueue(userID, shardSize, req.GetPriority())
 	maxOutstandingRequests := q.queues.limits.MaxOutstandingPerTenant(userID)
 
 	if queue == nil {
diff --git a/pkg/scheduler/queue/user_queues.go b/pkg/scheduler/queue/user_queues.go
index a437afce83..dafe0b6fd0 100644
--- a/pkg/scheduler/queue/user_queues.go
+++ b/pkg/scheduler/queue/user_queues.go
@@ -17,15 +17,9 @@ type Limits interface {
 	// of outstanding requests per tenant per request queue.
 	MaxOutstandingPerTenant(user string) int
 
-	// ReservedHighPriorityQueriers returns the minimum number of queriers dedicated for high priority
-	// queue per tenant. All queriers still handle priority queue first, but this provides extra protection on
-	// high priority queries from slow normal queries exhausting all queriers.
-	// ReservedHighPriorityQueriers is capped by MaxQueriersPerUser.
-	// If less than 1, it will be applied as a percentage of MaxQueriersPerUser.
-	ReservedHighPriorityQueriers(user string) float64
-
-	// HighPriorityQueries returns list of definitions for high priority query.
-	HighPriorityQueries(user string) []validation.HighPriorityQuery
+	// QueryPriority returns query priority config for the tenant, including different priorities,
+	// their attributes, and how many reserved queriers each priority has.
+	QueryPriority(user string) validation.QueryPriority
 }
 
 // querier holds information about a querier registered in the queue.
@@ -118,7 +112,7 @@ func (q *queues) deleteQueue(userID string) {
 // MaxQueriers is used to compute which queriers should handle requests for this user.
 // If maxQueriers is <= 0, all queriers can handle this user's requests.
 // If maxQueriers has changed since the last call, queriers for this are recomputed.
-func (q *queues) getOrAddQueue(userID string, maxQueriers int, highPriority bool) chan Request {
+func (q *queues) getOrAddQueue(userID string, maxQueriers int, priority int64) chan Request {
 	// Empty user is not allowed, as that would break our users list ("" is used for free spot).
 	if userID == "" {
 		return nil
@@ -163,12 +157,12 @@ func (q *queues) getOrAddQueue(userID string, maxQueriers int, highPriority bool
 
 	if uq.maxQueriers != maxQueriers {
 		uq.maxQueriers = maxQueriers
-		uq.queriers, uq.reservedQueriers = shuffleQueriersForUser(uq.seed, maxQueriers, q.sortedQueriers, q.limits.ReservedHighPriorityQueriers(userID), nil)
+		uq.queriers = shuffleQueriersForUser(uq.seed, maxQueriers, q.sortedQueriers, nil)
 	}
 
-	if highPriority {
-		return uq.highPriorityQueue
-	}
+	//if highPriority {
+	//	return uq.highPriorityQueue
+	//}
 
 	return uq.normalQueue
 }
@@ -319,24 +313,24 @@ func (q *queues) recomputeUserQueriers() {
 	scratchpad := make([]string, 0, len(q.sortedQueriers))
 
 	for _, uq := range q.userQueues {
-		userID := q.users[uq.index]
-		uq.queriers, uq.reservedQueriers = shuffleQueriersForUser(uq.seed, uq.maxQueriers, q.sortedQueriers, q.limits.ReservedHighPriorityQueriers(userID), scratchpad)
+		//userID := q.users[uq.index]
+		uq.queriers = shuffleQueriersForUser(uq.seed, uq.maxQueriers, q.sortedQueriers, scratchpad)
 	}
 }
 
 // shuffleQueriersForUser returns nil if queriersToSelect is 0 or there are not enough queriers to select from.
 // In that case *all* queriers should be used.
 // Scratchpad is used for shuffling, to avoid new allocations. If nil, new slice is allocated.
-func shuffleQueriersForUser(userSeed int64, queriersToSelect int, allSortedQueriers []string, numOfReservedQueriersFloat float64, scratchpad []string) (map[string]struct{}, map[string]struct{}) {
-	numOfReservedQueriers := getNumOfReservedQueriers(queriersToSelect, len(allSortedQueriers), numOfReservedQueriersFloat)
-	reservedQueriers := make(map[string]struct{}, numOfReservedQueriers)
+func shuffleQueriersForUser(userSeed int64, queriersToSelect int, allSortedQueriers []string, scratchpad []string) map[string]struct{} {
+	//numOfReservedQueriers := getNumOfReservedQueriers(queriersToSelect, len(allSortedQueriers), numOfReservedQueriersFloat)
+	//reservedQueriers := make(map[string]struct{}, numOfReservedQueriers)
 
-	if queriersToSelect == 0 || len(allSortedQueriers) <= queriersToSelect {
-		for i := 0; i < numOfReservedQueriers; i++ {
-			reservedQueriers[allSortedQueriers[i]] = struct{}{}
-		}
-		return nil, reservedQueriers
-	}
+	//if queriersToSelect == 0 || len(allSortedQueriers) <= queriersToSelect {
+	//	for i := 0; i < numOfReservedQueriers; i++ {
+	//		reservedQueriers[allSortedQueriers[i]] = struct{}{}
+	//	}
+	//	return nil, reservedQueriers
+	//}
 
 	queriers := make(map[string]struct{}, queriersToSelect)
 	rnd := rand.New(rand.NewSource(userSeed))
@@ -348,15 +342,15 @@ func shuffleQueriersForUser(userSeed int64, queriersToSelect int, allSortedQueri
 	for i := 0; i < queriersToSelect; i++ {
 		r := rnd.Intn(last + 1)
 		queriers[scratchpad[r]] = struct{}{}
-		if i < numOfReservedQueriers {
-			reservedQueriers[scratchpad[r]] = struct{}{}
-		}
+		//if i < numOfReservedQueriers {
+		//	reservedQueriers[scratchpad[r]] = struct{}{}
+		//}
 		// move selected item to the end, it won't be selected anymore.
 		scratchpad[r], scratchpad[last] = scratchpad[last], scratchpad[r]
 		last--
 	}
 
-	return queriers, reservedQueriers
+	return queriers
 }
 
 func getNumOfReservedQueriers(queriersToSelect int, totalNumOfQueriers int, reservedQueriers float64) int {
@@ -379,10 +373,9 @@ func getNumOfReservedQueriers(queriersToSelect int, totalNumOfQueriers int, rese
 
 // MockLimits implements the Limits interface. Used in tests only.
 type MockLimits struct {
-	MaxOutstanding               int
-	maxQueriersPerUser           float64
-	reservedHighPriorityQueriers float64
-	highPriorityQueries          []validation.HighPriorityQuery
+	MaxOutstanding     int
+	maxQueriersPerUser float64
+	queryPriority      validation.QueryPriority
 }
 
 func (l MockLimits) MaxQueriersPerUser(user string) float64 {
@@ -393,10 +386,6 @@ func (l MockLimits) MaxOutstandingPerTenant(_ string) int {
 	return l.MaxOutstanding
 }
 
-func (l MockLimits) ReservedHighPriorityQueriers(_ string) float64 {
-	return l.reservedHighPriorityQueriers
-}
-
-func (l MockLimits) HighPriorityQueries(_ string) []validation.HighPriorityQuery {
-	return l.highPriorityQueries
+func (l MockLimits) QueryPriority(_ string) validation.QueryPriority {
+	return l.queryPriority
 }
diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go
index cbd673badf..d4f7ac8541 100644
--- a/pkg/scheduler/scheduler.go
+++ b/pkg/scheduler/scheduler.go
@@ -154,7 +154,7 @@ type schedulerRequest struct {
 	queryID         uint64
 	request         *httpgrpc.HTTPRequest
 	statsEnabled    bool
-	highPriority    bool
+	priority        int64
 
 	enqueueTime time.Time
 
@@ -166,8 +166,8 @@ type schedulerRequest struct {
 	parentSpanContext opentracing.SpanContext
 }
 
-func (s schedulerRequest) IsHighPriority() bool {
-	return s.highPriority
+func (s schedulerRequest) GetPriority() int64 {
+	return s.priority
 }
 
 // FrontendLoop handles connection from frontend.
@@ -298,7 +298,7 @@ func (s *Scheduler) enqueueRequest(frontendContext context.Context, frontendAddr
 		queryID:         msg.QueryID,
 		request:         msg.HttpRequest,
 		statsEnabled:    msg.StatsEnabled,
-		highPriority:    msg.HighPriority,
+		priority:        msg.Priority,
 	}
 
 	now := time.Now()
diff --git a/pkg/scheduler/schedulerpb/scheduler.pb.go b/pkg/scheduler/schedulerpb/scheduler.pb.go
index 5933f21c77..2f352a8138 100644
--- a/pkg/scheduler/schedulerpb/scheduler.pb.go
+++ b/pkg/scheduler/schedulerpb/scheduler.pb.go
@@ -219,7 +219,7 @@ type FrontendToScheduler struct {
 	UserID       string                `protobuf:"bytes,4,opt,name=userID,proto3" json:"userID,omitempty"`
 	HttpRequest  *httpgrpc.HTTPRequest `protobuf:"bytes,5,opt,name=httpRequest,proto3" json:"httpRequest,omitempty"`
 	StatsEnabled bool                  `protobuf:"varint,6,opt,name=statsEnabled,proto3" json:"statsEnabled,omitempty"`
-	HighPriority bool                  `protobuf:"varint,7,opt,name=highPriority,proto3" json:"highPriority,omitempty"`
+	Priority     int64                 `protobuf:"varint,7,opt,name=priority,proto3" json:"priority,omitempty"`
 }
 
 func (m *FrontendToScheduler) Reset()      { *m = FrontendToScheduler{} }
@@ -296,11 +296,11 @@ func (m *FrontendToScheduler) GetStatsEnabled() bool {
 	return false
 }
 
-func (m *FrontendToScheduler) GetHighPriority() bool {
+func (m *FrontendToScheduler) GetPriority() int64 {
 	if m != nil {
-		return m.HighPriority
+		return m.Priority
 	}
-	return false
+	return 0
 }
 
 type SchedulerToFrontend struct {
@@ -446,49 +446,49 @@ func init() {
 func init() { proto.RegisterFile("scheduler.proto", fileDescriptor_2b3fc28395a6d9c5) }
 
 var fileDescriptor_2b3fc28395a6d9c5 = []byte{
-	// 662 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x94, 0xcd, 0x4e, 0xdb, 0x40,
-	0x10, 0xc7, 0xbd, 0x21, 0x09, 0x30, 0xa1, 0xc5, 0x5d, 0xa0, 0x4d, 0x23, 0xba, 0x44, 0x56, 0x55,
-	0xa5, 0x1c, 0x92, 0x2a, 0xad, 0xd4, 0x1e, 0x50, 0xa5, 0x14, 0x4c, 0x89, 0x4a, 0x1d, 0x70, 0x1c,
-	0xf5, 0xe3, 0x12, 0x91, 0x64, 0x49, 0x22, 0xc0, 0x6b, 0xd6, 0xeb, 0xa2, 0xdc, 0xfa, 0x08, 0x7d,
-	0x8c, 0x1e, 0xfa, 0x20, 0xbd, 0x54, 0xe2, 0xc8, 0xa1, 0x87, 0x62, 0x2e, 0x3d, 0xf2, 0x08, 0x55,
-	0xfc, 0x91, 0x3a, 0x21, 0x01, 0x6e, 0xb3, 0xe3, 0xff, 0xdf, 0x9e, 0xf9, 0xcd, 0xac, 0x61, 0xde,
-	0x6e, 0x76, 0x68, 0xcb, 0x39, 0xa4, 0x3c, 0x6f, 0x71, 0x26, 0x18, 0x4e, 0x0d, 0x12, 0x56, 0x23,
-	0xb3, 0xd8, 0x66, 0x6d, 0xe6, 0xe5, 0x0b, 0xfd, 0xc8, 0x97, 0x64, 0x5e, 0xb4, 0xbb, 0xa2, 0xe3,
-	0x34, 0xf2, 0x4d, 0x76, 0x54, 0x38, 0xa1, 0x7b, 0x5f, 0xe8, 0x09, 0xe3, 0x07, 0x76, 0xa1, 0xc9,
-	0x8e, 0x8e, 0x98, 0x59, 0xe8, 0x08, 0x61, 0xb5, 0xb9, 0xd5, 0x1c, 0x04, 0xbe, 0x4b, 0x29, 0x02,
-	0xde, 0x75, 0x28, 0xef, 0x52, 0x6e, 0xb0, 0x6a, 0xf8, 0x0d, 0xbc, 0x0c, 0xb3, 0xc7, 0x7e, 0xb6,
-	0xbc, 0x91, 0x46, 0x59, 0x94, 0x9b, 0xd5, 0xff, 0x27, 0x94, 0x5f, 0x08, 0xf0, 0x40, 0x6b, 0xb0,
-	0xc0, 0x8f, 0xd3, 0x30, 0xdd, 0xd7, 0xf4, 0x02, 0x4b, 0x5c, 0x0f, 0x8f, 0xf8, 0x25, 0xa4, 0xfa,
-	0x9f, 0xd5, 0xe9, 0xb1, 0x43, 0x6d, 0x91, 0x8e, 0x65, 0x51, 0x2e, 0x55, 0x5c, 0xca, 0x0f, 0x4a,
-	0xd9, 0x32, 0x8c, 0x9d, 0xe0, 0xa1, 0x1e, 0x55, 0xe2, 0x1c, 0xcc, 0xef, 0x73, 0x66, 0x0a, 0x6a,
-	0xb6, 0x4a, 0xad, 0x16, 0xa7, 0xb6, 0x9d, 0x9e, 0xf2, 0xaa, 0x19, 0x4d, 0xe3, 0xfb, 0x90, 0x74,
-	0x6c, 0xaf, 0xdc, 0xb8, 0x27, 0x08, 0x4e, 0x58, 0x81, 0x39, 0x5b, 0xec, 0x09, 0x5b, 0x35, 0xf7,
-	0x1a, 0x87, 0xb4, 0x95, 0x4e, 0x64, 0x51, 0x6e, 0x46, 0x1f, 0xca, 0x29, 0x3f, 0x62, 0xb0, 0xb0,
-	0x19, 0xbc, 0x2f, 0x4a, 0xe1, 0x15, 0xc4, 0x45, 0xcf, 0xa2, 0x5e, 0x37, 0x77, 0x8b, 0x8f, 0xf3,
-	0x91, 0x19, 0xe4, 0xc7, 0xe8, 0x8d, 0x9e, 0x45, 0x75, 0xcf, 0x31, 0xae, 0xee, 0xd8, 0xf8, 0xba,
-	0x23, 0xd0, 0xa6, 0x86, 0xa1, 0x4d, 0xea, 0x68, 0x04, 0x66, 0xe2, 0xd6, 0x30, 0x47, 0x51, 0x24,
-	0xaf, 0xa2, 0xe8, 0x6b, 0x3a, 0xdd, 0x76, 0x67, 0x87, 0x77, 0x19, 0xef, 0x8a, 0x5e, 0x7a, 0xda,
-	0xd7, 0x44, 0x73, 0xca, 0x01, 0x2c, 0x44, 0xa6, 0x1f, 0x82, 0xc0, 0xaf, 0x21, 0xd9, 0x7f, 0x95,
-	0x63, 0x07, 0xbc, 0x9e, 0x0c, 0xf1, 0x1a, 0xe3, 0xa8, 0x7a, 0x6a, 0x3d, 0x70, 0xe1, 0x45, 0x48,
-	0x50, 0xce, 0x19, 0x0f, 0x48, 0xf9, 0x07, 0x65, 0x0d, 0x96, 0x35, 0x26, 0xba, 0xfb, 0xbd, 0x60,
-	0xcb, 0xaa, 0x1d, 0x47, 0xb4, 0xd8, 0x89, 0x19, 0x36, 0x75, 0xfd, 0xa6, 0xae, 0xc0, 0xa3, 0x09,
-	0x6e, 0xdb, 0x62, 0xa6, 0x4d, 0x57, 0xd7, 0xe0, 0xc1, 0x84, 0x49, 0xe2, 0x19, 0x88, 0x97, 0xb5,
-	0xb2, 0x21, 0x4b, 0x38, 0x05, 0xd3, 0xaa, 0xb6, 0x5b, 0x53, 0x6b, 0xaa, 0x8c, 0x30, 0x40, 0x72,
-	0xbd, 0xa4, 0xad, 0xab, 0xdb, 0x72, 0x6c, 0xb5, 0x09, 0x0f, 0x27, 0xf6, 0x85, 0x93, 0x10, 0xab,
-	0xbc, 0x93, 0x25, 0x9c, 0x85, 0x65, 0xa3, 0x52, 0xa9, 0xbf, 0x2f, 0x69, 0x9f, 0xea, 0xba, 0xba,
-	0x5b, 0x53, 0xab, 0x46, 0xb5, 0xbe, 0xa3, 0xea, 0x75, 0x43, 0xd5, 0x4a, 0x9a, 0x21, 0x23, 0x3c,
-	0x0b, 0x09, 0x55, 0xd7, 0x2b, 0xba, 0x1c, 0xc3, 0xf7, 0xe0, 0x4e, 0x75, 0xab, 0x66, 0x18, 0x65,
-	0xed, 0x6d, 0x7d, 0xa3, 0xf2, 0x41, 0x93, 0xa7, 0x8a, 0xbf, 0x51, 0x84, 0xf7, 0x26, 0xe3, 0xe1,
-	0x75, 0xab, 0x41, 0x2a, 0x08, 0xb7, 0x19, 0xb3, 0xf0, 0xca, 0x10, 0xee, 0xab, 0x77, 0x3a, 0xb3,
-	0x32, 0x69, 0x1e, 0x81, 0x56, 0x91, 0x72, 0xe8, 0x19, 0xc2, 0x26, 0x2c, 0x8d, 0x45, 0x86, 0x9f,
-	0x0e, 0xf9, 0xaf, 0x1b, 0x4a, 0x66, 0xf5, 0x36, 0x52, 0x7f, 0x02, 0x45, 0x0b, 0x16, 0xa3, 0xdd,
-	0x0d, 0xd6, 0xe9, 0x23, 0xcc, 0x85, 0xb1, 0xd7, 0x5f, 0xf6, 0xa6, 0xeb, 0x97, 0xc9, 0xde, 0xb4,
-	0x70, 0x7e, 0x87, 0x6f, 0x4a, 0xa7, 0xe7, 0x44, 0x3a, 0x3b, 0x27, 0xd2, 0xe5, 0x39, 0x41, 0x5f,
-	0x5d, 0x82, 0xbe, 0xbb, 0x04, 0xfd, 0x74, 0x09, 0x3a, 0x75, 0x09, 0xfa, 0xe3, 0x12, 0xf4, 0xd7,
-	0x25, 0xd2, 0xa5, 0x4b, 0xd0, 0xb7, 0x0b, 0x22, 0x9d, 0x5e, 0x10, 0xe9, 0xec, 0x82, 0x48, 0x9f,
-	0xa3, 0x7f, 0xe0, 0x46, 0xd2, 0xfb, 0x79, 0x3e, 0xff, 0x17, 0x00, 0x00, 0xff, 0xff, 0x38, 0x3c,
-	0xc5, 0x8f, 0xa8, 0x05, 0x00, 0x00,
+	// 661 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0xcd, 0x4e, 0xdb, 0x40,
+	0x10, 0xf6, 0xe6, 0x0f, 0x98, 0xd0, 0xe2, 0x2e, 0xd0, 0xa6, 0x11, 0x5d, 0x2c, 0xab, 0xaa, 0x52,
+	0x0e, 0x49, 0x95, 0x56, 0x6a, 0x0f, 0xa8, 0x52, 0x0a, 0xa6, 0x44, 0xa5, 0x0e, 0x6c, 0x1c, 0xf5,
+	0xe7, 0x12, 0x91, 0x64, 0x49, 0x22, 0xc0, 0x6b, 0xd6, 0x76, 0x51, 0x6e, 0x7d, 0x84, 0x3e, 0x44,
+	0x0f, 0x7d, 0x94, 0x5e, 0x2a, 0x71, 0xe4, 0xd0, 0x43, 0x31, 0x97, 0x1e, 0x79, 0x84, 0x2a, 0x8e,
+	0xe3, 0x3a, 0x90, 0x00, 0xb7, 0x99, 0xf1, 0xf7, 0x79, 0xe7, 0xfb, 0x66, 0x76, 0x61, 0xce, 0x6e,
+	0x76, 0x58, 0xcb, 0x3d, 0x60, 0x22, 0x6f, 0x09, 0xee, 0x70, 0x9c, 0x0e, 0x0b, 0x56, 0x23, 0xbb,
+	0xd0, 0xe6, 0x6d, 0xee, 0xd7, 0x0b, 0xfd, 0x68, 0x00, 0xc9, 0xbe, 0x68, 0x77, 0x9d, 0x8e, 0xdb,
+	0xc8, 0x37, 0xf9, 0x61, 0xe1, 0x98, 0xed, 0x7e, 0x61, 0xc7, 0x5c, 0xec, 0xdb, 0x85, 0x26, 0x3f,
+	0x3c, 0xe4, 0x66, 0xa1, 0xe3, 0x38, 0x56, 0x5b, 0x58, 0xcd, 0x30, 0x18, 0xb0, 0xd4, 0x22, 0xe0,
+	0x1d, 0x97, 0x89, 0x2e, 0x13, 0x06, 0xaf, 0x0e, 0xcf, 0xc0, 0x4b, 0x30, 0x73, 0x34, 0xa8, 0x96,
+	0xd7, 0x33, 0x48, 0x41, 0xb9, 0x19, 0xfa, 0xbf, 0xa0, 0xfe, 0x42, 0x80, 0x43, 0xac, 0xc1, 0x03,
+	0x3e, 0xce, 0xc0, 0x54, 0x1f, 0xd3, 0x0b, 0x28, 0x09, 0x3a, 0x4c, 0xf1, 0x4b, 0x48, 0xf7, 0x8f,
+	0xa5, 0xec, 0xc8, 0x65, 0xb6, 0x93, 0x89, 0x29, 0x28, 0x97, 0x2e, 0x2e, 0xe6, 0xc3, 0x56, 0x36,
+	0x0d, 0x63, 0x3b, 0xf8, 0x48, 0xa3, 0x48, 0x9c, 0x83, 0xb9, 0x3d, 0xc1, 0x4d, 0x87, 0x99, 0xad,
+	0x52, 0xab, 0x25, 0x98, 0x6d, 0x67, 0xe2, 0x7e, 0x37, 0x97, 0xcb, 0xf8, 0x3e, 0xa4, 0x5c, 0xdb,
+	0x6f, 0x37, 0xe1, 0x03, 0x82, 0x0c, 0xab, 0x30, 0x6b, 0x3b, 0xbb, 0x8e, 0xad, 0x99, 0xbb, 0x8d,
+	0x03, 0xd6, 0xca, 0x24, 0x15, 0x94, 0x9b, 0xa6, 0x23, 0x35, 0xf5, 0x7b, 0x0c, 0xe6, 0x37, 0x82,
+	0xff, 0x45, 0x5d, 0x78, 0x05, 0x09, 0xa7, 0x67, 0x31, 0x5f, 0xcd, 0xdd, 0xe2, 0xe3, 0x7c, 0x64,
+	0x06, 0xf9, 0x31, 0x78, 0xa3, 0x67, 0x31, 0xea, 0x33, 0xc6, 0xf5, 0x1d, 0x1b, 0xdf, 0x77, 0xc4,
+	0xb4, 0xf8, 0xa8, 0x69, 0x93, 0x14, 0x5d, 0x32, 0x33, 0x79, 0x6b, 0x33, 0x2f, 0x5b, 0x91, 0xba,
+	0x6a, 0x05, 0xce, 0xc2, 0xb4, 0x25, 0xba, 0x5c, 0x74, 0x9d, 0x5e, 0x66, 0x4a, 0x41, 0xb9, 0x38,
+	0x0d, 0x73, 0x75, 0x1f, 0xe6, 0x23, 0x53, 0x1f, 0x1a, 0x80, 0x5f, 0x43, 0xaa, 0xff, 0x0b, 0xd7,
+	0x0e, 0x7c, 0x7a, 0x32, 0xe2, 0xd3, 0x18, 0x46, 0xd5, 0x47, 0xd3, 0x80, 0x85, 0x17, 0x20, 0xc9,
+	0x84, 0xe0, 0x22, 0x70, 0x68, 0x90, 0xa8, 0xab, 0xb0, 0xa4, 0x73, 0xa7, 0xbb, 0xd7, 0x0b, 0xb6,
+	0xab, 0xda, 0x71, 0x9d, 0x16, 0x3f, 0x36, 0x87, 0x62, 0xae, 0xdf, 0xd0, 0x65, 0x78, 0x34, 0x81,
+	0x6d, 0x5b, 0xdc, 0xb4, 0xd9, 0xca, 0x2a, 0x3c, 0x98, 0x30, 0x41, 0x3c, 0x0d, 0x89, 0xb2, 0x5e,
+	0x36, 0x64, 0x09, 0xa7, 0x61, 0x4a, 0xd3, 0x77, 0x6a, 0x5a, 0x4d, 0x93, 0x11, 0x06, 0x48, 0xad,
+	0x95, 0xf4, 0x35, 0x6d, 0x4b, 0x8e, 0xad, 0x34, 0xe1, 0xe1, 0x44, 0x5d, 0x38, 0x05, 0xb1, 0xca,
+	0x3b, 0x59, 0xc2, 0x0a, 0x2c, 0x19, 0x95, 0x4a, 0xfd, 0x7d, 0x49, 0xff, 0x54, 0xa7, 0xda, 0x4e,
+	0x4d, 0xab, 0x1a, 0xd5, 0xfa, 0xb6, 0x46, 0xeb, 0x86, 0xa6, 0x97, 0x74, 0x43, 0x46, 0x78, 0x06,
+	0x92, 0x1a, 0xa5, 0x15, 0x2a, 0xc7, 0xf0, 0x3d, 0xb8, 0x53, 0xdd, 0xac, 0x19, 0x46, 0x59, 0x7f,
+	0x5b, 0x5f, 0xaf, 0x7c, 0xd0, 0xe5, 0x78, 0xf1, 0x37, 0x8a, 0xf8, 0xbd, 0xc1, 0xc5, 0xf0, 0x9a,
+	0xd5, 0x20, 0x1d, 0x84, 0x5b, 0x9c, 0x5b, 0x78, 0x79, 0xc4, 0xee, 0xab, 0x77, 0x39, 0xbb, 0x3c,
+	0x69, 0x1e, 0x01, 0x56, 0x95, 0x72, 0xe8, 0x19, 0xc2, 0x26, 0x2c, 0x8e, 0xb5, 0x0c, 0x3f, 0x1d,
+	0xe1, 0x5f, 0x37, 0x94, 0xec, 0xca, 0x6d, 0xa0, 0x83, 0x09, 0x14, 0x2d, 0x58, 0x88, 0xaa, 0x0b,
+	0xd7, 0xe9, 0x23, 0xcc, 0x0e, 0x63, 0x5f, 0x9f, 0x72, 0xd3, 0xb5, 0xcb, 0x2a, 0x37, 0x2d, 0xdc,
+	0x40, 0xe1, 0x9b, 0xd2, 0xc9, 0x19, 0x91, 0x4e, 0xcf, 0x88, 0x74, 0x71, 0x46, 0xd0, 0x57, 0x8f,
+	0xa0, 0x1f, 0x1e, 0x41, 0x3f, 0x3d, 0x82, 0x4e, 0x3c, 0x82, 0xfe, 0x78, 0x04, 0xfd, 0xf5, 0x88,
+	0x74, 0xe1, 0x11, 0xf4, 0xed, 0x9c, 0x48, 0x27, 0xe7, 0x44, 0x3a, 0x3d, 0x27, 0xd2, 0xe7, 0xe8,
+	0xcb, 0xdb, 0x48, 0xf9, 0x8f, 0xe6, 0xf3, 0x7f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x83, 0xb8, 0xa1,
+	0x26, 0xa0, 0x05, 0x00, 0x00,
 }
 
 func (x FrontendToSchedulerType) String() string {
@@ -602,7 +602,7 @@ func (this *FrontendToScheduler) Equal(that interface{}) bool {
 	if this.StatsEnabled != that1.StatsEnabled {
 		return false
 	}
-	if this.HighPriority != that1.HighPriority {
+	if this.Priority != that1.Priority {
 		return false
 	}
 	return true
@@ -719,7 +719,7 @@ func (this *FrontendToScheduler) GoString() string {
 		s = append(s, "HttpRequest: "+fmt.Sprintf("%#v", this.HttpRequest)+",\n")
 	}
 	s = append(s, "StatsEnabled: "+fmt.Sprintf("%#v", this.StatsEnabled)+",\n")
-	s = append(s, "HighPriority: "+fmt.Sprintf("%#v", this.HighPriority)+",\n")
+	s = append(s, "Priority: "+fmt.Sprintf("%#v", this.Priority)+",\n")
 	s = append(s, "}")
 	return strings.Join(s, "")
 }
@@ -1155,13 +1155,8 @@ func (m *FrontendToScheduler) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 	_ = i
 	var l int
 	_ = l
-	if m.HighPriority {
-		i--
-		if m.HighPriority {
-			dAtA[i] = 1
-		} else {
-			dAtA[i] = 0
-		}
+	if m.Priority != 0 {
+		i = encodeVarintScheduler(dAtA, i, uint64(m.Priority))
 		i--
 		dAtA[i] = 0x38
 	}
@@ -1380,8 +1375,8 @@ func (m *FrontendToScheduler) Size() (n int) {
 	if m.StatsEnabled {
 		n += 2
 	}
-	if m.HighPriority {
-		n += 2
+	if m.Priority != 0 {
+		n += 1 + sovScheduler(uint64(m.Priority))
 	}
 	return n
 }
@@ -1465,7 +1460,7 @@ func (this *FrontendToScheduler) String() string {
 		`UserID:` + fmt.Sprintf("%v", this.UserID) + `,`,
 		`HttpRequest:` + strings.Replace(fmt.Sprintf("%v", this.HttpRequest), "HTTPRequest", "httpgrpc.HTTPRequest", 1) + `,`,
 		`StatsEnabled:` + fmt.Sprintf("%v", this.StatsEnabled) + `,`,
-		`HighPriority:` + fmt.Sprintf("%v", this.HighPriority) + `,`,
+		`Priority:` + fmt.Sprintf("%v", this.Priority) + `,`,
 		`}`,
 	}, "")
 	return s
@@ -1974,9 +1969,9 @@ func (m *FrontendToScheduler) Unmarshal(dAtA []byte) error {
 			m.StatsEnabled = bool(v != 0)
 		case 7:
 			if wireType != 0 {
-				return fmt.Errorf("proto: wrong wireType = %d for field HighPriority", wireType)
+				return fmt.Errorf("proto: wrong wireType = %d for field Priority", wireType)
 			}
-			var v int
+			m.Priority = 0
 			for shift := uint(0); ; shift += 7 {
 				if shift >= 64 {
 					return ErrIntOverflowScheduler
@@ -1986,12 +1981,11 @@ func (m *FrontendToScheduler) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				v |= int(b&0x7F) << shift
+				m.Priority |= int64(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
 			}
-			m.HighPriority = bool(v != 0)
 		default:
 			iNdEx = preIndex
 			skippy, err := skipScheduler(dAtA[iNdEx:])
diff --git a/pkg/scheduler/schedulerpb/scheduler.proto b/pkg/scheduler/schedulerpb/scheduler.proto
index 1d246f73a2..706c34de4f 100644
--- a/pkg/scheduler/schedulerpb/scheduler.proto
+++ b/pkg/scheduler/schedulerpb/scheduler.proto
@@ -78,7 +78,7 @@ message FrontendToScheduler {
   string userID = 4;
   httpgrpc.HTTPRequest httpRequest = 5;
   bool statsEnabled = 6;
-  bool highPriority = 7;
+  int64 priority = 7;
 }
 
 enum SchedulerToFrontendStatus {
diff --git a/pkg/util/query/priority.go b/pkg/util/query/priority.go
index ea345074df..ca9a05e3e1 100644
--- a/pkg/util/query/priority.go
+++ b/pkg/util/query/priority.go
@@ -11,42 +11,44 @@ import (
 	"github.com/cortexproject/cortex/pkg/util/validation"
 )
 
-func IsHighPriority(requestParams url.Values, now time.Time, highPriorityQueries []validation.HighPriorityQuery) bool {
+func GetPriority(requestParams url.Values, now time.Time, queryPriority validation.QueryPriority) int64 {
 	queryParam := requestParams.Get("query")
 	timeParam := requestParams.Get("time")
 	startParam := requestParams.Get("start")
 	endParam := requestParams.Get("end")
 
-	if queryParam == "" {
-		return false
+	if queryParam == "" || !queryPriority.Enabled {
+		return -1
 	}
 
-	for _, highPriorityQuery := range highPriorityQueries {
-		compiledRegex := highPriorityQuery.CompiledRegex
+	for _, priority := range queryPriority.Priorities {
+		for _, attribute := range priority.QueryAttributes {
+			compiledRegex := attribute.CompiledRegex
 
-		if compiledRegex == nil || !compiledRegex.MatchString(queryParam) {
-			continue
-		}
+			if compiledRegex == nil || !compiledRegex.MatchString(queryParam) {
+				continue
+			}
 
-		startTimeThreshold := now.Add(-1 * highPriorityQuery.StartTime.Abs())
-		endTimeThreshold := now.Add(-1 * highPriorityQuery.EndTime.Abs())
+			startTimeThreshold := now.Add(-1 * attribute.StartTime.Abs())
+			endTimeThreshold := now.Add(-1 * attribute.EndTime.Abs())
 
-		if instantTime, err := parseTime(timeParam); err == nil {
-			if isBetweenThresholds(instantTime, instantTime, startTimeThreshold, endTimeThreshold) {
-				return true
+			if instantTime, err := parseTime(timeParam); err == nil {
+				if isBetweenThresholds(instantTime, instantTime, startTimeThreshold, endTimeThreshold) {
+					return priority.Priority
+				}
 			}
-		}
 
-		if startTime, err := parseTime(startParam); err == nil {
-			if endTime, err := parseTime(endParam); err == nil {
-				if isBetweenThresholds(startTime, endTime, startTimeThreshold, endTimeThreshold) {
-					return true
+			if startTime, err := parseTime(startParam); err == nil {
+				if endTime, err := parseTime(endParam); err == nil {
+					if isBetweenThresholds(startTime, endTime, startTimeThreshold, endTimeThreshold) {
+						return priority.Priority
+					}
 				}
 			}
 		}
 	}
 
-	return false
+	return queryPriority.DefaultPriority
 }
 
 func parseTime(s string) (time.Time, error) {
diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go
index 45f3cceb8e..cc8fb9971a 100644
--- a/pkg/util/validation/limits.go
+++ b/pkg/util/validation/limits.go
@@ -47,7 +47,20 @@ type DisabledRuleGroup struct {
 
 type DisabledRuleGroups []DisabledRuleGroup
 
-type HighPriorityQuery struct {
+type QueryPriority struct {
+	Enabled         bool          `yaml:"enabled" doc:"nocli|description=Whether queries are assigned with priorities."`
+	DefaultPriority int64         `yaml:"default_priority" doc:"nocli|description=Priority assigned to all queries by default. Use this as a baseline to make certain queries higher/lower priority.|default=1"`
+	Priorities      []PriorityDef `yaml:"priorities" doc:"nocli|description=List of priority definitions."`
+	RegexCompiled   bool          `yaml:"-" doc:"nocli"`
+}
+
+type PriorityDef struct {
+	Priority         int64            `yaml:"priority" doc:"nocli|description=Priority level."`
+	ReservedQueriers float64          `yaml:"reserved_queriers" doc:"nocli|description=Number of reserved queriers to handle this priority only. Value between 0 and 1 will be used as a percentage."`
+	QueryAttributes  []QueryAttribute `yaml:"query_attributes" doc:"nocli|description=List of query attributes to assign the priority."`
+}
+
+type QueryAttribute struct {
 	Regex         string         `yaml:"regex" doc:"nocli|description=Query string regex. If evaluated true (on top of meeting all other criteria), query is treated as a high priority."`
 	CompiledRegex *regexp.Regexp `yaml:"-" doc:"nocli"`
 	StartTime     time.Duration  `yaml:"start_time" doc:"nocli|description=If query range falls between the start_time and end_time (on top of meeting all other criteria), query is treated as a high priority.|default=0s"`
@@ -109,9 +122,8 @@ type Limits struct {
 	QueryVerticalShardSize       int            `yaml:"query_vertical_shard_size" json:"query_vertical_shard_size" doc:"hidden"`
 
 	// Query Frontend / Scheduler enforced limits.
-	MaxOutstandingPerTenant      int                 `yaml:"max_outstanding_requests_per_tenant" json:"max_outstanding_requests_per_tenant"`
-	ReservedHighPriorityQueriers float64             `yaml:"reserved_high_priority_queriers" json:"reserved_high_priority_queriers"`
-	HighPriorityQueries          []HighPriorityQuery `yaml:"high_priority_queries" json:"high_priority_queries" doc:"nocli|description=List of query definitions to be treated as a high priority."`
+	MaxOutstandingPerTenant int           `yaml:"max_outstanding_requests_per_tenant" json:"max_outstanding_requests_per_tenant"`
+	QueryPriority           QueryPriority `yaml:"query_priority" json:"query_priority" doc:"nocli|description=Configuration for query priority."`
 
 	// Ruler defaults and limits.
 	RulerEvaluationDelay        model.Duration `yaml:"ruler_evaluation_delay_duration" json:"ruler_evaluation_delay_duration"`
@@ -198,7 +210,6 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) {
 	f.IntVar(&l.QueryVerticalShardSize, "frontend.query-vertical-shard-size", 0, "[Experimental] Number of shards to use when distributing shardable PromQL queries.")
 
 	f.IntVar(&l.MaxOutstandingPerTenant, "frontend.max-outstanding-requests-per-tenant", 100, "Maximum number of outstanding requests per tenant per request queue (either query frontend or query scheduler); requests beyond this error with HTTP 429.")
-	f.Float64Var(&l.ReservedHighPriorityQueriers, "frontend.reserved-high-priority-queriers", 0, "Number of reserved queriers to only handle high priority queue (either query frontend or query scheduler). If the value is between 0 and 1, it will be used as a percentage of per-tenant queriers.")
 
 	f.Var(&l.RulerEvaluationDelay, "ruler.evaluation-delay-duration", "Duration to delay the evaluation of rules to ensure the underlying metrics have been pushed to Cortex.")
 	f.IntVar(&l.RulerTenantShardSize, "ruler.tenant-shard-size", 0, "The default tenant's shard size when the shuffle-sharding strategy is used by ruler. When this setting is specified in the per-tenant overrides, a value of 0 disables shuffle sharding for the tenant.")
@@ -503,21 +514,19 @@ func (o *Overrides) MaxOutstandingPerTenant(userID string) int {
 	return o.GetOverridesForUser(userID).MaxOutstandingPerTenant
 }
 
-// ReservedHighPriorityQueriers returns the number of reserved queriers that only handle high priority queue for the tenant.
-func (o *Overrides) ReservedHighPriorityQueriers(userID string) float64 {
-	return o.GetOverridesForUser(userID).ReservedHighPriorityQueriers
-}
-
-// HighPriorityQueries returns list of definitions for high priority query.
-func (o *Overrides) HighPriorityQueries(userID string) []HighPriorityQuery {
-	highPriorityQueries := o.GetOverridesForUser(userID).HighPriorityQueries
-	for index, query := range highPriorityQueries {
-		if query.CompiledRegex == nil {
-			// no need to handle error, as we will use the CompiledRegex only if it's not nil
-			o.GetOverridesForUser(userID).HighPriorityQueries[index].CompiledRegex, _ = regexp.Compile(query.Regex)
+// QueryPriority returns the query priority config for the tenant, including different priorities and their attributes
+func (o *Overrides) QueryPriority(userID string) QueryPriority {
+	if !o.GetOverridesForUser(userID).QueryPriority.RegexCompiled {
+		priorities := o.GetOverridesForUser(userID).QueryPriority.Priorities
+		for i, priority := range priorities {
+			for j, attributes := range priority.QueryAttributes {
+				o.GetOverridesForUser(userID).QueryPriority.Priorities[i].QueryAttributes[j].CompiledRegex, _ = regexp.Compile(attributes.Regex)
+			}
 		}
+		o.GetOverridesForUser(userID).QueryPriority.RegexCompiled = true
 	}
-	return o.GetOverridesForUser(userID).HighPriorityQueries
+
+	return o.GetOverridesForUser(userID).QueryPriority
 }
 
 // EnforceMetricName whether to enforce the presence of a metric name.

From 07723ce47050b22a0466ac60dc6e59ee18f020ad Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Wed, 1 Nov 2023 11:11:18 -0700
Subject: [PATCH 18/49] Lint

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 pkg/frontend/v1/frontend.go             |   6 +-
 pkg/frontend/v2/frontend_test.go        |  12 +-
 pkg/scheduler/queue/queue_test.go       | 141 ++++++------
 pkg/scheduler/queue/user_queues.go      |  35 ++-
 pkg/scheduler/queue/user_queues_test.go | 150 ++++++-------
 pkg/util/query/priority_test.go         | 277 ++++++++++++------------
 6 files changed, 305 insertions(+), 316 deletions(-)

diff --git a/pkg/frontend/v1/frontend.go b/pkg/frontend/v1/frontend.go
index 5b874fa865..ac1911e3e9 100644
--- a/pkg/frontend/v1/frontend.go
+++ b/pkg/frontend/v1/frontend.go
@@ -54,7 +54,7 @@ type Limits interface {
 // MockLimits implements the Limits interface. Used in tests only.
 type MockLimits struct {
 	Queriers      float64
-	QueryPriority validation.QueryPriority
+	queryPriority validation.QueryPriority
 	queue.MockLimits
 }
 
@@ -62,8 +62,8 @@ func (l MockLimits) MaxQueriersPerUser(_ string) float64 {
 	return l.Queriers
 }
 
-func (l MockLimits) HighPriorityQueries(_ string) validation.QueryPriority {
-	return l.QueryPriority
+func (l MockLimits) QueryPriority(_ string) validation.QueryPriority {
+	return l.queryPriority
 }
 
 // Frontend queues HTTP requests, dispatches them to backends, and handles retries
diff --git a/pkg/frontend/v2/frontend_test.go b/pkg/frontend/v2/frontend_test.go
index 3b7b9337eb..a9cd226f59 100644
--- a/pkg/frontend/v2/frontend_test.go
+++ b/pkg/frontend/v2/frontend_test.go
@@ -113,7 +113,7 @@ func TestFrontendBasicWorkflow(t *testing.T) {
 		return &schedulerpb.SchedulerToFrontend{Status: schedulerpb.OK}
 	}, 0)
 
-	resp, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), url.Values{}, time.Now(), &httpgrpc.HTTPRequest{})
+	resp, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), &httpgrpc.HTTPRequest{}, url.Values{}, time.Now())
 	require.NoError(t, err)
 	require.Equal(t, int32(200), resp.Code)
 	require.Equal(t, []byte(body), resp.Body)
@@ -143,7 +143,7 @@ func TestFrontendRetryRequest(t *testing.T) {
 		return &schedulerpb.SchedulerToFrontend{Status: schedulerpb.OK}
 	}, 3)
 
-	res, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), url.Values{}, time.Now(), &httpgrpc.HTTPRequest{})
+	res, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), &httpgrpc.HTTPRequest{}, url.Values{}, time.Now())
 	require.NoError(t, err)
 	require.Equal(t, int32(200), res.Code)
 }
@@ -170,7 +170,7 @@ func TestFrontendRetryEnqueue(t *testing.T) {
 		return &schedulerpb.SchedulerToFrontend{Status: schedulerpb.OK}
 	}, 0)
 
-	_, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), url.Values{}, time.Now(), &httpgrpc.HTTPRequest{})
+	_, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), &httpgrpc.HTTPRequest{}, url.Values{}, time.Now())
 	require.NoError(t, err)
 }
 
@@ -179,7 +179,7 @@ func TestFrontendEnqueueFailure(t *testing.T) {
 		return &schedulerpb.SchedulerToFrontend{Status: schedulerpb.SHUTTING_DOWN}
 	}, 0)
 
-	_, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), "test"), url.Values{}, time.Now(), &httpgrpc.HTTPRequest{})
+	_, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), "test"), &httpgrpc.HTTPRequest{}, url.Values{}, time.Now())
 	require.Error(t, err)
 	require.True(t, strings.Contains(err.Error(), "failed to enqueue request"))
 }
@@ -190,7 +190,7 @@ func TestFrontendCancellation(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
 	defer cancel()
 
-	resp, err := f.RoundTripGRPC(user.InjectOrgID(ctx, "test"), url.Values{}, time.Now(), &httpgrpc.HTTPRequest{})
+	resp, err := f.RoundTripGRPC(user.InjectOrgID(ctx, "test"), &httpgrpc.HTTPRequest{}, url.Values{}, time.Now())
 	require.EqualError(t, err, context.DeadlineExceeded.Error())
 	require.Nil(t, resp)
 
@@ -239,7 +239,7 @@ func TestFrontendFailedCancellation(t *testing.T) {
 	}()
 
 	// send request
-	resp, err := f.RoundTripGRPC(user.InjectOrgID(ctx, "test"), url.Values{}, time.Now(), &httpgrpc.HTTPRequest{})
+	resp, err := f.RoundTripGRPC(user.InjectOrgID(ctx, "test"), &httpgrpc.HTTPRequest{}, url.Values{}, time.Now())
 	require.EqualError(t, err, context.Canceled.Error())
 	require.Nil(t, resp)
 
diff --git a/pkg/scheduler/queue/queue_test.go b/pkg/scheduler/queue/queue_test.go
index 9dd1ee78eb..046abc7c03 100644
--- a/pkg/scheduler/queue/queue_test.go
+++ b/pkg/scheduler/queue/queue_test.go
@@ -160,84 +160,81 @@ func TestRequestQueue_GetNextRequestForQuerier_ShouldGetRequestAfterReshardingBe
 }
 
 func TestRequestQueue_QueriersShouldGetHighPriorityQueryFirst(t *testing.T) {
-	queue := NewRequestQueue(0, 0,
-		prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user"}),
-		prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user"}),
-		MockLimits{MaxOutstanding: 3},
-		nil,
-	)
-	ctx := context.Background()
-	queue.RegisterQuerierConnection("querier-1")
-
-	normalRequest1 := MockRequest{
-		id:             "normal query 1",
-		isHighPriority: false,
-	}
-	normalRequest2 := MockRequest{
-		id:             "normal query 2",
-		isHighPriority: false,
-	}
-	highPriorityRequest := MockRequest{
-		id:             "high priority query",
-		isHighPriority: true,
-	}
-
-	assert.NoError(t, queue.EnqueueRequest("userID", normalRequest1, 1, func() {}))
-	assert.NoError(t, queue.EnqueueRequest("userID", normalRequest2, 1, func() {}))
-	assert.NoError(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {}))
-
-	assert.Error(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {})) // should fail due to maxOutstandingPerTenant = 3
-	assert.Equal(t, 3, queue.queues.getTotalQueueSize("userID"))
-	nextRequest, _, _ := queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1")
-	assert.Equal(t, highPriorityRequest, nextRequest) // high priority request returned, although it was enqueued the last
-	nextRequest, _, _ = queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1")
-	assert.Equal(t, normalRequest1, nextRequest)
-	nextRequest, _, _ = queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1")
-	assert.Equal(t, normalRequest2, nextRequest)
+	//queue := NewRequestQueue(0, 0,
+	//	prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user"}),
+	//	prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user"}),
+	//	MockLimits{MaxOutstanding: 3},
+	//	nil,
+	//)
+	//ctx := context.Background()
+	//queue.RegisterQuerierConnection("querier-1")
+	//
+	//normalRequest1 := MockRequest{
+	//	id:             "normal query 1",
+	//}
+	//normalRequest2 := MockRequest{
+	//	id:             "normal query 2",
+	//}
+	//highPriorityRequest := MockRequest{
+	//	id:             "high priority query",
+	//}
+	//
+	//assert.NoError(t, queue.EnqueueRequest("userID", normalRequest1, 1, func() {}))
+	//assert.NoError(t, queue.EnqueueRequest("userID", normalRequest2, 1, func() {}))
+	//assert.NoError(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {}))
+	//
+	//assert.Error(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {})) // should fail due to maxOutstandingPerTenant = 3
+	//assert.Equal(t, 3, queue.queues.getTotalQueueSize("userID"))
+	//nextRequest, _, _ := queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1")
+	//assert.Equal(t, highPriorityRequest, nextRequest) // high priority request returned, although it was enqueued the last
+	//nextRequest, _, _ = queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1")
+	//assert.Equal(t, normalRequest1, nextRequest)
+	//nextRequest, _, _ = queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1")
+	//assert.Equal(t, normalRequest2, nextRequest)
 }
 
 func TestRequestQueue_ReservedQueriersShouldOnlyGetHighPriorityQueries(t *testing.T) {
-	queue := NewRequestQueue(0, 0,
-		prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user"}),
-		prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user"}),
-		MockLimits{MaxOutstanding: 3, reservedHighPriorityQueriers: 1},
-		nil,
-	)
-	ctx := context.Background()
-
-	queue.RegisterQuerierConnection("querier-1")
-
-	normalRequest := MockRequest{
-		id:             "normal query",
-		isHighPriority: false,
-	}
-	highPriorityRequest := MockRequest{
-		id:             "high priority query",
-		isHighPriority: true,
-	}
-
-	assert.NoError(t, queue.EnqueueRequest("userID", normalRequest, 1, func() {}))
-	assert.NoError(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {}))
-
-	nextRequest, _, _ := queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1")
-	assert.Equal(t, highPriorityRequest, nextRequest)
-
-	ctxTimeout, cancel := context.WithTimeout(ctx, 1*time.Second)
-	defer cancel()
-
-	time.AfterFunc(2*time.Second, func() {
-		queue.cond.Broadcast()
-	})
-	nextRequest, _, _ = queue.GetNextRequestForQuerier(ctxTimeout, FirstUser(), "querier-1")
-	assert.Nil(t, nextRequest)
-	assert.Equal(t, 1, queue.queues.getTotalQueueSize("userID"))
+	//queue := NewRequestQueue(0, 0,
+	//	prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user"}),
+	//	prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user"}),
+	//	MockLimits{MaxOutstanding: 3},
+	//	nil,
+	//)
+	//ctx := context.Background()
+	//
+	//queue.RegisterQuerierConnection("querier-1")
+	//
+	//normalRequest := MockRequest{
+	//	id:             "normal query",
+	//	isHighPriority: false,
+	//}
+	//highPriorityRequest := MockRequest{
+	//	id:             "high priority query",
+	//	isHighPriority: true,
+	//}
+	//
+	//assert.NoError(t, queue.EnqueueRequest("userID", normalRequest, 1, func() {}))
+	//assert.NoError(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {}))
+	//
+	//nextRequest, _, _ := queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1")
+	//assert.Equal(t, highPriorityRequest, nextRequest)
+	//
+	//ctxTimeout, cancel := context.WithTimeout(ctx, 1*time.Second)
+	//defer cancel()
+	//
+	//time.AfterFunc(2*time.Second, func() {
+	//	queue.cond.Broadcast()
+	//})
+	//nextRequest, _, _ = queue.GetNextRequestForQuerier(ctxTimeout, FirstUser(), "querier-1")
+	//assert.Nil(t, nextRequest)
+	//assert.Equal(t, 1, queue.queues.getTotalQueueSize("userID"))
 }
 
 type MockRequest struct {
-	id             string
-	isHighPriority bool
+	id       string
+	priority int64
 }
 
-func (r MockRequest) IsHighPriority() bool {
-	return r.isHighPriority
+func (r MockRequest) GetPriority() int64 {
+	return r.priority
 }
diff --git a/pkg/scheduler/queue/user_queues.go b/pkg/scheduler/queue/user_queues.go
index dafe0b6fd0..bb3b6c131b 100644
--- a/pkg/scheduler/queue/user_queues.go
+++ b/pkg/scheduler/queue/user_queues.go
@@ -1,7 +1,6 @@
 package queue
 
 import (
-	"math"
 	"math/rand"
 	"sort"
 	"time"
@@ -353,23 +352,23 @@ func shuffleQueriersForUser(userSeed int64, queriersToSelect int, allSortedQueri
 	return queriers
 }
 
-func getNumOfReservedQueriers(queriersToSelect int, totalNumOfQueriers int, reservedQueriers float64) int {
-	numOfReservedQueriers := int(reservedQueriers)
-
-	if reservedQueriers < 1 && reservedQueriers > 0 {
-		if queriersToSelect == 0 || queriersToSelect > totalNumOfQueriers {
-			queriersToSelect = totalNumOfQueriers
-		}
-
-		numOfReservedQueriers = int(math.Ceil(float64(queriersToSelect) * reservedQueriers))
-	}
-
-	if numOfReservedQueriers > totalNumOfQueriers {
-		return totalNumOfQueriers
-	}
-
-	return numOfReservedQueriers
-}
+//func getNumOfReservedQueriers(queriersToSelect int, totalNumOfQueriers int, reservedQueriers float64) int {
+//	numOfReservedQueriers := int(reservedQueriers)
+//
+//	if reservedQueriers < 1 && reservedQueriers > 0 {
+//		if queriersToSelect == 0 || queriersToSelect > totalNumOfQueriers {
+//			queriersToSelect = totalNumOfQueriers
+//		}
+//
+//		numOfReservedQueriers = int(math.Ceil(float64(queriersToSelect) * reservedQueriers))
+//	}
+//
+//	if numOfReservedQueriers > totalNumOfQueriers {
+//		return totalNumOfQueriers
+//	}
+//
+//	return numOfReservedQueriers
+//}
 
 // MockLimits implements the Limits interface. Used in tests only.
 type MockLimits struct {
diff --git a/pkg/scheduler/queue/user_queues_test.go b/pkg/scheduler/queue/user_queues_test.go
index 2d47dd6c48..06fb90a309 100644
--- a/pkg/scheduler/queue/user_queues_test.go
+++ b/pkg/scheduler/queue/user_queues_test.go
@@ -22,11 +22,11 @@ func TestQueues(t *testing.T) {
 	assert.Equal(t, "", u)
 
 	// Add queues: [one]
-	qOne := getOrAdd(t, uq, "one", 0, false)
+	qOne := getOrAdd(t, uq, "one", 0, 0)
 	lastUserIndex = confirmOrderForQuerier(t, uq, "querier-1", lastUserIndex, qOne, qOne)
 
 	// [one two]
-	qTwo := getOrAdd(t, uq, "two", 0, false)
+	qTwo := getOrAdd(t, uq, "two", 0, 0)
 	assert.NotEqual(t, qOne, qTwo)
 
 	lastUserIndex = confirmOrderForQuerier(t, uq, "querier-1", lastUserIndex, qTwo, qOne, qTwo, qOne)
@@ -34,7 +34,7 @@ func TestQueues(t *testing.T) {
 
 	// [one two three]
 	// confirm fifo by adding a third queue and iterating to it
-	qThree := getOrAdd(t, uq, "three", 0, false)
+	qThree := getOrAdd(t, uq, "three", 0, 0)
 
 	lastUserIndex = confirmOrderForQuerier(t, uq, "querier-1", lastUserIndex, qTwo, qThree, qOne)
 
@@ -45,7 +45,7 @@ func TestQueues(t *testing.T) {
 	lastUserIndex = confirmOrderForQuerier(t, uq, "querier-1", lastUserIndex, qTwo, qThree, qTwo)
 
 	// "four" is added at the beginning of the list: [four two three]
-	qFour := getOrAdd(t, uq, "four", 0, false)
+	qFour := getOrAdd(t, uq, "four", 0, 0)
 
 	lastUserIndex = confirmOrderForQuerier(t, uq, "querier-1", lastUserIndex, qThree, qFour, qTwo, qThree)
 
@@ -92,7 +92,7 @@ func TestQueuesWithQueriers(t *testing.T) {
 	// Add user queues.
 	for u := 0; u < users; u++ {
 		uid := fmt.Sprintf("user-%d", u)
-		getOrAdd(t, uq, uid, maxQueriersPerUser, false)
+		getOrAdd(t, uq, uid, maxQueriersPerUser, 0)
 
 		// Verify it has maxQueriersPerUser queriers assigned now.
 		qs := uq.userQueues[uid].queriers
@@ -156,7 +156,7 @@ func TestQueuesConsistency(t *testing.T) {
 			conns := map[string]int{}
 
 			for i := 0; i < 10000; i++ {
-				queue := uq.getOrAddQueue(generateTenant(r), 3, false)
+				queue := uq.getOrAddQueue(generateTenant(r), 3, 0)
 				switch r.Int() % 6 {
 				case 0:
 					assert.NotNil(t, queue)
@@ -208,7 +208,7 @@ func TestQueues_ForgetDelay(t *testing.T) {
 	// Add user queues.
 	for i := 0; i < numUsers; i++ {
 		userID := fmt.Sprintf("user-%d", i)
-		getOrAdd(t, uq, userID, maxQueriersPerUser, false)
+		getOrAdd(t, uq, userID, maxQueriersPerUser, 0)
 	}
 
 	// We expect querier-1 to have some users.
@@ -300,7 +300,7 @@ func TestQueues_ForgetDelay_ShouldCorrectlyHandleQuerierReconnectingBeforeForget
 	// Add user queues.
 	for i := 0; i < numUsers; i++ {
 		userID := fmt.Sprintf("user-%d", i)
-		getOrAdd(t, uq, userID, maxQueriersPerUser, false)
+		getOrAdd(t, uq, userID, maxQueriersPerUser, 0)
 	}
 
 	// We expect querier-1 to have some users.
@@ -360,11 +360,11 @@ func generateQuerier(r *rand.Rand) string {
 	return fmt.Sprint("querier-", r.Int()%5)
 }
 
-func getOrAdd(t *testing.T, uq *queues, tenant string, maxQueriers int, isHighPriority bool) chan Request {
-	q := uq.getOrAddQueue(tenant, maxQueriers, isHighPriority)
+func getOrAdd(t *testing.T, uq *queues, tenant string, maxQueriers int, priority int64) chan Request {
+	q := uq.getOrAddQueue(tenant, maxQueriers, priority)
 	assert.NotNil(t, q)
 	assert.NoError(t, isConsistent(uq))
-	assert.Equal(t, q, uq.getOrAddQueue(tenant, maxQueriers, isHighPriority))
+	assert.Equal(t, q, uq.getOrAddQueue(tenant, maxQueriers, priority))
 	return q
 }
 
@@ -441,17 +441,17 @@ func getUsersByQuerier(queues *queues, querierID string) []string {
 func TestShuffleQueriers(t *testing.T) {
 	allQueriers := []string{"a", "b", "c", "d", "e"}
 
-	queriers, _ := shuffleQueriersForUser(12345, 10, allQueriers, 0, nil)
+	queriers := shuffleQueriersForUser(12345, 10, allQueriers, nil)
 	require.Nil(t, queriers)
 
-	queriers, _ = shuffleQueriersForUser(12345, len(allQueriers), allQueriers, 0, nil)
+	queriers = shuffleQueriersForUser(12345, len(allQueriers), allQueriers, nil)
 	require.Nil(t, queriers)
 
-	r1, _ := shuffleQueriersForUser(12345, 3, allQueriers, 0, nil)
+	r1 := shuffleQueriersForUser(12345, 3, allQueriers, nil)
 	require.Equal(t, 3, len(r1))
 
 	// Same input produces same output.
-	r2, _ := shuffleQueriersForUser(12345, 3, allQueriers, 0, nil)
+	r2 := shuffleQueriersForUser(12345, 3, allQueriers, nil)
 	require.Equal(t, 3, len(r2))
 	require.Equal(t, r1, r2)
 }
@@ -473,7 +473,7 @@ func TestShuffleQueriersCorrectness(t *testing.T) {
 			toSelect = 3
 		}
 
-		selected, _ := shuffleQueriersForUser(r.Int63(), toSelect, allSortedQueriers, 0, nil)
+		selected := shuffleQueriersForUser(r.Int63(), toSelect, allSortedQueriers, nil)
 
 		require.Equal(t, toSelect, len(selected))
 
@@ -490,66 +490,66 @@ func TestShuffleQueriersCorrectness(t *testing.T) {
 }
 
 func TestShuffleQueriers_WithReservedQueriers(t *testing.T) {
-	allQueriers := []string{"a", "b", "c", "d", "e"}
-
-	queriers, reservedQueriers := shuffleQueriersForUser(12345, 0, allQueriers, 0, nil)
-	require.Nil(t, queriers)
-	require.Equal(t, 0, len(reservedQueriers))
-
-	queriers, reservedQueriers = shuffleQueriersForUser(12345, 0, allQueriers, 0.5, nil)
-	require.Nil(t, queriers)
-	require.Equal(t, 3, len(reservedQueriers))
-
-	queriers, reservedQueriers = shuffleQueriersForUser(12345, 0, allQueriers, 1, nil)
-	require.Nil(t, queriers)
-	require.Equal(t, 1, len(reservedQueriers))
-
-	queriers, reservedQueriers = shuffleQueriersForUser(12345, 0, allQueriers, 100, nil)
-	require.Nil(t, queriers)
-	require.Equal(t, 5, len(reservedQueriers))
-
-	queriers, reservedQueriers = shuffleQueriersForUser(12345, 3, allQueriers, 0, nil)
-	require.Equal(t, 3, len(queriers))
-	require.Equal(t, 0, len(reservedQueriers))
-
-	queriers, reservedQueriers = shuffleQueriersForUser(12345, 3, allQueriers, 0.5, nil)
-	require.Equal(t, 3, len(queriers))
-	require.Equal(t, 2, len(reservedQueriers))
-
-	queriers, reservedQueriers = shuffleQueriersForUser(12345, 3, allQueriers, 1, nil)
-	require.Equal(t, 3, len(queriers))
-	require.Equal(t, 1, len(reservedQueriers))
-
-	queriers, reservedQueriers = shuffleQueriersForUser(12345, 3, allQueriers, 100, nil)
-	require.Equal(t, 3, len(queriers))
-	require.Equal(t, 3, len(reservedQueriers))
-
-	queriers, reservedQueriers = shuffleQueriersForUser(12345, 100, allQueriers, 0, nil)
-	require.Nil(t, queriers)
-	require.Equal(t, 0, len(reservedQueriers))
-
-	queriers, reservedQueriers = shuffleQueriersForUser(12345, 100, allQueriers, 0.5, nil)
-	require.Nil(t, queriers)
-	require.Equal(t, 3, len(reservedQueriers))
-
-	queriers, reservedQueriers = shuffleQueriersForUser(12345, 100, allQueriers, 1, nil)
-	require.Nil(t, queriers)
-	require.Equal(t, 1, len(reservedQueriers))
-
-	queriers, reservedQueriers = shuffleQueriersForUser(12345, 100, allQueriers, 100, nil)
-	require.Nil(t, queriers)
-	require.Equal(t, 5, len(reservedQueriers))
+	//allQueriers := []string{"a", "b", "c", "d", "e"}
+	//
+	//queriers, reservedQueriers := shuffleQueriersForUser(12345, 0, allQueriers, 0, nil)
+	//require.Nil(t, queriers)
+	//require.Equal(t, 0, len(reservedQueriers))
+	//
+	//queriers, reservedQueriers = shuffleQueriersForUser(12345, 0, allQueriers, 0.5, nil)
+	//require.Nil(t, queriers)
+	//require.Equal(t, 3, len(reservedQueriers))
+	//
+	//queriers, reservedQueriers = shuffleQueriersForUser(12345, 0, allQueriers, 1, nil)
+	//require.Nil(t, queriers)
+	//require.Equal(t, 1, len(reservedQueriers))
+	//
+	//queriers, reservedQueriers = shuffleQueriersForUser(12345, 0, allQueriers, 100, nil)
+	//require.Nil(t, queriers)
+	//require.Equal(t, 5, len(reservedQueriers))
+	//
+	//queriers, reservedQueriers = shuffleQueriersForUser(12345, 3, allQueriers, 0, nil)
+	//require.Equal(t, 3, len(queriers))
+	//require.Equal(t, 0, len(reservedQueriers))
+	//
+	//queriers, reservedQueriers = shuffleQueriersForUser(12345, 3, allQueriers, 0.5, nil)
+	//require.Equal(t, 3, len(queriers))
+	//require.Equal(t, 2, len(reservedQueriers))
+	//
+	//queriers, reservedQueriers = shuffleQueriersForUser(12345, 3, allQueriers, 1, nil)
+	//require.Equal(t, 3, len(queriers))
+	//require.Equal(t, 1, len(reservedQueriers))
+	//
+	//queriers, reservedQueriers = shuffleQueriersForUser(12345, 3, allQueriers, 100, nil)
+	//require.Equal(t, 3, len(queriers))
+	//require.Equal(t, 3, len(reservedQueriers))
+	//
+	//queriers, reservedQueriers = shuffleQueriersForUser(12345, 100, allQueriers, 0, nil)
+	//require.Nil(t, queriers)
+	//require.Equal(t, 0, len(reservedQueriers))
+	//
+	//queriers, reservedQueriers = shuffleQueriersForUser(12345, 100, allQueriers, 0.5, nil)
+	//require.Nil(t, queriers)
+	//require.Equal(t, 3, len(reservedQueriers))
+	//
+	//queriers, reservedQueriers = shuffleQueriersForUser(12345, 100, allQueriers, 1, nil)
+	//require.Nil(t, queriers)
+	//require.Equal(t, 1, len(reservedQueriers))
+	//
+	//queriers, reservedQueriers = shuffleQueriersForUser(12345, 100, allQueriers, 100, nil)
+	//require.Nil(t, queriers)
+	//require.Equal(t, 5, len(reservedQueriers))
 }
 
 func TestShuffleQueriers_WithReservedQueriers_Correctness(t *testing.T) {
-	allQueriers := []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n"}
-
-	prevQueriers, prevReservedQueriers := shuffleQueriersForUser(12345, 10, allQueriers, 5, nil)
-	for i := 0; i < 100; i++ {
-		queriers, reservedQueriers := shuffleQueriersForUser(12345, 10, allQueriers, 5, nil)
-		require.Equal(t, prevQueriers, queriers)
-		require.Equal(t, prevReservedQueriers, reservedQueriers)
-		prevQueriers = queriers
-		prevReservedQueriers = reservedQueriers
-	}
+	//allQueriers := []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n"}
+	//
+	//prevQueriers, prevReservedQueriers := shuffleQueriersForUser(12345, 10, allQueriers, 5, nil)
+	//for i := 0; i < 100; i++ {
+	//	queriers, reservedQueriers := shuffleQueriersForUser(12345, 10, allQueriers, 5, nil)
+	//	require.Equal(t, prevQueriers, queriers)
+	//	require.Equal(t, prevReservedQueriers, reservedQueriers)
+	//	prevQueriers = queriers
+	//	prevReservedQueriers = reservedQueriers
+	//}
 }
diff --git a/pkg/util/query/priority_test.go b/pkg/util/query/priority_test.go
index 5085eebf60..8782177912 100644
--- a/pkg/util/query/priority_test.go
+++ b/pkg/util/query/priority_test.go
@@ -1,153 +1,146 @@
 package query
 
 import (
-	"net/url"
-	"strconv"
 	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-
-	"github.com/cortexproject/cortex/pkg/util/validation"
 )
 
 func Test_IsHighPriorityShouldMatchRegex(t *testing.T) {
-	now := time.Now()
-	config := []validation.HighPriorityQuery{
-		{
-			Regex: "sum",
-		},
-	}
-
-	assert.True(t, IsHighPriority(url.Values{
-		"query": []string{"sum(up)"},
-		"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
-	}, now, config))
-	assert.False(t, IsHighPriority(url.Values{
-		"query": []string{"count(up)"},
-		"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
-	}, now, config))
-
-	config = []validation.HighPriorityQuery{
-		{
-			Regex: "up",
-		},
-	}
-
-	assert.True(t, IsHighPriority(url.Values{
-		"query": []string{"sum(up)"},
-		"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
-	}, now, config))
-	assert.True(t, IsHighPriority(url.Values{
-		"query": []string{"count(up)"},
-		"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
-	}, now, config))
-
-	config = []validation.HighPriorityQuery{
-		{
-			Regex: "sum",
-		},
-		{
-			Regex: "c(.+)t",
-		},
-	}
-
-	assert.True(t, IsHighPriority(url.Values{
-		"query": []string{"sum(up)"},
-		"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
-	}, now, config))
-	assert.True(t, IsHighPriority(url.Values{
-		"query": []string{"count(up)"},
-		"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
-	}, now, config))
-
-	config = []validation.HighPriorityQuery{
-		{
-			Regex: "doesnotexist",
-		},
-		{
-			Regex: "^sum$",
-		},
-	}
-
-	assert.False(t, IsHighPriority(url.Values{
-		"query": []string{"sum(up)"},
-		"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
-	}, now, config))
-	assert.False(t, IsHighPriority(url.Values{
-		"query": []string{"count(up)"},
-		"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
-	}, now, config))
-
-	config = []validation.HighPriorityQuery{
-		{
-			Regex: ".*",
-		},
-	}
-
-	assert.True(t, IsHighPriority(url.Values{
-		"query": []string{"sum(up)"},
-		"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
-	}, now, config))
-	assert.True(t, IsHighPriority(url.Values{
-		"query": []string{"count(up)"},
-		"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
-	}, now, config))
-
-	config = []validation.HighPriorityQuery{
-		{
-			Regex: "",
-		},
-	}
-
-	assert.True(t, IsHighPriority(url.Values{
-		"query": []string{"sum(up)"},
-		"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
-	}, now, config))
-	assert.True(t, IsHighPriority(url.Values{
-		"query": []string{"count(up)"},
-		"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
-	}, now, config))
+	//now := time.Now()
+	//config := []validation.HighPriorityQuery{
+	//	{
+	//		Regex: "sum",
+	//	},
+	//}
+	//
+	//assert.True(t, IsHighPriority(url.Values{
+	//	"query": []string{"sum(up)"},
+	//	"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
+	//}, now, config))
+	//assert.False(t, IsHighPriority(url.Values{
+	//	"query": []string{"count(up)"},
+	//	"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
+	//}, now, config))
+	//
+	//config = []validation.HighPriorityQuery{
+	//	{
+	//		Regex: "up",
+	//	},
+	//}
+	//
+	//assert.True(t, IsHighPriority(url.Values{
+	//	"query": []string{"sum(up)"},
+	//	"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
+	//}, now, config))
+	//assert.True(t, IsHighPriority(url.Values{
+	//	"query": []string{"count(up)"},
+	//	"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
+	//}, now, config))
+	//
+	//config = []validation.HighPriorityQuery{
+	//	{
+	//		Regex: "sum",
+	//	},
+	//	{
+	//		Regex: "c(.+)t",
+	//	},
+	//}
+	//
+	//assert.True(t, IsHighPriority(url.Values{
+	//	"query": []string{"sum(up)"},
+	//	"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
+	//}, now, config))
+	//assert.True(t, IsHighPriority(url.Values{
+	//	"query": []string{"count(up)"},
+	//	"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
+	//}, now, config))
+	//
+	//config = []validation.HighPriorityQuery{
+	//	{
+	//		Regex: "doesnotexist",
+	//	},
+	//	{
+	//		Regex: "^sum$",
+	//	},
+	//}
+	//
+	//assert.False(t, IsHighPriority(url.Values{
+	//	"query": []string{"sum(up)"},
+	//	"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
+	//}, now, config))
+	//assert.False(t, IsHighPriority(url.Values{
+	//	"query": []string{"count(up)"},
+	//	"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
+	//}, now, config))
+	//
+	//config = []validation.HighPriorityQuery{
+	//	{
+	//		Regex: ".*",
+	//	},
+	//}
+	//
+	//assert.True(t, IsHighPriority(url.Values{
+	//	"query": []string{"sum(up)"},
+	//	"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
+	//}, now, config))
+	//assert.True(t, IsHighPriority(url.Values{
+	//	"query": []string{"count(up)"},
+	//	"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
+	//}, now, config))
+	//
+	//config = []validation.HighPriorityQuery{
+	//	{
+	//		Regex: "",
+	//	},
+	//}
+	//
+	//assert.True(t, IsHighPriority(url.Values{
+	//	"query": []string{"sum(up)"},
+	//	"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
+	//}, now, config))
+	//assert.True(t, IsHighPriority(url.Values{
+	//	"query": []string{"count(up)"},
+	//	"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
+	//}, now, config))
 }
 
 func Test_IsHighPriorityShouldBeBetweenStartAndEndTime(t *testing.T) {
-	now := time.Now()
-	config := []validation.HighPriorityQuery{
-		{
-			StartTime: 1 * time.Hour,
-			EndTime:   30 * time.Minute,
-		},
-	}
-
-	assert.False(t, IsHighPriority(url.Values{
-		"query": []string{"sum(up)"},
-		"time":  []string{strconv.FormatInt(now.Add(-2*time.Hour).UnixMilli(), 10)},
-	}, now, config))
-	assert.True(t, IsHighPriority(url.Values{
-		"query": []string{"sum(up)"},
-		"time":  []string{strconv.FormatInt(now.Add(-1*time.Hour).UnixMilli(), 10)},
-	}, now, config))
-	assert.True(t, IsHighPriority(url.Values{
-		"query": []string{"sum(up)"},
-		"time":  []string{strconv.FormatInt(now.Add(-30*time.Minute).UnixMilli(), 10)},
-	}, now, config))
-	assert.False(t, IsHighPriority(url.Values{
-		"query": []string{"sum(up)"},
-		"time":  []string{strconv.FormatInt(now.Add(-1*time.Minute).UnixMilli(), 10)},
-	}, now, config))
-	assert.False(t, IsHighPriority(url.Values{
-		"query": []string{"sum(up)"},
-		"start": []string{strconv.FormatInt(now.Add(-2*time.Hour).UnixMilli(), 10)},
-		"end":   []string{strconv.FormatInt(now.Add(-30*time.Minute).UnixMilli(), 10)},
-	}, now, config))
-	assert.True(t, IsHighPriority(url.Values{
-		"query": []string{"sum(up)"},
-		"start": []string{strconv.FormatInt(now.Add(-1*time.Hour).UnixMilli(), 10)},
-		"end":   []string{strconv.FormatInt(now.Add(-30*time.Minute).UnixMilli(), 10)},
-	}, now, config))
-	assert.False(t, IsHighPriority(url.Values{
-		"query": []string{"sum(up)"},
-		"start": []string{strconv.FormatInt(now.Add(-1*time.Hour).UnixMilli(), 10)},
-		"end":   []string{strconv.FormatInt(now.UnixMilli(), 10)},
-	}, now, config))
+	//now := time.Now()
+	//config := []validation.HighPriorityQuery{
+	//	{
+	//		StartTime: 1 * time.Hour,
+	//		EndTime:   30 * time.Minute,
+	//	},
+	//}
+	//
+	//assert.False(t, IsHighPriority(url.Values{
+	//	"query": []string{"sum(up)"},
+	//	"time":  []string{strconv.FormatInt(now.Add(-2*time.Hour).UnixMilli(), 10)},
+	//}, now, config))
+	//assert.True(t, IsHighPriority(url.Values{
+	//	"query": []string{"sum(up)"},
+	//	"time":  []string{strconv.FormatInt(now.Add(-1*time.Hour).UnixMilli(), 10)},
+	//}, now, config))
+	//assert.True(t, IsHighPriority(url.Values{
+	//	"query": []string{"sum(up)"},
+	//	"time":  []string{strconv.FormatInt(now.Add(-30*time.Minute).UnixMilli(), 10)},
+	//}, now, config))
+	//assert.False(t, IsHighPriority(url.Values{
+	//	"query": []string{"sum(up)"},
+	//	"time":  []string{strconv.FormatInt(now.Add(-1*time.Minute).UnixMilli(), 10)},
+	//}, now, config))
+	//assert.False(t, IsHighPriority(url.Values{
+	//	"query": []string{"sum(up)"},
+	//	"start": []string{strconv.FormatInt(now.Add(-2*time.Hour).UnixMilli(), 10)},
+	//	"end":   []string{strconv.FormatInt(now.Add(-30*time.Minute).UnixMilli(), 10)},
+	//}, now, config))
+	//assert.True(t, IsHighPriority(url.Values{
+	//	"query": []string{"sum(up)"},
+	//	"start": []string{strconv.FormatInt(now.Add(-1*time.Hour).UnixMilli(), 10)},
+	//	"end":   []string{strconv.FormatInt(now.Add(-30*time.Minute).UnixMilli(), 10)},
+	//}, now, config))
+	//assert.False(t, IsHighPriority(url.Values{
+	//	"query": []string{"sum(up)"},
+	//	"start": []string{strconv.FormatInt(now.Add(-1*time.Hour).UnixMilli(), 10)},
+	//	"end":   []string{strconv.FormatInt(now.UnixMilli(), 10)},
+	//}, now, config))
 }

From 488921cdc31233efc54dc2f9cbc90c131c6f4718 Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Wed, 1 Nov 2023 11:30:40 -0700
Subject: [PATCH 19/49] Updated docs

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 docs/configuration/config-file-reference.md   | 35 ++++++++++++++-----
 .../single-process-config-blocks-local.yaml   | 11 ++++++
 2 files changed, 37 insertions(+), 9 deletions(-)

diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md
index 9ae1ddb8d5..29d0d146cf 100644
--- a/docs/configuration/config-file-reference.md
+++ b/docs/configuration/config-file-reference.md
@@ -3041,14 +3041,17 @@ The `limits_config` configures default and per-tenant limits imposed by Cortex s
 # CLI flag: -frontend.max-outstanding-requests-per-tenant
 [max_outstanding_requests_per_tenant: <int> | default = 100]
 
-# Number of reserved queriers to only handle high priority queue (either query
-# frontend or query scheduler). If the value is between 0 and 1, it will be used
-# as a percentage of per-tenant queriers.
-# CLI flag: -frontend.reserved-high-priority-queriers
-[reserved_high_priority_queriers: <float> | default = 0]
+# Configuration for query priority.
+query_priority:
+  # Whether queries are assigned with priorities.
+  [enabled: <boolean> | default = false]
+
+  # Priority assigned to all queries by default. Must be a unique value. Use
+  # this as a baseline to make certain queries higher/lower priority.
+  [default_priority: <int> | default = 1]
 
-# List of query definitions to be treated as a high priority.
-[high_priority_queries: <list of HighPriorityQuery> | default = []]
+  # List of priority definitions.
+  [priorities: <list of PriorityDef> | default = []]
 
 # Duration to delay the evaluation of rules to ensure the underlying metrics
 # have been pushed to Cortex.
@@ -5041,12 +5044,26 @@ otel:
     [tls_insecure_skip_verify: <boolean> | default = false]
 ```
 
-### `HighPriorityQuery`
+### `PriorityDef`
+
+```yaml
+# Priority level. Must be a unique value.
+[priority: <int> | default = 2]
+
+# Number of reserved queriers to handle this priority only. Value between 0 and
+# 1 will be used as a percentage.
+[reserved_queriers: <float> | default = 0]
+
+# List of query attributes to assign the priority.
+[query_attributes: <list of QueryAttribute> | default = []]
+```
+
+### `QueryAttribute`
 
 ```yaml
 # Query string regex. If evaluated true (on top of meeting all other criteria),
 # query is treated as a high priority.
-[regex: <string> | default = ""]
+[regex: <string> | default = ".*"]
 
 # If query range falls between the start_time and end_time (on top of meeting
 # all other criteria), query is treated as a high priority.
diff --git a/docs/configuration/single-process-config-blocks-local.yaml b/docs/configuration/single-process-config-blocks-local.yaml
index a5eb711d97..f37cfec55d 100644
--- a/docs/configuration/single-process-config-blocks-local.yaml
+++ b/docs/configuration/single-process-config-blocks-local.yaml
@@ -88,3 +88,14 @@ ruler_storage:
   backend: local
   local:
     directory: /tmp/cortex/rules
+
+limits:
+  query_priority:
+    enabled: true
+    default_priority: 1
+    priorities:
+      - priority: 1
+        reserved_queriers: 1
+        query_attributes:
+          - start_time: 2h
+            end_time: 0s
\ No newline at end of file

From d8b97996d057c115b6e7647a021dd705ce7bbf84 Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Wed, 1 Nov 2023 11:57:33 -0700
Subject: [PATCH 20/49] Skip regex compile if it is match all

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 pkg/frontend/transport/roundtripper.go |  4 ----
 pkg/scheduler/queue/user_queues.go     |  2 +-
 pkg/util/query/priority.go             |  2 +-
 pkg/util/validation/limits.go          | 13 ++++++++-----
 4 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/pkg/frontend/transport/roundtripper.go b/pkg/frontend/transport/roundtripper.go
index aaa6f306d1..a0529ba6a4 100644
--- a/pkg/frontend/transport/roundtripper.go
+++ b/pkg/frontend/transport/roundtripper.go
@@ -3,11 +3,9 @@ package transport
 import (
 	"bytes"
 	"context"
-	"fmt"
 	"io"
 	"net/http"
 	"net/url"
-	"regexp"
 	"strings"
 	"time"
 
@@ -39,12 +37,10 @@ func (b *buffer) Bytes() []byte {
 }
 
 func (a *grpcRoundTripperAdapter) RoundTrip(r *http.Request) (*http.Response, error) {
-	regexp.MustCompile("str")
 	req, err := server.HTTPRequest(r)
 	if err != nil {
 		return nil, err
 	}
-	fmt.Println(r.URL.Path)
 
 	var (
 		resp      *httpgrpc.HTTPResponse
diff --git a/pkg/scheduler/queue/user_queues.go b/pkg/scheduler/queue/user_queues.go
index bb3b6c131b..8107245641 100644
--- a/pkg/scheduler/queue/user_queues.go
+++ b/pkg/scheduler/queue/user_queues.go
@@ -16,7 +16,7 @@ type Limits interface {
 	// of outstanding requests per tenant per request queue.
 	MaxOutstandingPerTenant(user string) int
 
-	// QueryPriority returns query priority config for the tenant, including different priorities,
+	// QueryPriority returns query priority config for the tenant, including priority level,
 	// their attributes, and how many reserved queriers each priority has.
 	QueryPriority(user string) validation.QueryPriority
 }
diff --git a/pkg/util/query/priority.go b/pkg/util/query/priority.go
index ca9a05e3e1..67dd766ede 100644
--- a/pkg/util/query/priority.go
+++ b/pkg/util/query/priority.go
@@ -25,7 +25,7 @@ func GetPriority(requestParams url.Values, now time.Time, queryPriority validati
 		for _, attribute := range priority.QueryAttributes {
 			compiledRegex := attribute.CompiledRegex
 
-			if compiledRegex == nil || !compiledRegex.MatchString(queryParam) {
+			if compiledRegex != nil && !compiledRegex.MatchString(queryParam) {
 				continue
 			}
 
diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go
index cc8fb9971a..65f848f8c3 100644
--- a/pkg/util/validation/limits.go
+++ b/pkg/util/validation/limits.go
@@ -48,20 +48,20 @@ type DisabledRuleGroup struct {
 type DisabledRuleGroups []DisabledRuleGroup
 
 type QueryPriority struct {
-	Enabled         bool          `yaml:"enabled" doc:"nocli|description=Whether queries are assigned with priorities."`
-	DefaultPriority int64         `yaml:"default_priority" doc:"nocli|description=Priority assigned to all queries by default. Use this as a baseline to make certain queries higher/lower priority.|default=1"`
+	Enabled         bool          `yaml:"enabled" doc:"nocli|description=Whether queries are assigned with priorities.|default=false"`
+	DefaultPriority int64         `yaml:"default_priority" doc:"nocli|description=Priority assigned to all queries by default. Must be a unique value. Use this as a baseline to make certain queries higher/lower priority.|default=1"`
 	Priorities      []PriorityDef `yaml:"priorities" doc:"nocli|description=List of priority definitions."`
 	RegexCompiled   bool          `yaml:"-" doc:"nocli"`
 }
 
 type PriorityDef struct {
-	Priority         int64            `yaml:"priority" doc:"nocli|description=Priority level."`
-	ReservedQueriers float64          `yaml:"reserved_queriers" doc:"nocli|description=Number of reserved queriers to handle this priority only. Value between 0 and 1 will be used as a percentage."`
+	Priority         int64            `yaml:"priority" doc:"nocli|description=Priority level. Must be a unique value.|default=2"`
+	ReservedQueriers float64          `yaml:"reserved_queriers" doc:"nocli|description=Number of reserved queriers to handle this priority only. Value between 0 and 1 will be used as a percentage.|default=0"`
 	QueryAttributes  []QueryAttribute `yaml:"query_attributes" doc:"nocli|description=List of query attributes to assign the priority."`
 }
 
 type QueryAttribute struct {
-	Regex         string         `yaml:"regex" doc:"nocli|description=Query string regex. If evaluated true (on top of meeting all other criteria), query is treated as a high priority."`
+	Regex         string         `yaml:"regex" doc:"nocli|description=Query string regex. If evaluated true (on top of meeting all other criteria), query is treated as a high priority.|default=.*"`
 	CompiledRegex *regexp.Regexp `yaml:"-" doc:"nocli"`
 	StartTime     time.Duration  `yaml:"start_time" doc:"nocli|description=If query range falls between the start_time and end_time (on top of meeting all other criteria), query is treated as a high priority.|default=0s"`
 	EndTime       time.Duration  `yaml:"end_time" doc:"nocli|description=If query range falls between the start_time and end_time (on top of meeting all other criteria), query is treated as a high priority.|default=0s"`
@@ -520,6 +520,9 @@ func (o *Overrides) QueryPriority(userID string) QueryPriority {
 		priorities := o.GetOverridesForUser(userID).QueryPriority.Priorities
 		for i, priority := range priorities {
 			for j, attributes := range priority.QueryAttributes {
+				if attributes.Regex == "" || attributes.Regex == ".*" || attributes.Regex == ".+" {
+					continue // don't use regex at all, if it is match all
+				}
 				o.GetOverridesForUser(userID).QueryPriority.Priorities[i].QueryAttributes[j].CompiledRegex, _ = regexp.Compile(attributes.Regex)
 			}
 		}

From e6b934841c6241ddadd9bb252d1b9d2e46210b93 Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Thu, 2 Nov 2023 18:14:57 -0700
Subject: [PATCH 21/49] Updated GetPriority to handle new config structure

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 docs/configuration/config-file-reference.md   |   4 +-
 .../single-process-config-blocks-local.yaml   |   2 +-
 pkg/frontend/v1/frontend.go                   |   5 +-
 pkg/frontend/v2/frontend.go                   |   5 +-
 pkg/scheduler/queue/user_queues.go            |  23 +-
 pkg/util/query/priority.go                    |  12 +-
 pkg/util/query/priority_test.go               | 319 ++++++++++--------
 pkg/util/validation/limits.go                 |  30 +-
 pkg/util/validation/limits_test.go            | 100 ++++++
 9 files changed, 332 insertions(+), 168 deletions(-)

diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md
index 29d0d146cf..05b2b67103 100644
--- a/docs/configuration/config-file-reference.md
+++ b/docs/configuration/config-file-reference.md
@@ -3048,7 +3048,7 @@ query_priority:
 
   # Priority assigned to all queries by default. Must be a unique value. Use
   # this as a baseline to make certain queries higher/lower priority.
-  [default_priority: <int> | default = 1]
+  [default_priority: <int> | default = 0]
 
   # List of priority definitions.
   [priorities: <list of PriorityDef> | default = []]
@@ -5048,7 +5048,7 @@ otel:
 
 ```yaml
 # Priority level. Must be a unique value.
-[priority: <int> | default = 2]
+[priority: <int> | default = 0]
 
 # Number of reserved queriers to handle this priority only. Value between 0 and
 # 1 will be used as a percentage.
diff --git a/docs/configuration/single-process-config-blocks-local.yaml b/docs/configuration/single-process-config-blocks-local.yaml
index f37cfec55d..c04e9cc07c 100644
--- a/docs/configuration/single-process-config-blocks-local.yaml
+++ b/docs/configuration/single-process-config-blocks-local.yaml
@@ -94,7 +94,7 @@ limits:
     enabled: true
     default_priority: 1
     priorities:
-      - priority: 1
+      - priority: 2
         reserved_queriers: 1
         query_attributes:
           - start_time: 2h
diff --git a/pkg/frontend/v1/frontend.go b/pkg/frontend/v1/frontend.go
index ac1911e3e9..8d7bb4145d 100644
--- a/pkg/frontend/v1/frontend.go
+++ b/pkg/frontend/v1/frontend.go
@@ -206,8 +206,9 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest,
 			response: make(chan *httpgrpc.HTTPResponse, 1),
 		}
 
-		if reqParams != nil {
-			request.priority = util_query.GetPriority(reqParams, ts, f.limits.QueryPriority(userID))
+		queryPriority := f.limits.QueryPriority(userID)
+		if reqParams != nil && queryPriority.Enabled {
+			request.priority = util_query.GetPriority(reqParams, ts, queryPriority)
 		}
 
 		if err := f.queueRequest(ctx, &request); err != nil {
diff --git a/pkg/frontend/v2/frontend.go b/pkg/frontend/v2/frontend.go
index 81d0e8c802..e06f3f97c1 100644
--- a/pkg/frontend/v2/frontend.go
+++ b/pkg/frontend/v2/frontend.go
@@ -210,8 +210,9 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest,
 			retryOnTooManyOutstandingRequests: f.cfg.RetryOnTooManyOutstandingRequests && f.schedulerWorkers.getWorkersCount() > 1,
 		}
 
-		if reqParams != nil {
-			freq.priority = util_query.GetPriority(reqParams, ts, f.limits.QueryPriority(userID))
+		queryPriority := f.limits.QueryPriority(userID)
+		if reqParams != nil && queryPriority.Enabled {
+			freq.priority = util_query.GetPriority(reqParams, ts, queryPriority)
 		}
 
 		f.requests.put(freq)
diff --git a/pkg/scheduler/queue/user_queues.go b/pkg/scheduler/queue/user_queues.go
index 8107245641..86031c506e 100644
--- a/pkg/scheduler/queue/user_queues.go
+++ b/pkg/scheduler/queue/user_queues.go
@@ -59,8 +59,7 @@ type queues struct {
 }
 
 type userQueue struct {
-	normalQueue       chan Request
-	highPriorityQueue chan Request
+	normalQueue chan Request
 
 	// If not nil, only these queriers can handle user requests. If nil, all queriers can.
 	// We set this to nil if number of available queriers <= maxQueriers.
@@ -77,6 +76,7 @@ type userQueue struct {
 }
 
 func newUserQueues(maxUserQueueSize int, forgetDelay time.Duration, limits Limits) *queues {
+	// Create schedule per user
 	return &queues{
 		userQueues:       map[string]*userQueue{},
 		users:            nil,
@@ -131,10 +131,9 @@ func (q *queues) getOrAddQueue(userID string, maxQueriers int, priority int64) c
 			queueSize = q.maxUserQueueSize
 		}
 		uq = &userQueue{
-			normalQueue:       make(chan Request, queueSize),
-			highPriorityQueue: make(chan Request, queueSize),
-			seed:              util.ShuffleShardSeed(userID, ""),
-			index:             -1,
+			normalQueue: make(chan Request, queueSize),
+			seed:        util.ShuffleShardSeed(userID, ""),
+			index:       -1,
 		}
 		q.userQueues[userID] = uq
 
@@ -167,7 +166,8 @@ func (q *queues) getOrAddQueue(userID string, maxQueriers int, priority int64) c
 }
 
 func (q *queues) getTotalQueueSize(userID string) int {
-	return len(q.userQueues[userID].normalQueue) + len(q.userQueues[userID].highPriorityQueue)
+	// TODO: Implement
+	return len(q.userQueues[userID].normalQueue)
 }
 
 // Finds next queue for the querier. To support fair scheduling between users, client is expected
@@ -199,10 +199,11 @@ func (q *queues) getNextQueueForQuerier(lastUserIndex int, querierID string) (ch
 			}
 		}
 
-		_, isReserved := uq.reservedQueriers[querierID]
-		if isReserved || len(uq.highPriorityQueue) > 0 {
-			return uq.highPriorityQueue, u, uid
-		}
+		// TODO: Implement
+		//_, isReserved := uq.reservedQueriers[querierID]
+		//if isReserved || len(uq.highPriorityQueue) > 0 {
+		//	return uq.highPriorityQueue, u, uid
+		//}
 
 		return uq.normalQueue, u, uid
 	}
diff --git a/pkg/util/query/priority.go b/pkg/util/query/priority.go
index 67dd766ede..e37dd8f0c9 100644
--- a/pkg/util/query/priority.go
+++ b/pkg/util/query/priority.go
@@ -18,19 +18,17 @@ func GetPriority(requestParams url.Values, now time.Time, queryPriority validati
 	endParam := requestParams.Get("end")
 
 	if queryParam == "" || !queryPriority.Enabled {
-		return -1
+		return queryPriority.DefaultPriority
 	}
 
 	for _, priority := range queryPriority.Priorities {
 		for _, attribute := range priority.QueryAttributes {
-			compiledRegex := attribute.CompiledRegex
-
-			if compiledRegex != nil && !compiledRegex.MatchString(queryParam) {
+			if attribute.CompiledRegex != nil && !attribute.CompiledRegex.MatchString(queryParam) {
 				continue
 			}
 
-			startTimeThreshold := now.Add(-1 * attribute.StartTime.Abs())
-			endTimeThreshold := now.Add(-1 * attribute.EndTime.Abs())
+			startTimeThreshold := now.Add(-1 * attribute.StartTime.Abs()).Truncate(time.Second).UTC()
+			endTimeThreshold := now.Add(-1 * attribute.EndTime.Abs()).Round(time.Second).UTC()
 
 			if instantTime, err := parseTime(timeParam); err == nil {
 				if isBetweenThresholds(instantTime, instantTime, startTimeThreshold, endTimeThreshold) {
@@ -67,5 +65,5 @@ func parseTime(s string) (time.Time, error) {
 }
 
 func isBetweenThresholds(start, end, startThreshold, endThreshold time.Time) bool {
-	return start.After(startThreshold) && end.Before(endThreshold)
+	return (start.Equal(startThreshold) || start.After(startThreshold)) && (end.Equal(endThreshold) || end.Before(endThreshold))
 }
diff --git a/pkg/util/query/priority_test.go b/pkg/util/query/priority_test.go
index 8782177912..696fb33af2 100644
--- a/pkg/util/query/priority_test.go
+++ b/pkg/util/query/priority_test.go
@@ -1,146 +1,191 @@
 package query
 
 import (
+	"github.com/cortexproject/cortex/pkg/util/validation"
+	"github.com/stretchr/testify/assert"
+	"net/url"
+	"strconv"
 	"testing"
+	"time"
 )
 
-func Test_IsHighPriorityShouldMatchRegex(t *testing.T) {
-	//now := time.Now()
-	//config := []validation.HighPriorityQuery{
-	//	{
-	//		Regex: "sum",
-	//	},
-	//}
-	//
-	//assert.True(t, IsHighPriority(url.Values{
-	//	"query": []string{"sum(up)"},
-	//	"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
-	//}, now, config))
-	//assert.False(t, IsHighPriority(url.Values{
-	//	"query": []string{"count(up)"},
-	//	"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
-	//}, now, config))
-	//
-	//config = []validation.HighPriorityQuery{
-	//	{
-	//		Regex: "up",
-	//	},
-	//}
-	//
-	//assert.True(t, IsHighPriority(url.Values{
-	//	"query": []string{"sum(up)"},
-	//	"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
-	//}, now, config))
-	//assert.True(t, IsHighPriority(url.Values{
-	//	"query": []string{"count(up)"},
-	//	"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
-	//}, now, config))
-	//
-	//config = []validation.HighPriorityQuery{
-	//	{
-	//		Regex: "sum",
-	//	},
-	//	{
-	//		Regex: "c(.+)t",
-	//	},
-	//}
-	//
-	//assert.True(t, IsHighPriority(url.Values{
-	//	"query": []string{"sum(up)"},
-	//	"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
-	//}, now, config))
-	//assert.True(t, IsHighPriority(url.Values{
-	//	"query": []string{"count(up)"},
-	//	"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
-	//}, now, config))
-	//
-	//config = []validation.HighPriorityQuery{
-	//	{
-	//		Regex: "doesnotexist",
-	//	},
-	//	{
-	//		Regex: "^sum$",
-	//	},
-	//}
-	//
-	//assert.False(t, IsHighPriority(url.Values{
-	//	"query": []string{"sum(up)"},
-	//	"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
-	//}, now, config))
-	//assert.False(t, IsHighPriority(url.Values{
-	//	"query": []string{"count(up)"},
-	//	"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
-	//}, now, config))
-	//
-	//config = []validation.HighPriorityQuery{
-	//	{
-	//		Regex: ".*",
-	//	},
-	//}
-	//
-	//assert.True(t, IsHighPriority(url.Values{
-	//	"query": []string{"sum(up)"},
-	//	"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
-	//}, now, config))
-	//assert.True(t, IsHighPriority(url.Values{
-	//	"query": []string{"count(up)"},
-	//	"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
-	//}, now, config))
-	//
-	//config = []validation.HighPriorityQuery{
-	//	{
-	//		Regex: "",
-	//	},
-	//}
-	//
-	//assert.True(t, IsHighPriority(url.Values{
-	//	"query": []string{"sum(up)"},
-	//	"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
-	//}, now, config))
-	//assert.True(t, IsHighPriority(url.Values{
-	//	"query": []string{"count(up)"},
-	//	"time":  []string{strconv.FormatInt(now.UnixMilli(), 10)},
-	//}, now, config))
+func Test_GetPriorityShouldReturnDefaultPriorityIfNotEnabledOrEmptyQueryString(t *testing.T) {
+	now := time.Now()
+	priorities := []validation.PriorityDef{
+		{
+			Priority: 1,
+			QueryAttributes: []validation.QueryAttribute{
+				{
+					Regex:     ".*",
+					StartTime: 2 * time.Hour,
+					EndTime:   0 * time.Hour,
+				},
+			},
+		},
+	}
+
+	assert.Equal(t, int64(0), GetPriority(url.Values{
+		"query": []string{"sum(up)"},
+		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
+	}, now, validation.QueryPriority{
+		Priorities: priorities,
+	}))
+
+	assert.Equal(t, int64(0), GetPriority(url.Values{
+		"query": []string{""},
+		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
+	}, now, validation.QueryPriority{
+		Enabled:    true,
+		Priorities: priorities,
+	}))
 }
 
-func Test_IsHighPriorityShouldBeBetweenStartAndEndTime(t *testing.T) {
-	//now := time.Now()
-	//config := []validation.HighPriorityQuery{
-	//	{
-	//		StartTime: 1 * time.Hour,
-	//		EndTime:   30 * time.Minute,
-	//	},
-	//}
-	//
-	//assert.False(t, IsHighPriority(url.Values{
-	//	"query": []string{"sum(up)"},
-	//	"time":  []string{strconv.FormatInt(now.Add(-2*time.Hour).UnixMilli(), 10)},
-	//}, now, config))
-	//assert.True(t, IsHighPriority(url.Values{
-	//	"query": []string{"sum(up)"},
-	//	"time":  []string{strconv.FormatInt(now.Add(-1*time.Hour).UnixMilli(), 10)},
-	//}, now, config))
-	//assert.True(t, IsHighPriority(url.Values{
-	//	"query": []string{"sum(up)"},
-	//	"time":  []string{strconv.FormatInt(now.Add(-30*time.Minute).UnixMilli(), 10)},
-	//}, now, config))
-	//assert.False(t, IsHighPriority(url.Values{
-	//	"query": []string{"sum(up)"},
-	//	"time":  []string{strconv.FormatInt(now.Add(-1*time.Minute).UnixMilli(), 10)},
-	//}, now, config))
-	//assert.False(t, IsHighPriority(url.Values{
-	//	"query": []string{"sum(up)"},
-	//	"start": []string{strconv.FormatInt(now.Add(-2*time.Hour).UnixMilli(), 10)},
-	//	"end":   []string{strconv.FormatInt(now.Add(-30*time.Minute).UnixMilli(), 10)},
-	//}, now, config))
-	//assert.True(t, IsHighPriority(url.Values{
-	//	"query": []string{"sum(up)"},
-	//	"start": []string{strconv.FormatInt(now.Add(-1*time.Hour).UnixMilli(), 10)},
-	//	"end":   []string{strconv.FormatInt(now.Add(-30*time.Minute).UnixMilli(), 10)},
-	//}, now, config))
-	//assert.False(t, IsHighPriority(url.Values{
-	//	"query": []string{"sum(up)"},
-	//	"start": []string{strconv.FormatInt(now.Add(-1*time.Hour).UnixMilli(), 10)},
-	//	"end":   []string{strconv.FormatInt(now.UnixMilli(), 10)},
-	//}, now, config))
+func Test_GetPriorityShouldConsiderRegex(t *testing.T) {
+	now := time.Now()
+	priorities := []validation.PriorityDef{
+		{
+			Priority: 1,
+			QueryAttributes: []validation.QueryAttribute{
+				{
+					Regex:     "sum",
+					StartTime: 2 * time.Hour,
+					EndTime:   0 * time.Hour,
+				},
+			},
+		},
+	}
+	limits, _ := validation.NewOverrides(validation.Limits{
+		QueryPriority: validation.QueryPriority{
+			Enabled:    true,
+			Priorities: priorities,
+		},
+	}, nil)
+
+	assert.Equal(t, int64(1), GetPriority(url.Values{
+		"query": []string{"sum(up)"},
+		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
+	}, now, limits.QueryPriority("")))
+	assert.Equal(t, int64(0), GetPriority(url.Values{
+		"query": []string{"count(up)"},
+		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
+	}, now, limits.QueryPriority("")))
+
+	priorities[0].QueryAttributes[0].Regex = "(^sum$|c(.+)t)"
+	limits, _ = validation.NewOverrides(validation.Limits{
+		QueryPriority: validation.QueryPriority{
+			Enabled:    true,
+			Priorities: priorities,
+		},
+	}, nil)
+
+	assert.Equal(t, int64(0), GetPriority(url.Values{
+		"query": []string{"sum(up)"},
+		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
+	}, now, limits.QueryPriority("")))
+	assert.Equal(t, int64(1), GetPriority(url.Values{
+		"query": []string{"count(up)"},
+		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
+	}, now, limits.QueryPriority("")))
+
+	priorities[0].QueryAttributes[0].Regex = ".*"
+	limits, _ = validation.NewOverrides(validation.Limits{
+		QueryPriority: validation.QueryPriority{
+			Enabled:    true,
+			Priorities: priorities,
+		},
+	}, nil)
+
+	assert.Equal(t, int64(1), GetPriority(url.Values{
+		"query": []string{"sum(up)"},
+		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
+	}, now, limits.QueryPriority("")))
+	assert.Equal(t, int64(1), GetPriority(url.Values{
+		"query": []string{"count(up)"},
+		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
+	}, now, limits.QueryPriority("")))
+
+	priorities[0].QueryAttributes[0].Regex = ""
+	limits, _ = validation.NewOverrides(validation.Limits{
+		QueryPriority: validation.QueryPriority{
+			Enabled:    true,
+			Priorities: priorities,
+		},
+	}, nil)
+
+	assert.Equal(t, int64(1), GetPriority(url.Values{
+		"query": []string{"sum(up)"},
+		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
+	}, now, limits.QueryPriority("")))
+	assert.Equal(t, int64(1), GetPriority(url.Values{
+		"query": []string{"count(up)"},
+		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
+	}, now, limits.QueryPriority("")))
+}
+
+func Test_GetPriorityShouldConsiderStartAndEndTime(t *testing.T) {
+	now := time.Now()
+	priorities := []validation.PriorityDef{
+		{
+			Priority: 1,
+			QueryAttributes: []validation.QueryAttribute{
+				{
+					StartTime: 45 * time.Minute,
+					EndTime:   15 * time.Minute,
+				},
+			},
+		},
+	}
+	limits, _ := validation.NewOverrides(validation.Limits{
+		QueryPriority: validation.QueryPriority{
+			Enabled:    true,
+			Priorities: priorities,
+		},
+	}, nil)
+
+	assert.Equal(t, int64(0), GetPriority(url.Values{
+		"query": []string{"sum(up)"},
+		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
+	}, now, limits.QueryPriority("")))
+	assert.Equal(t, int64(1), GetPriority(url.Values{
+		"query": []string{"sum(up)"},
+		"time":  []string{strconv.FormatInt(now.Add(-30*time.Minute).Unix(), 10)},
+	}, now, limits.QueryPriority("")))
+	assert.Equal(t, int64(0), GetPriority(url.Values{
+		"query": []string{"sum(up)"},
+		"time":  []string{strconv.FormatInt(now.Add(-60*time.Minute).Unix(), 10)},
+	}, now, limits.QueryPriority("")))
+	assert.Equal(t, int64(1), GetPriority(url.Values{
+		"query": []string{"sum(up)"},
+		"time":  []string{strconv.FormatInt(now.Add(-45*time.Minute).Unix(), 10)},
+	}, now, limits.QueryPriority("")))
+	assert.Equal(t, int64(1), GetPriority(url.Values{
+		"query": []string{"sum(up)"},
+		"time":  []string{strconv.FormatInt(now.Add(-15*time.Minute).Unix(), 10)},
+	}, now, limits.QueryPriority("")))
+
+	assert.Equal(t, int64(1), GetPriority(url.Values{
+		"query": []string{"sum(up)"},
+		"start": []string{strconv.FormatInt(now.Add(-45*time.Minute).Unix(), 10)},
+		"end":   []string{strconv.FormatInt(now.Add(-15*time.Minute).Unix(), 10)},
+	}, now, limits.QueryPriority("")))
+	assert.Equal(t, int64(0), GetPriority(url.Values{
+		"query": []string{"sum(up)"},
+		"start": []string{strconv.FormatInt(now.Add(-50*time.Minute).Unix(), 10)},
+		"end":   []string{strconv.FormatInt(now.Add(-15*time.Minute).Unix(), 10)},
+	}, now, limits.QueryPriority("")))
+	assert.Equal(t, int64(0), GetPriority(url.Values{
+		"query": []string{"sum(up)"},
+		"start": []string{strconv.FormatInt(now.Add(-45*time.Minute).Unix(), 10)},
+		"end":   []string{strconv.FormatInt(now.Add(-10*time.Minute).Unix(), 10)},
+	}, now, limits.QueryPriority("")))
+	assert.Equal(t, int64(0), GetPriority(url.Values{
+		"query": []string{"sum(up)"},
+		"start": []string{strconv.FormatInt(now.Add(-60*time.Minute).Unix(), 10)},
+		"end":   []string{strconv.FormatInt(now.Add(-45*time.Minute).Unix(), 10)},
+	}, now, limits.QueryPriority("")))
+	assert.Equal(t, int64(0), GetPriority(url.Values{
+		"query": []string{"sum(up)"},
+		"start": []string{strconv.FormatInt(now.Add(-15*time.Minute).Unix(), 10)},
+		"end":   []string{strconv.FormatInt(now.Add(-1*time.Minute).Unix(), 10)},
+	}, now, limits.QueryPriority("")))
 }
diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go
index 65f848f8c3..10db7b9bc9 100644
--- a/pkg/util/validation/limits.go
+++ b/pkg/util/validation/limits.go
@@ -18,6 +18,7 @@ import (
 )
 
 var errMaxGlobalSeriesPerUserValidation = errors.New("The ingester.max-global-series-per-user limit is unsupported if distributor.shard-by-all-labels is disabled")
+var errDuplicateQueryPriorities = errors.New("There is a duplicate entry of priorities. Make sure they are all unique, including the default priority")
 
 // Supported values for enum limits
 const (
@@ -49,13 +50,13 @@ type DisabledRuleGroups []DisabledRuleGroup
 
 type QueryPriority struct {
 	Enabled         bool          `yaml:"enabled" doc:"nocli|description=Whether queries are assigned with priorities.|default=false"`
-	DefaultPriority int64         `yaml:"default_priority" doc:"nocli|description=Priority assigned to all queries by default. Must be a unique value. Use this as a baseline to make certain queries higher/lower priority.|default=1"`
+	DefaultPriority int64         `yaml:"default_priority" doc:"nocli|description=Priority assigned to all queries by default. Must be a unique value. Use this as a baseline to make certain queries higher/lower priority.|default=0"`
 	Priorities      []PriorityDef `yaml:"priorities" doc:"nocli|description=List of priority definitions."`
 	RegexCompiled   bool          `yaml:"-" doc:"nocli"`
 }
 
 type PriorityDef struct {
-	Priority         int64            `yaml:"priority" doc:"nocli|description=Priority level. Must be a unique value.|default=2"`
+	Priority         int64            `yaml:"priority" doc:"nocli|description=Priority level. Must be a unique value.|default=0"`
 	ReservedQueriers float64          `yaml:"reserved_queriers" doc:"nocli|description=Number of reserved queriers to handle this priority only. Value between 0 and 1 will be used as a percentage.|default=0"`
 	QueryAttributes  []QueryAttribute `yaml:"query_attributes" doc:"nocli|description=List of query attributes to assign the priority."`
 }
@@ -250,6 +251,19 @@ func (l *Limits) Validate(shardByAllLabels bool) error {
 		return errMaxGlobalSeriesPerUserValidation
 	}
 
+	if l.QueryPriority.Enabled {
+		queryPriority := l.QueryPriority
+		prioritySet := map[int64]struct{}{}
+		prioritySet[queryPriority.DefaultPriority] = struct{}{}
+		for _, priority := range queryPriority.Priorities {
+			if _, exists := prioritySet[priority.Priority]; exists {
+				return errDuplicateQueryPriorities
+			}
+
+			prioritySet[priority.Priority] = struct{}{}
+		}
+	}
+
 	return nil
 }
 
@@ -516,14 +530,18 @@ func (o *Overrides) MaxOutstandingPerTenant(userID string) int {
 
 // QueryPriority returns the query priority config for the tenant, including different priorities and their attributes
 func (o *Overrides) QueryPriority(userID string) QueryPriority {
-	if !o.GetOverridesForUser(userID).QueryPriority.RegexCompiled {
-		priorities := o.GetOverridesForUser(userID).QueryPriority.Priorities
+	queryPriority := o.GetOverridesForUser(userID).QueryPriority
+
+	if !queryPriority.RegexCompiled {
+		priorities := queryPriority.Priorities
 		for i, priority := range priorities {
 			for j, attributes := range priority.QueryAttributes {
 				if attributes.Regex == "" || attributes.Regex == ".*" || attributes.Regex == ".+" {
-					continue // don't use regex at all, if it is match all
+					// if it matches all, don't use regex
+					o.GetOverridesForUser(userID).QueryPriority.Priorities[i].QueryAttributes[j].CompiledRegex = nil
+				} else {
+					o.GetOverridesForUser(userID).QueryPriority.Priorities[i].QueryAttributes[j].CompiledRegex, _ = regexp.Compile(attributes.Regex)
 				}
-				o.GetOverridesForUser(userID).QueryPriority.Priorities[i].QueryAttributes[j].CompiledRegex, _ = regexp.Compile(attributes.Regex)
 			}
 		}
 		o.GetOverridesForUser(userID).QueryPriority.RegexCompiled = true
diff --git a/pkg/util/validation/limits_test.go b/pkg/util/validation/limits_test.go
index 2631354893..f2de1300e6 100644
--- a/pkg/util/validation/limits_test.go
+++ b/pkg/util/validation/limits_test.go
@@ -61,6 +61,47 @@ func TestLimits_Validate(t *testing.T) {
 			shardByAllLabels: true,
 			expected:         nil,
 		},
+		"duplicate priority entries": {
+			limits: Limits{QueryPriority: QueryPriority{
+				Enabled: true,
+				Priorities: []PriorityDef{
+					{
+						Priority: 1,
+					},
+					{
+						Priority: 1,
+					},
+				},
+			}},
+			expected: errDuplicateQueryPriorities,
+		},
+		"priority matches default priority": {
+			limits: Limits{QueryPriority: QueryPriority{
+				Enabled:         true,
+				DefaultPriority: 1,
+				Priorities: []PriorityDef{
+					{
+						Priority: 1,
+					},
+				},
+			}},
+			expected: errDuplicateQueryPriorities,
+		},
+		"all priorities are unique": {
+			limits: Limits{QueryPriority: QueryPriority{
+				Enabled:         true,
+				DefaultPriority: 1,
+				Priorities: []PriorityDef{
+					{
+						Priority: 2,
+					},
+					{
+						Priority: 3,
+					},
+				},
+			}},
+			expected: nil,
+		},
 	}
 
 	for testName, testData := range tests {
@@ -113,6 +154,65 @@ func TestOverridesManager_GetOverrides(t *testing.T) {
 	require.Equal(t, 0, ov.MaxLabelsSizeBytes("user2"))
 }
 
+func TestQueryPriority(t *testing.T) {
+	type testCase struct {
+		regex            string
+		compiledRegexNil bool
+	}
+	testCases := []testCase{
+		{
+			regex:            "",
+			compiledRegexNil: true,
+		},
+		{
+			regex:            ".*",
+			compiledRegexNil: true,
+		},
+		{
+			regex:            ".+",
+			compiledRegexNil: true,
+		},
+		{
+			regex:            "some_metric",
+			compiledRegexNil: false,
+		},
+	}
+
+	for _, tc := range testCases {
+		overrides, err := NewOverrides(Limits{
+			QueryPriority: QueryPriority{
+				Enabled:    true,
+				Priorities: getPriorities(tc.regex, 1*time.Hour, 0*time.Hour),
+			},
+		}, nil)
+		queryPriority := overrides.QueryPriority("")
+
+		assert.NoError(t, err)
+		assert.Equal(t, 1, len(queryPriority.Priorities[0].QueryAttributes))
+		if tc.compiledRegexNil {
+			assert.Nil(t, queryPriority.Priorities[0].QueryAttributes[0].CompiledRegex)
+		} else {
+			assert.NotNil(t, queryPriority.Priorities[0].QueryAttributes[0].CompiledRegex)
+		}
+		assert.True(t, queryPriority.RegexCompiled)
+	}
+}
+
+func getPriorities(regex string, startTime, endTime time.Duration) []PriorityDef {
+	return []PriorityDef{
+		{
+			Priority: 1,
+			QueryAttributes: []QueryAttribute{
+				{
+					Regex:     regex,
+					StartTime: startTime,
+					EndTime:   endTime,
+				},
+			},
+		},
+	}
+}
+
 func TestLimitsLoadingFromYaml(t *testing.T) {
 	SetDefaultLimitsForYAMLUnmarshalling(Limits{
 		MaxLabelNameLength: 100,

From 5eb2d730647d135c78e1d86d4648f2a2bb6fc675 Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Tue, 7 Nov 2023 12:19:26 -0800
Subject: [PATCH 22/49] Add priority queue to user queue

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 pkg/frontend/v1/frontend.go          |  2 +-
 pkg/scheduler/queue/queue.go         | 15 +++-----
 pkg/scheduler/queue/request_queue.go | 49 ++++++++++++++++++++++++
 pkg/scheduler/queue/user_queues.go   | 57 +++++++++++++++++-----------
 pkg/scheduler/scheduler.go           |  2 +-
 pkg/util/priority_queue.go           | 29 ++++----------
 6 files changed, 99 insertions(+), 55 deletions(-)
 create mode 100644 pkg/scheduler/queue/request_queue.go

diff --git a/pkg/frontend/v1/frontend.go b/pkg/frontend/v1/frontend.go
index 8d7bb4145d..d3734e9bcf 100644
--- a/pkg/frontend/v1/frontend.go
+++ b/pkg/frontend/v1/frontend.go
@@ -101,7 +101,7 @@ type request struct {
 	priority int64
 }
 
-func (r request) GetPriority() int64 {
+func (r request) Priority() int64 {
 	return r.priority
 }
 
diff --git a/pkg/scheduler/queue/queue.go b/pkg/scheduler/queue/queue.go
index cf20a3573a..a2e44cba83 100644
--- a/pkg/scheduler/queue/queue.go
+++ b/pkg/scheduler/queue/queue.go
@@ -45,7 +45,7 @@ func FirstUser() UserIndex {
 
 // Request stored into the queue.
 type Request interface {
-	GetPriority() int64
+	util.PriorityOp
 }
 
 // RequestQueue holds incoming requests in per-user queues. It also assigns each user specified number of queriers,
@@ -98,7 +98,7 @@ func (q *RequestQueue) EnqueueRequest(userID string, req Request, maxQueriers fl
 	}
 
 	shardSize := util.DynamicShardSize(maxQueriers, len(q.queues.queriers))
-	queue := q.queues.getOrAddQueue(userID, shardSize, req.GetPriority())
+	queue := q.queues.getOrAddQueue(userID, shardSize)
 	maxOutstandingRequests := q.queues.limits.MaxOutstandingPerTenant(userID)
 
 	if queue == nil {
@@ -108,12 +108,12 @@ func (q *RequestQueue) EnqueueRequest(userID string, req Request, maxQueriers fl
 
 	q.totalRequests.WithLabelValues(userID).Inc()
 
-	if q.queues.getTotalQueueSize(userID) >= maxOutstandingRequests {
+	if queue.length() >= maxOutstandingRequests {
 		q.discardedRequests.WithLabelValues(userID).Inc()
 		return ErrTooManyRequests
 	}
 
-	queue <- req
+	queue.enqueueRequest(req)
 	q.queueLength.WithLabelValues(userID).Inc()
 	q.cond.Broadcast()
 	// Call this function while holding a lock. This guarantees that no querier can fetch the request before function returns.
@@ -150,16 +150,13 @@ FindQueue:
 	for {
 		queue, userID, idx := q.queues.getNextQueueForQuerier(last.last, querierID)
 		last.last = idx
-		if queue == nil || len(queue) < 1 {
+		if queue == nil {
 			break
 		}
 
 		// Pick next request from the queue.
 		for {
-			request := <-queue
-			if q.queues.getTotalQueueSize(userID) == 0 {
-				q.queues.deleteQueue(userID)
-			}
+			request := queue.dequeueRequest()
 
 			q.queueLength.WithLabelValues(userID).Dec()
 
diff --git a/pkg/scheduler/queue/request_queue.go b/pkg/scheduler/queue/request_queue.go
new file mode 100644
index 0000000000..66c66e905e
--- /dev/null
+++ b/pkg/scheduler/queue/request_queue.go
@@ -0,0 +1,49 @@
+package queue
+
+import "github.com/cortexproject/cortex/pkg/util"
+
+type requestQueue interface {
+	enqueueRequest(Request)
+	dequeueRequest() Request
+	length() int
+}
+
+type FIFORequestQueue struct {
+	queue chan Request
+}
+
+func NewFIFOQueue(queue chan Request) *FIFORequestQueue {
+	return &FIFORequestQueue{queue: queue}
+}
+
+func (f *FIFORequestQueue) enqueueRequest(r Request) {
+	f.queue <- r
+}
+
+func (f *FIFORequestQueue) dequeueRequest() Request {
+	return <-f.queue
+}
+
+func (f *FIFORequestQueue) length() int {
+	return len(f.queue)
+}
+
+type PriorityRequestQueue struct {
+	queue *util.PriorityQueue
+}
+
+func NewPriorityRequestQueue(queue *util.PriorityQueue) *PriorityRequestQueue {
+	return &PriorityRequestQueue{queue: queue}
+}
+
+func (f *PriorityRequestQueue) enqueueRequest(r Request) {
+	f.queue.Enqueue(r)
+}
+
+func (f *PriorityRequestQueue) dequeueRequest() Request {
+	return f.queue.Dequeue()
+}
+
+func (f *PriorityRequestQueue) length() int {
+	return f.queue.Length()
+}
diff --git a/pkg/scheduler/queue/user_queues.go b/pkg/scheduler/queue/user_queues.go
index 86031c506e..e20bc0f905 100644
--- a/pkg/scheduler/queue/user_queues.go
+++ b/pkg/scheduler/queue/user_queues.go
@@ -59,12 +59,12 @@ type queues struct {
 }
 
 type userQueue struct {
-	normalQueue chan Request
+	queue requestQueue
 
 	// If not nil, only these queriers can handle user requests. If nil, all queriers can.
 	// We set this to nil if number of available queriers <= maxQueriers.
 	queriers         map[string]struct{}
-	reservedQueriers map[string]struct{}
+	reservedQueriers map[string]int64
 	maxQueriers      int
 
 	// Seed for shuffle sharding of queriers. This seed is based on userID only and is therefore consistent
@@ -76,7 +76,6 @@ type userQueue struct {
 }
 
 func newUserQueues(maxUserQueueSize int, forgetDelay time.Duration, limits Limits) *queues {
-	// Create schedule per user
 	return &queues{
 		userQueues:       map[string]*userQueue{},
 		users:            nil,
@@ -111,7 +110,7 @@ func (q *queues) deleteQueue(userID string) {
 // MaxQueriers is used to compute which queriers should handle requests for this user.
 // If maxQueriers is <= 0, all queriers can handle this user's requests.
 // If maxQueriers has changed since the last call, queriers for this are recomputed.
-func (q *queues) getOrAddQueue(userID string, maxQueriers int, priority int64) chan Request {
+func (q *queues) getOrAddQueue(userID string, maxQueriers int) requestQueue {
 	// Empty user is not allowed, as that would break our users list ("" is used for free spot).
 	if userID == "" {
 		return nil
@@ -122,6 +121,7 @@ func (q *queues) getOrAddQueue(userID string, maxQueriers int, priority int64) c
 	}
 
 	uq := q.userQueues[userID]
+	queryPriority := q.limits.QueryPriority(userID)
 
 	if uq == nil {
 		queueSize := q.limits.MaxOutstandingPerTenant(userID)
@@ -131,10 +131,16 @@ func (q *queues) getOrAddQueue(userID string, maxQueriers int, priority int64) c
 			queueSize = q.maxUserQueueSize
 		}
 		uq = &userQueue{
-			normalQueue: make(chan Request, queueSize),
-			seed:        util.ShuffleShardSeed(userID, ""),
-			index:       -1,
+			seed:  util.ShuffleShardSeed(userID, ""),
+			index: -1,
 		}
+
+		if queryPriority.Enabled {
+			uq.queue = NewPriorityRequestQueue(util.NewPriorityQueue(nil))
+		} else {
+			uq.queue = NewFIFOQueue(make(chan Request, queueSize))
+		}
+
 		q.userQueues[userID] = uq
 
 		// Add user to the list of users... find first free spot, and put it there.
@@ -156,24 +162,16 @@ func (q *queues) getOrAddQueue(userID string, maxQueriers int, priority int64) c
 	if uq.maxQueriers != maxQueriers {
 		uq.maxQueriers = maxQueriers
 		uq.queriers = shuffleQueriersForUser(uq.seed, maxQueriers, q.sortedQueriers, nil)
+		uq.reservedQueriers = getReservedQueriers(uq.queriers, q.limits.QueryPriority(userID).Priorities)
 	}
 
-	//if highPriority {
-	//	return uq.highPriorityQueue
-	//}
-
-	return uq.normalQueue
-}
-
-func (q *queues) getTotalQueueSize(userID string) int {
-	// TODO: Implement
-	return len(q.userQueues[userID].normalQueue)
+	return uq.queue
 }
 
 // Finds next queue for the querier. To support fair scheduling between users, client is expected
 // to pass last user index returned by this function as argument. Is there was no previous
 // last user index, use -1.
-func (q *queues) getNextQueueForQuerier(lastUserIndex int, querierID string) (chan Request, string, int) {
+func (q *queues) getNextQueueForQuerier(lastUserIndex int, querierID string) (requestQueue, string, int) {
 	uid := lastUserIndex
 
 	for iters := 0; iters < len(q.users); iters++ {
@@ -199,13 +197,12 @@ func (q *queues) getNextQueueForQuerier(lastUserIndex int, querierID string) (ch
 			}
 		}
 
-		// TODO: Implement
-		//_, isReserved := uq.reservedQueriers[querierID]
-		//if isReserved || len(uq.highPriorityQueue) > 0 {
-		//	return uq.highPriorityQueue, u, uid
+		//TODO: jungjust, reserved queriers
+		//if priority, isReserved := uq.reservedQueriers[querierID]; isReserved {
+		//	return uq.queues[priority], u, uid
 		//}
 
-		return uq.normalQueue, u, uid
+		return uq.queue, u, uid
 	}
 	return nil, "", uid
 }
@@ -353,6 +350,20 @@ func shuffleQueriersForUser(userSeed int64, queriersToSelect int, allSortedQueri
 	return queriers
 }
 
+func getReservedQueriers(queriers map[string]struct{}, priorities []validation.PriorityDef) map[string]int64 {
+	// TODO jungjust:
+	//reservedQueriers := make(map[string]int64)
+	//
+	//cnt := 0
+	//for querierID, _ := range queriers {
+	//
+	//	//reservedQueriers
+	//	// TODO jungjust: get diff priorities
+	//	cnt++
+	//}
+	return map[string]int64{}
+}
+
 //func getNumOfReservedQueriers(queriersToSelect int, totalNumOfQueriers int, reservedQueriers float64) int {
 //	numOfReservedQueriers := int(reservedQueriers)
 //
diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go
index d4f7ac8541..4c3a0afcf8 100644
--- a/pkg/scheduler/scheduler.go
+++ b/pkg/scheduler/scheduler.go
@@ -166,7 +166,7 @@ type schedulerRequest struct {
 	parentSpanContext opentracing.SpanContext
 }
 
-func (s schedulerRequest) GetPriority() int64 {
+func (s schedulerRequest) Priority() int64 {
 	return s.priority
 }
 
diff --git a/pkg/util/priority_queue.go b/pkg/util/priority_queue.go
index 1c2fbadd3b..4cbc86e3ff 100644
--- a/pkg/util/priority_queue.go
+++ b/pkg/util/priority_queue.go
@@ -13,18 +13,16 @@ type PriorityQueue struct {
 	cond        *sync.Cond
 	closing     bool
 	closed      bool
-	hit         map[string]struct{}
 	queue       queue
 	lengthGauge prometheus.Gauge
 }
 
-// Op is an operation on the priority queue.
-type Op interface {
-	Key() string
+// PriorityOp is an operation on the priority queue.
+type PriorityOp interface {
 	Priority() int64 // The larger the number the higher the priority.
 }
 
-type queue []Op
+type queue []PriorityOp
 
 func (q queue) Len() int           { return len(q) }
 func (q queue) Less(i, j int) bool { return q[i].Priority() > q[j].Priority() }
@@ -33,7 +31,7 @@ func (q queue) Swap(i, j int)      { q[i], q[j] = q[j], q[i] }
 // Push and Pop use pointer receivers because they modify the slice's length,
 // not just its contents.
 func (q *queue) Push(x interface{}) {
-	*q = append(*q, x.(Op))
+	*q = append(*q, x.(PriorityOp))
 }
 
 func (q *queue) Pop() interface{} {
@@ -47,7 +45,6 @@ func (q *queue) Pop() interface{} {
 // NewPriorityQueue makes a new priority queue.
 func NewPriorityQueue(lengthGauge prometheus.Gauge) *PriorityQueue {
 	pq := &PriorityQueue{
-		hit:         map[string]struct{}{},
 		lengthGauge: lengthGauge,
 	}
 	pq.cond = sync.NewCond(&pq.lock)
@@ -77,13 +74,11 @@ func (pq *PriorityQueue) DiscardAndClose() {
 	defer pq.lock.Unlock()
 	pq.closed = true
 	pq.queue = nil
-	pq.hit = map[string]struct{}{}
 	pq.cond.Broadcast()
 }
 
-// Enqueue adds an operation to the queue in priority order. Returns
-// true if added; false if the operation was already on the queue.
-func (pq *PriorityQueue) Enqueue(op Op) bool {
+// Enqueue adds an operation to the queue in priority order.
+func (pq *PriorityQueue) Enqueue(op PriorityOp) {
 	pq.lock.Lock()
 	defer pq.lock.Unlock()
 
@@ -91,23 +86,16 @@ func (pq *PriorityQueue) Enqueue(op Op) bool {
 		panic("enqueue on closed queue")
 	}
 
-	_, enqueued := pq.hit[op.Key()]
-	if enqueued {
-		return false
-	}
-
-	pq.hit[op.Key()] = struct{}{}
 	heap.Push(&pq.queue, op)
 	pq.cond.Broadcast()
 	if pq.lengthGauge != nil {
 		pq.lengthGauge.Inc()
 	}
-	return true
 }
 
 // Dequeue will return the op with the highest priority; block if queue is
 // empty; returns nil if queue is closed.
-func (pq *PriorityQueue) Dequeue() Op {
+func (pq *PriorityQueue) Dequeue() PriorityOp {
 	pq.lock.Lock()
 	defer pq.lock.Unlock()
 
@@ -120,8 +108,7 @@ func (pq *PriorityQueue) Dequeue() Op {
 		return nil
 	}
 
-	op := heap.Pop(&pq.queue).(Op)
-	delete(pq.hit, op.Key())
+	op := heap.Pop(&pq.queue).(PriorityOp)
 	if pq.lengthGauge != nil {
 		pq.lengthGauge.Dec()
 	}

From f064b3a7457005e516aacf6829f99fb7a518ff8d Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Tue, 7 Nov 2023 16:40:51 -0800
Subject: [PATCH 23/49] Nits + priority queue test added

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 pkg/scheduler/queue/queue.go            |  3 ++
 pkg/scheduler/queue/queue_test.go       | 62 ++++++++++++-------------
 pkg/scheduler/queue/request_queue.go    |  9 ++++
 pkg/scheduler/queue/user_queues.go      | 15 ++----
 pkg/scheduler/queue/user_queues_test.go | 26 +++++------
 5 files changed, 59 insertions(+), 56 deletions(-)

diff --git a/pkg/scheduler/queue/queue.go b/pkg/scheduler/queue/queue.go
index a2e44cba83..98031843ab 100644
--- a/pkg/scheduler/queue/queue.go
+++ b/pkg/scheduler/queue/queue.go
@@ -157,6 +157,9 @@ FindQueue:
 		// Pick next request from the queue.
 		for {
 			request := queue.dequeueRequest()
+			if queue.length() == 0 {
+				q.queues.deleteQueue(userID)
+			}
 
 			q.queueLength.WithLabelValues(userID).Dec()
 
diff --git a/pkg/scheduler/queue/queue_test.go b/pkg/scheduler/queue/queue_test.go
index 046abc7c03..e77dc33e6e 100644
--- a/pkg/scheduler/queue/queue_test.go
+++ b/pkg/scheduler/queue/queue_test.go
@@ -3,6 +3,7 @@ package queue
 import (
 	"context"
 	"fmt"
+	"github.com/cortexproject/cortex/pkg/util/validation"
 	"strconv"
 	"sync"
 	"testing"
@@ -160,40 +161,37 @@ func TestRequestQueue_GetNextRequestForQuerier_ShouldGetRequestAfterReshardingBe
 }
 
 func TestRequestQueue_QueriersShouldGetHighPriorityQueryFirst(t *testing.T) {
-	//queue := NewRequestQueue(0, 0,
-	//	prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user"}),
-	//	prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user"}),
-	//	MockLimits{MaxOutstanding: 3},
-	//	nil,
-	//)
-	//ctx := context.Background()
-	//queue.RegisterQuerierConnection("querier-1")
-	//
-	//normalRequest1 := MockRequest{
-	//	id:             "normal query 1",
-	//}
-	//normalRequest2 := MockRequest{
-	//	id:             "normal query 2",
-	//}
-	//highPriorityRequest := MockRequest{
-	//	id:             "high priority query",
-	//}
-	//
-	//assert.NoError(t, queue.EnqueueRequest("userID", normalRequest1, 1, func() {}))
-	//assert.NoError(t, queue.EnqueueRequest("userID", normalRequest2, 1, func() {}))
-	//assert.NoError(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {}))
-	//
-	//assert.Error(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {})) // should fail due to maxOutstandingPerTenant = 3
-	//assert.Equal(t, 3, queue.queues.getTotalQueueSize("userID"))
-	//nextRequest, _, _ := queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1")
-	//assert.Equal(t, highPriorityRequest, nextRequest) // high priority request returned, although it was enqueued the last
-	//nextRequest, _, _ = queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1")
-	//assert.Equal(t, normalRequest1, nextRequest)
-	//nextRequest, _, _ = queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1")
-	//assert.Equal(t, normalRequest2, nextRequest)
+	queue := NewRequestQueue(0, 0,
+		prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user"}),
+		prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user"}),
+		MockLimits{MaxOutstanding: 3, queryPriority: validation.QueryPriority{Enabled: true}},
+		nil,
+	)
+	ctx := context.Background()
+	queue.RegisterQuerierConnection("querier-1")
+
+	normalRequest1 := MockRequest{
+		id: "normal query 1",
+	}
+	normalRequest2 := MockRequest{
+		id: "normal query 2",
+	}
+	highPriorityRequest := MockRequest{
+		id:       "high priority query",
+		priority: 1,
+	}
+
+	assert.NoError(t, queue.EnqueueRequest("userID", normalRequest1, 1, func() {}))
+	assert.NoError(t, queue.EnqueueRequest("userID", normalRequest2, 1, func() {}))
+	assert.NoError(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {}))
+
+	assert.Error(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {})) // should fail due to maxOutstandingPerTenant = 3
+	nextRequest, _, _ := queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1")
+	assert.Equal(t, highPriorityRequest, nextRequest) // high priority request returned, although it was enqueued the last
 }
 
 func TestRequestQueue_ReservedQueriersShouldOnlyGetHighPriorityQueries(t *testing.T) {
+	// TODO: justinjung04
 	//queue := NewRequestQueue(0, 0,
 	//	prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user"}),
 	//	prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user"}),
@@ -235,6 +233,6 @@ type MockRequest struct {
 	priority int64
 }
 
-func (r MockRequest) GetPriority() int64 {
+func (r MockRequest) Priority() int64 {
 	return r.priority
 }
diff --git a/pkg/scheduler/queue/request_queue.go b/pkg/scheduler/queue/request_queue.go
index 66c66e905e..d6ff100fa6 100644
--- a/pkg/scheduler/queue/request_queue.go
+++ b/pkg/scheduler/queue/request_queue.go
@@ -6,6 +6,7 @@ type requestQueue interface {
 	enqueueRequest(Request)
 	dequeueRequest() Request
 	length() int
+	closeQueue()
 }
 
 type FIFORequestQueue struct {
@@ -28,6 +29,10 @@ func (f *FIFORequestQueue) length() int {
 	return len(f.queue)
 }
 
+func (f *FIFORequestQueue) closeQueue() {
+	close(f.queue)
+}
+
 type PriorityRequestQueue struct {
 	queue *util.PriorityQueue
 }
@@ -47,3 +52,7 @@ func (f *PriorityRequestQueue) dequeueRequest() Request {
 func (f *PriorityRequestQueue) length() int {
 	return f.queue.Length()
 }
+
+func (f *PriorityRequestQueue) closeQueue() {
+	f.queue.DiscardAndClose()
+}
diff --git a/pkg/scheduler/queue/user_queues.go b/pkg/scheduler/queue/user_queues.go
index e20bc0f905..3eb4cf0581 100644
--- a/pkg/scheduler/queue/user_queues.go
+++ b/pkg/scheduler/queue/user_queues.go
@@ -97,6 +97,7 @@ func (q *queues) deleteQueue(userID string) {
 		return
 	}
 
+	uq.queue.closeQueue()
 	delete(q.userQueues, userID)
 	q.users[uq.index] = ""
 
@@ -197,7 +198,7 @@ func (q *queues) getNextQueueForQuerier(lastUserIndex int, querierID string) (re
 			}
 		}
 
-		//TODO: jungjust, reserved queriers
+		//TODO: justinjung04, reserved queriers
 		//if priority, isReserved := uq.reservedQueriers[querierID]; isReserved {
 		//	return uq.queues[priority], u, uid
 		//}
@@ -351,19 +352,11 @@ func shuffleQueriersForUser(userSeed int64, queriersToSelect int, allSortedQueri
 }
 
 func getReservedQueriers(queriers map[string]struct{}, priorities []validation.PriorityDef) map[string]int64 {
-	// TODO jungjust:
-	//reservedQueriers := make(map[string]int64)
-	//
-	//cnt := 0
-	//for querierID, _ := range queriers {
-	//
-	//	//reservedQueriers
-	//	// TODO jungjust: get diff priorities
-	//	cnt++
-	//}
+	// TODO: justinjung04
 	return map[string]int64{}
 }
 
+// TODO: justinjung04
 //func getNumOfReservedQueriers(queriersToSelect int, totalNumOfQueriers int, reservedQueriers float64) int {
 //	numOfReservedQueriers := int(reservedQueriers)
 //
diff --git a/pkg/scheduler/queue/user_queues_test.go b/pkg/scheduler/queue/user_queues_test.go
index 06fb90a309..48a8640d50 100644
--- a/pkg/scheduler/queue/user_queues_test.go
+++ b/pkg/scheduler/queue/user_queues_test.go
@@ -22,11 +22,11 @@ func TestQueues(t *testing.T) {
 	assert.Equal(t, "", u)
 
 	// Add queues: [one]
-	qOne := getOrAdd(t, uq, "one", 0, 0)
+	qOne := getOrAdd(t, uq, "one", 0)
 	lastUserIndex = confirmOrderForQuerier(t, uq, "querier-1", lastUserIndex, qOne, qOne)
 
 	// [one two]
-	qTwo := getOrAdd(t, uq, "two", 0, 0)
+	qTwo := getOrAdd(t, uq, "two", 0)
 	assert.NotEqual(t, qOne, qTwo)
 
 	lastUserIndex = confirmOrderForQuerier(t, uq, "querier-1", lastUserIndex, qTwo, qOne, qTwo, qOne)
@@ -34,7 +34,7 @@ func TestQueues(t *testing.T) {
 
 	// [one two three]
 	// confirm fifo by adding a third queue and iterating to it
-	qThree := getOrAdd(t, uq, "three", 0, 0)
+	qThree := getOrAdd(t, uq, "three", 0)
 
 	lastUserIndex = confirmOrderForQuerier(t, uq, "querier-1", lastUserIndex, qTwo, qThree, qOne)
 
@@ -45,7 +45,7 @@ func TestQueues(t *testing.T) {
 	lastUserIndex = confirmOrderForQuerier(t, uq, "querier-1", lastUserIndex, qTwo, qThree, qTwo)
 
 	// "four" is added at the beginning of the list: [four two three]
-	qFour := getOrAdd(t, uq, "four", 0, 0)
+	qFour := getOrAdd(t, uq, "four", 0)
 
 	lastUserIndex = confirmOrderForQuerier(t, uq, "querier-1", lastUserIndex, qThree, qFour, qTwo, qThree)
 
@@ -92,7 +92,7 @@ func TestQueuesWithQueriers(t *testing.T) {
 	// Add user queues.
 	for u := 0; u < users; u++ {
 		uid := fmt.Sprintf("user-%d", u)
-		getOrAdd(t, uq, uid, maxQueriersPerUser, 0)
+		getOrAdd(t, uq, uid, maxQueriersPerUser)
 
 		// Verify it has maxQueriersPerUser queriers assigned now.
 		qs := uq.userQueues[uid].queriers
@@ -156,7 +156,7 @@ func TestQueuesConsistency(t *testing.T) {
 			conns := map[string]int{}
 
 			for i := 0; i < 10000; i++ {
-				queue := uq.getOrAddQueue(generateTenant(r), 3, 0)
+				queue := uq.getOrAddQueue(generateTenant(r), 3)
 				switch r.Int() % 6 {
 				case 0:
 					assert.NotNil(t, queue)
@@ -208,7 +208,7 @@ func TestQueues_ForgetDelay(t *testing.T) {
 	// Add user queues.
 	for i := 0; i < numUsers; i++ {
 		userID := fmt.Sprintf("user-%d", i)
-		getOrAdd(t, uq, userID, maxQueriersPerUser, 0)
+		getOrAdd(t, uq, userID, maxQueriersPerUser)
 	}
 
 	// We expect querier-1 to have some users.
@@ -300,7 +300,7 @@ func TestQueues_ForgetDelay_ShouldCorrectlyHandleQuerierReconnectingBeforeForget
 	// Add user queues.
 	for i := 0; i < numUsers; i++ {
 		userID := fmt.Sprintf("user-%d", i)
-		getOrAdd(t, uq, userID, maxQueriersPerUser, 0)
+		getOrAdd(t, uq, userID, maxQueriersPerUser)
 	}
 
 	// We expect querier-1 to have some users.
@@ -360,16 +360,16 @@ func generateQuerier(r *rand.Rand) string {
 	return fmt.Sprint("querier-", r.Int()%5)
 }
 
-func getOrAdd(t *testing.T, uq *queues, tenant string, maxQueriers int, priority int64) chan Request {
-	q := uq.getOrAddQueue(tenant, maxQueriers, priority)
+func getOrAdd(t *testing.T, uq *queues, tenant string, maxQueriers int) requestQueue {
+	q := uq.getOrAddQueue(tenant, maxQueriers)
 	assert.NotNil(t, q)
 	assert.NoError(t, isConsistent(uq))
-	assert.Equal(t, q, uq.getOrAddQueue(tenant, maxQueriers, priority))
+	assert.Equal(t, q, uq.getOrAddQueue(tenant, maxQueriers))
 	return q
 }
 
-func confirmOrderForQuerier(t *testing.T, uq *queues, querier string, lastUserIndex int, qs ...chan Request) int {
-	var n chan Request
+func confirmOrderForQuerier(t *testing.T, uq *queues, querier string, lastUserIndex int, qs ...requestQueue) int {
+	var n requestQueue
 	for _, q := range qs {
 		n, _, lastUserIndex = uq.getNextQueueForQuerier(lastUserIndex, querier)
 		assert.Equal(t, q, n)

From e41e4c1ab6d2b110893ec81e7271567833f51743 Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Wed, 8 Nov 2023 08:42:23 -0800
Subject: [PATCH 24/49] Create user request queue test

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 pkg/scheduler/queue/queue.go                  |  7 ++-
 pkg/scheduler/queue/user_queues.go            | 61 +++++--------------
 pkg/scheduler/queue/user_queues_test.go       |  6 +-
 ...request_queue.go => user_request_queue.go} | 13 ++--
 .../queue/user_request_queue_test.go          | 57 +++++++++++++++++
 pkg/util/priority_queue.go                    |  8 ++-
 pkg/util/priority_queue_test.go               |  7 +--
 7 files changed, 97 insertions(+), 62 deletions(-)
 rename pkg/scheduler/queue/{request_queue.go => user_request_queue.go} (72%)
 create mode 100644 pkg/scheduler/queue/user_request_queue_test.go

diff --git a/pkg/scheduler/queue/queue.go b/pkg/scheduler/queue/queue.go
index 98031843ab..e6c55ca53d 100644
--- a/pkg/scheduler/queue/queue.go
+++ b/pkg/scheduler/queue/queue.go
@@ -156,7 +156,12 @@ FindQueue:
 
 		// Pick next request from the queue.
 		for {
-			request := queue.dequeueRequest()
+			request := queue.dequeueRequest(q.queues.getMinPriority(userID, querierID))
+			if request == nil {
+				// the queue does not contain request with the min priority, break to wait for more requests
+				break
+			}
+
 			if queue.length() == 0 {
 				q.queues.deleteQueue(userID)
 			}
diff --git a/pkg/scheduler/queue/user_queues.go b/pkg/scheduler/queue/user_queues.go
index 3eb4cf0581..1c34f41762 100644
--- a/pkg/scheduler/queue/user_queues.go
+++ b/pkg/scheduler/queue/user_queues.go
@@ -59,13 +59,12 @@ type queues struct {
 }
 
 type userQueue struct {
-	queue requestQueue
+	queue userRequestQueue
 
 	// If not nil, only these queriers can handle user requests. If nil, all queriers can.
 	// We set this to nil if number of available queriers <= maxQueriers.
-	queriers         map[string]struct{}
-	reservedQueriers map[string]int64
-	maxQueriers      int
+	queriers    map[string]struct{}
+	maxQueriers int
 
 	// Seed for shuffle sharding of queriers. This seed is based on userID only and is therefore consistent
 	// between different frontends.
@@ -111,7 +110,7 @@ func (q *queues) deleteQueue(userID string) {
 // MaxQueriers is used to compute which queriers should handle requests for this user.
 // If maxQueriers is <= 0, all queriers can handle this user's requests.
 // If maxQueriers has changed since the last call, queriers for this are recomputed.
-func (q *queues) getOrAddQueue(userID string, maxQueriers int) requestQueue {
+func (q *queues) getOrAddQueue(userID string, maxQueriers int) userRequestQueue {
 	// Empty user is not allowed, as that would break our users list ("" is used for free spot).
 	if userID == "" {
 		return nil
@@ -139,7 +138,7 @@ func (q *queues) getOrAddQueue(userID string, maxQueriers int) requestQueue {
 		if queryPriority.Enabled {
 			uq.queue = NewPriorityRequestQueue(util.NewPriorityQueue(nil))
 		} else {
-			uq.queue = NewFIFOQueue(make(chan Request, queueSize))
+			uq.queue = NewFIFORequestQueue(make(chan Request, queueSize))
 		}
 
 		q.userQueues[userID] = uq
@@ -163,7 +162,6 @@ func (q *queues) getOrAddQueue(userID string, maxQueriers int) requestQueue {
 	if uq.maxQueriers != maxQueriers {
 		uq.maxQueriers = maxQueriers
 		uq.queriers = shuffleQueriersForUser(uq.seed, maxQueriers, q.sortedQueriers, nil)
-		uq.reservedQueriers = getReservedQueriers(uq.queriers, q.limits.QueryPriority(userID).Priorities)
 	}
 
 	return uq.queue
@@ -172,7 +170,7 @@ func (q *queues) getOrAddQueue(userID string, maxQueriers int) requestQueue {
 // Finds next queue for the querier. To support fair scheduling between users, client is expected
 // to pass last user index returned by this function as argument. Is there was no previous
 // last user index, use -1.
-func (q *queues) getNextQueueForQuerier(lastUserIndex int, querierID string) (requestQueue, string, int) {
+func (q *queues) getNextQueueForQuerier(lastUserIndex int, querierID string) (userRequestQueue, string, int) {
 	uid := lastUserIndex
 
 	for iters := 0; iters < len(q.users); iters++ {
@@ -198,7 +196,7 @@ func (q *queues) getNextQueueForQuerier(lastUserIndex int, querierID string) (re
 			}
 		}
 
-		//TODO: justinjung04, reserved queriers
+		// TODO: justinjung04, reserved queriers
 		//if priority, isReserved := uq.reservedQueriers[querierID]; isReserved {
 		//	return uq.queues[priority], u, uid
 		//}
@@ -208,6 +206,13 @@ func (q *queues) getNextQueueForQuerier(lastUserIndex int, querierID string) (re
 	return nil, "", uid
 }
 
+func (q *queues) getMinPriority(userID string, querierID string) int64 {
+	// TODO: justinjung04 reserved querier
+	// check list of queriers and QueryPriority config
+	// from QueryPriority config, establish map of
+	return 0
+}
+
 func (q *queues) addQuerierConnection(querierID string) {
 	info := q.queriers[querierID]
 	if info != nil {
@@ -320,16 +325,6 @@ func (q *queues) recomputeUserQueriers() {
 // In that case *all* queriers should be used.
 // Scratchpad is used for shuffling, to avoid new allocations. If nil, new slice is allocated.
 func shuffleQueriersForUser(userSeed int64, queriersToSelect int, allSortedQueriers []string, scratchpad []string) map[string]struct{} {
-	//numOfReservedQueriers := getNumOfReservedQueriers(queriersToSelect, len(allSortedQueriers), numOfReservedQueriersFloat)
-	//reservedQueriers := make(map[string]struct{}, numOfReservedQueriers)
-
-	//if queriersToSelect == 0 || len(allSortedQueriers) <= queriersToSelect {
-	//	for i := 0; i < numOfReservedQueriers; i++ {
-	//		reservedQueriers[allSortedQueriers[i]] = struct{}{}
-	//	}
-	//	return nil, reservedQueriers
-	//}
-
 	queriers := make(map[string]struct{}, queriersToSelect)
 	rnd := rand.New(rand.NewSource(userSeed))
 
@@ -340,10 +335,6 @@ func shuffleQueriersForUser(userSeed int64, queriersToSelect int, allSortedQueri
 	for i := 0; i < queriersToSelect; i++ {
 		r := rnd.Intn(last + 1)
 		queriers[scratchpad[r]] = struct{}{}
-		//if i < numOfReservedQueriers {
-		//	reservedQueriers[scratchpad[r]] = struct{}{}
-		//}
-		// move selected item to the end, it won't be selected anymore.
 		scratchpad[r], scratchpad[last] = scratchpad[last], scratchpad[r]
 		last--
 	}
@@ -351,30 +342,6 @@ func shuffleQueriersForUser(userSeed int64, queriersToSelect int, allSortedQueri
 	return queriers
 }
 
-func getReservedQueriers(queriers map[string]struct{}, priorities []validation.PriorityDef) map[string]int64 {
-	// TODO: justinjung04
-	return map[string]int64{}
-}
-
-// TODO: justinjung04
-//func getNumOfReservedQueriers(queriersToSelect int, totalNumOfQueriers int, reservedQueriers float64) int {
-//	numOfReservedQueriers := int(reservedQueriers)
-//
-//	if reservedQueriers < 1 && reservedQueriers > 0 {
-//		if queriersToSelect == 0 || queriersToSelect > totalNumOfQueriers {
-//			queriersToSelect = totalNumOfQueriers
-//		}
-//
-//		numOfReservedQueriers = int(math.Ceil(float64(queriersToSelect) * reservedQueriers))
-//	}
-//
-//	if numOfReservedQueriers > totalNumOfQueriers {
-//		return totalNumOfQueriers
-//	}
-//
-//	return numOfReservedQueriers
-//}
-
 // MockLimits implements the Limits interface. Used in tests only.
 type MockLimits struct {
 	MaxOutstanding     int
diff --git a/pkg/scheduler/queue/user_queues_test.go b/pkg/scheduler/queue/user_queues_test.go
index 48a8640d50..729ae9699d 100644
--- a/pkg/scheduler/queue/user_queues_test.go
+++ b/pkg/scheduler/queue/user_queues_test.go
@@ -360,7 +360,7 @@ func generateQuerier(r *rand.Rand) string {
 	return fmt.Sprint("querier-", r.Int()%5)
 }
 
-func getOrAdd(t *testing.T, uq *queues, tenant string, maxQueriers int) requestQueue {
+func getOrAdd(t *testing.T, uq *queues, tenant string, maxQueriers int) userRequestQueue {
 	q := uq.getOrAddQueue(tenant, maxQueriers)
 	assert.NotNil(t, q)
 	assert.NoError(t, isConsistent(uq))
@@ -368,8 +368,8 @@ func getOrAdd(t *testing.T, uq *queues, tenant string, maxQueriers int) requestQ
 	return q
 }
 
-func confirmOrderForQuerier(t *testing.T, uq *queues, querier string, lastUserIndex int, qs ...requestQueue) int {
-	var n requestQueue
+func confirmOrderForQuerier(t *testing.T, uq *queues, querier string, lastUserIndex int, qs ...userRequestQueue) int {
+	var n userRequestQueue
 	for _, q := range qs {
 		n, _, lastUserIndex = uq.getNextQueueForQuerier(lastUserIndex, querier)
 		assert.Equal(t, q, n)
diff --git a/pkg/scheduler/queue/request_queue.go b/pkg/scheduler/queue/user_request_queue.go
similarity index 72%
rename from pkg/scheduler/queue/request_queue.go
rename to pkg/scheduler/queue/user_request_queue.go
index d6ff100fa6..b8ee1ecaf7 100644
--- a/pkg/scheduler/queue/request_queue.go
+++ b/pkg/scheduler/queue/user_request_queue.go
@@ -2,9 +2,9 @@ package queue
 
 import "github.com/cortexproject/cortex/pkg/util"
 
-type requestQueue interface {
+type userRequestQueue interface {
 	enqueueRequest(Request)
-	dequeueRequest() Request
+	dequeueRequest(minPriority int64) Request
 	length() int
 	closeQueue()
 }
@@ -13,7 +13,7 @@ type FIFORequestQueue struct {
 	queue chan Request
 }
 
-func NewFIFOQueue(queue chan Request) *FIFORequestQueue {
+func NewFIFORequestQueue(queue chan Request) *FIFORequestQueue {
 	return &FIFORequestQueue{queue: queue}
 }
 
@@ -21,7 +21,7 @@ func (f *FIFORequestQueue) enqueueRequest(r Request) {
 	f.queue <- r
 }
 
-func (f *FIFORequestQueue) dequeueRequest() Request {
+func (f *FIFORequestQueue) dequeueRequest(_ int64) Request {
 	return <-f.queue
 }
 
@@ -45,7 +45,10 @@ func (f *PriorityRequestQueue) enqueueRequest(r Request) {
 	f.queue.Enqueue(r)
 }
 
-func (f *PriorityRequestQueue) dequeueRequest() Request {
+func (f *PriorityRequestQueue) dequeueRequest(minPriority int64) Request {
+	if f.queue.Peek().Priority() < minPriority {
+		return nil
+	}
 	return f.queue.Dequeue()
 }
 
diff --git a/pkg/scheduler/queue/user_request_queue_test.go b/pkg/scheduler/queue/user_request_queue_test.go
new file mode 100644
index 0000000000..aeaad7fb2a
--- /dev/null
+++ b/pkg/scheduler/queue/user_request_queue_test.go
@@ -0,0 +1,57 @@
+package queue
+
+import (
+	"github.com/cortexproject/cortex/pkg/util"
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestFIFORequestQueue(t *testing.T) {
+	queue := NewFIFORequestQueue(make(chan Request, 2))
+	request1 := MockRequest{
+		id:       "request 1",
+		priority: 1,
+	}
+	request2 := MockRequest{
+		id:       "request 2",
+		priority: 2,
+	}
+
+	queue.enqueueRequest(request1)
+	queue.enqueueRequest(request2)
+	assert.Equal(t, 2, queue.length())
+	assert.Equal(t, request1, queue.dequeueRequest(0))
+	assert.Equal(t, 1, queue.length())
+	assert.Equal(t, request2, queue.dequeueRequest(0))
+	assert.Equal(t, 0, queue.length())
+	queue.closeQueue()
+	assert.Panics(t, func() { queue.enqueueRequest(request1) })
+}
+
+func TestPriorityRequestQueue(t *testing.T) {
+	queue := NewPriorityRequestQueue(util.NewPriorityQueue(nil))
+	request1 := MockRequest{
+		id:       "request 1",
+		priority: 1,
+	}
+	request2 := MockRequest{
+		id:       "request 2",
+		priority: 2,
+	}
+
+	queue.enqueueRequest(request1)
+	queue.enqueueRequest(request2)
+	assert.Equal(t, 2, queue.length())
+	assert.Equal(t, request2, queue.dequeueRequest(0))
+	assert.Equal(t, 1, queue.length())
+	assert.Equal(t, request1, queue.dequeueRequest(0))
+	assert.Equal(t, 0, queue.length())
+
+	queue.enqueueRequest(request1)
+	queue.enqueueRequest(request2)
+	assert.Equal(t, 2, queue.length())
+	assert.Equal(t, request2, queue.dequeueRequest(2))
+	
+	queue.closeQueue()
+	assert.Panics(t, func() { queue.enqueueRequest(request1) })
+}
diff --git a/pkg/util/priority_queue.go b/pkg/util/priority_queue.go
index 4cbc86e3ff..26e0b13832 100644
--- a/pkg/util/priority_queue.go
+++ b/pkg/util/priority_queue.go
@@ -93,7 +93,7 @@ func (pq *PriorityQueue) Enqueue(op PriorityOp) {
 	}
 }
 
-// Dequeue will return the op with the highest priority; block if queue is
+// Dequeue will remove and return the op with the highest priority; block if queue is
 // empty; returns nil if queue is closed.
 func (pq *PriorityQueue) Dequeue() PriorityOp {
 	pq.lock.Lock()
@@ -114,3 +114,9 @@ func (pq *PriorityQueue) Dequeue() PriorityOp {
 	}
 	return op
 }
+
+// Peek will return the op with the highest priority without removing it from the queue
+func (pq *PriorityQueue) Peek() PriorityOp {
+	op := pq.queue[0]
+	return op
+}
diff --git a/pkg/util/priority_queue_test.go b/pkg/util/priority_queue_test.go
index 79eaf2784f..d7ee8bc202 100644
--- a/pkg/util/priority_queue_test.go
+++ b/pkg/util/priority_queue_test.go
@@ -2,7 +2,6 @@ package util
 
 import (
 	"runtime"
-	"strconv"
 	"testing"
 	"time"
 
@@ -15,10 +14,6 @@ func (i simpleItem) Priority() int64 {
 	return int64(i)
 }
 
-func (i simpleItem) Key() string {
-	return strconv.FormatInt(int64(i), 10)
-}
-
 func TestPriorityQueueBasic(t *testing.T) {
 	queue := NewPriorityQueue(nil)
 	assert.Equal(t, 0, queue.Length(), "Expected length = 0")
@@ -39,6 +34,7 @@ func TestPriorityQueuePriorities(t *testing.T) {
 	queue.Enqueue(simpleItem(1))
 	queue.Enqueue(simpleItem(2))
 
+	assert.Equal(t, simpleItem(2), queue.Peek().(simpleItem), "Expected to peek simpleItem(2)")
 	assert.Equal(t, simpleItem(2), queue.Dequeue().(simpleItem), "Expected to dequeue simpleItem(2)")
 	assert.Equal(t, simpleItem(1), queue.Dequeue().(simpleItem), "Expected to dequeue simpleItem(1)")
 
@@ -51,6 +47,7 @@ func TestPriorityQueuePriorities2(t *testing.T) {
 	queue.Enqueue(simpleItem(2))
 	queue.Enqueue(simpleItem(1))
 
+	assert.Equal(t, simpleItem(2), queue.Peek().(simpleItem), "Expected to peek simpleItem(2)")
 	assert.Equal(t, simpleItem(2), queue.Dequeue().(simpleItem), "Expected to dequeue simpleItem(2)")
 	assert.Equal(t, simpleItem(1), queue.Dequeue().(simpleItem), "Expected to dequeue simpleItem(1)")
 

From ca2cd1cd502f3c270f0e846fa1baefd95a4a010c Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Wed, 8 Nov 2023 16:47:39 -0800
Subject: [PATCH 25/49] Add reserved querier logic

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 pkg/frontend/v1/frontend.go                   |  13 +-
 pkg/frontend/v2/frontend.go                   |  14 +-
 pkg/scheduler/queue/queue.go                  |  24 ++-
 pkg/scheduler/queue/queue_test.go             |   2 +-
 pkg/scheduler/queue/user_queues.go            | 149 ++++++++++++------
 pkg/scheduler/queue/user_queues_test.go       |  10 +-
 pkg/scheduler/queue/user_request_queue.go     |  10 +-
 .../queue/user_request_queue_test.go          |  19 ++-
 pkg/util/priority_queue.go                    |   6 +-
 pkg/util/query/priority.go                    |  16 +-
 pkg/util/query/priority_test.go               |  99 +++++-------
 pkg/util/validation/limits.go                 |  18 ---
 pkg/util/validation/limits_test.go            |   1 -
 13 files changed, 225 insertions(+), 156 deletions(-)

diff --git a/pkg/frontend/v1/frontend.go b/pkg/frontend/v1/frontend.go
index d3734e9bcf..7b952fff2b 100644
--- a/pkg/frontend/v1/frontend.go
+++ b/pkg/frontend/v1/frontend.go
@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
+	"reflect"
 	"time"
 
 	"github.com/go-kit/log"
@@ -79,6 +80,9 @@ type Frontend struct {
 	requestQueue *queue.RequestQueue
 	activeUsers  *util.ActiveUsersCleanupService
 
+	queryPriority         validation.QueryPriority
+	compiledQueryPriority validation.QueryPriority
+
 	// Subservices manager.
 	subservices        *services.Manager
 	subservicesWatcher *services.FailureWatcher
@@ -207,8 +211,15 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest,
 		}
 
 		queryPriority := f.limits.QueryPriority(userID)
+
 		if reqParams != nil && queryPriority.Enabled {
-			request.priority = util_query.GetPriority(reqParams, ts, queryPriority)
+			queryPriorityChanged := !reflect.DeepEqual(f.queryPriority, queryPriority)
+			if queryPriorityChanged {
+				f.queryPriority = queryPriority
+				f.compiledQueryPriority = queryPriority
+			}
+
+			request.priority = util_query.GetPriority(reqParams, ts, &f.compiledQueryPriority, queryPriorityChanged)
 		}
 
 		if err := f.queueRequest(ctx, &request); err != nil {
diff --git a/pkg/frontend/v2/frontend.go b/pkg/frontend/v2/frontend.go
index e06f3f97c1..c99980100e 100644
--- a/pkg/frontend/v2/frontend.go
+++ b/pkg/frontend/v2/frontend.go
@@ -7,6 +7,7 @@ import (
 	"math/rand"
 	"net/http"
 	"net/url"
+	"reflect"
 	"sync"
 	"time"
 
@@ -30,6 +31,7 @@ import (
 	util_log "github.com/cortexproject/cortex/pkg/util/log"
 	util_query "github.com/cortexproject/cortex/pkg/util/query"
 	"github.com/cortexproject/cortex/pkg/util/services"
+	"github.com/cortexproject/cortex/pkg/util/validation"
 )
 
 // Config for a Frontend.
@@ -74,6 +76,9 @@ type Frontend struct {
 
 	lastQueryID atomic.Uint64
 
+	queryPriority         validation.QueryPriority
+	compiledQueryPriority validation.QueryPriority
+
 	// frontend workers will read from this channel, and send request to scheduler.
 	requestsCh chan *frontendRequest
 
@@ -211,8 +216,15 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest,
 		}
 
 		queryPriority := f.limits.QueryPriority(userID)
+
 		if reqParams != nil && queryPriority.Enabled {
-			freq.priority = util_query.GetPriority(reqParams, ts, queryPriority)
+			queryPriorityChanged := !reflect.DeepEqual(f.queryPriority, queryPriority)
+			if queryPriorityChanged {
+				f.queryPriority = queryPriority
+				f.compiledQueryPriority = queryPriority
+			}
+
+			freq.priority = util_query.GetPriority(reqParams, ts, &f.compiledQueryPriority, queryPriorityChanged)
 		}
 
 		f.requests.put(freq)
diff --git a/pkg/scheduler/queue/queue.go b/pkg/scheduler/queue/queue.go
index e6c55ca53d..3fc0fd16d3 100644
--- a/pkg/scheduler/queue/queue.go
+++ b/pkg/scheduler/queue/queue.go
@@ -98,7 +98,8 @@ func (q *RequestQueue) EnqueueRequest(userID string, req Request, maxQueriers fl
 	}
 
 	shardSize := util.DynamicShardSize(maxQueriers, len(q.queues.queriers))
-	queue := q.queues.getOrAddQueue(userID, shardSize)
+	priorityList, priorityEnabled := q.getPriorityList(userID)
+	queue := q.queues.getOrAddQueue(userID, shardSize, priorityList, priorityEnabled)
 	maxOutstandingRequests := q.queues.limits.MaxOutstandingPerTenant(userID)
 
 	if queue == nil {
@@ -123,6 +124,24 @@ func (q *RequestQueue) EnqueueRequest(userID string, req Request, maxQueriers fl
 	return nil
 }
 
+func (q *RequestQueue) getPriorityList(userID string) ([]int64, bool) {
+	var priorityList []int64
+
+	queryPriority := q.queues.limits.QueryPriority(userID)
+
+	if queryPriority.Enabled {
+		for _, priority := range queryPriority.Priorities {
+			reservedQuerierShardSize := util.DynamicShardSize(priority.ReservedQueriers, len(q.queues.queriers))
+
+			for i := 0; i < reservedQuerierShardSize; i++ {
+				priorityList = append(priorityList, priority.Priority)
+			}
+		}
+	}
+
+	return priorityList, queryPriority.Enabled
+}
+
 // GetNextRequestForQuerier find next user queue and takes the next request off of it. Will block if there are no requests.
 // By passing user index from previous call of this method, querier guarantees that it iterates over all users fairly.
 // If querier finds that request from the user is already expired, it can get a request for the same user by using UserIndex.ReuseLastUser.
@@ -156,7 +175,8 @@ FindQueue:
 
 		// Pick next request from the queue.
 		for {
-			request := queue.dequeueRequest(q.queues.getMinPriority(userID, querierID))
+			minPriority, checkMinPriority := q.queues.getMinPriority(userID, querierID)
+			request := queue.dequeueRequest(minPriority, checkMinPriority)
 			if request == nil {
 				// the queue does not contain request with the min priority, break to wait for more requests
 				break
diff --git a/pkg/scheduler/queue/queue_test.go b/pkg/scheduler/queue/queue_test.go
index e77dc33e6e..535647bc7a 100644
--- a/pkg/scheduler/queue/queue_test.go
+++ b/pkg/scheduler/queue/queue_test.go
@@ -3,7 +3,6 @@ package queue
 import (
 	"context"
 	"fmt"
-	"github.com/cortexproject/cortex/pkg/util/validation"
 	"strconv"
 	"sync"
 	"testing"
@@ -14,6 +13,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/cortexproject/cortex/pkg/util/services"
+	"github.com/cortexproject/cortex/pkg/util/validation"
 )
 
 func BenchmarkGetNextRequest(b *testing.B) {
diff --git a/pkg/scheduler/queue/user_queues.go b/pkg/scheduler/queue/user_queues.go
index 1c34f41762..7bf0f8c80b 100644
--- a/pkg/scheduler/queue/user_queues.go
+++ b/pkg/scheduler/queue/user_queues.go
@@ -2,6 +2,7 @@ package queue
 
 import (
 	"math/rand"
+	"reflect"
 	"sort"
 	"time"
 
@@ -63,8 +64,11 @@ type userQueue struct {
 
 	// If not nil, only these queriers can handle user requests. If nil, all queriers can.
 	// We set this to nil if number of available queriers <= maxQueriers.
-	queriers    map[string]struct{}
-	maxQueriers int
+	queriers         map[string]struct{}
+	reservedQueriers map[string]int64
+	priorityList     []int64
+	priorityEnabled  bool
+	maxQueriers      int
 
 	// Seed for shuffle sharding of queriers. This seed is based on userID only and is therefore consistent
 	// between different frontends.
@@ -110,7 +114,7 @@ func (q *queues) deleteQueue(userID string) {
 // MaxQueriers is used to compute which queriers should handle requests for this user.
 // If maxQueriers is <= 0, all queriers can handle this user's requests.
 // If maxQueriers has changed since the last call, queriers for this are recomputed.
-func (q *queues) getOrAddQueue(userID string, maxQueriers int) userRequestQueue {
+func (q *queues) getOrAddQueue(userID string, maxQueriers int, priorityList []int64, priorityEnabled bool) userRequestQueue {
 	// Empty user is not allowed, as that would break our users list ("" is used for free spot).
 	if userID == "" {
 		return nil
@@ -121,48 +125,12 @@ func (q *queues) getOrAddQueue(userID string, maxQueriers int) userRequestQueue
 	}
 
 	uq := q.userQueues[userID]
-	queryPriority := q.limits.QueryPriority(userID)
 
 	if uq == nil {
-		queueSize := q.limits.MaxOutstandingPerTenant(userID)
-		// 0 is the default value of the flag. If the old flag is set
-		// then we use its value for compatibility reason.
-		if q.maxUserQueueSize != 0 {
-			queueSize = q.maxUserQueueSize
-		}
-		uq = &userQueue{
-			seed:  util.ShuffleShardSeed(userID, ""),
-			index: -1,
-		}
-
-		if queryPriority.Enabled {
-			uq.queue = NewPriorityRequestQueue(util.NewPriorityQueue(nil))
-		} else {
-			uq.queue = NewFIFORequestQueue(make(chan Request, queueSize))
-		}
-
-		q.userQueues[userID] = uq
-
-		// Add user to the list of users... find first free spot, and put it there.
-		for ix, u := range q.users {
-			if u == "" {
-				uq.index = ix
-				q.users[ix] = userID
-				break
-			}
-		}
-
-		// ... or add to the end.
-		if uq.index < 0 {
-			uq.index = len(q.users)
-			q.users = append(q.users, userID)
-		}
+		uq = q.createUserQueue(userID)
 	}
 
-	if uq.maxQueriers != maxQueriers {
-		uq.maxQueriers = maxQueriers
-		uq.queriers = shuffleQueriersForUser(uq.seed, maxQueriers, q.sortedQueriers, nil)
-	}
+	q.updateUserQueuesAttributes(uq, userID, maxQueriers, priorityList, priorityEnabled)
 
 	return uq.queue
 }
@@ -196,21 +164,98 @@ func (q *queues) getNextQueueForQuerier(lastUserIndex int, querierID string) (us
 			}
 		}
 
-		// TODO: justinjung04, reserved queriers
-		//if priority, isReserved := uq.reservedQueriers[querierID]; isReserved {
-		//	return uq.queues[priority], u, uid
-		//}
-
 		return uq.queue, u, uid
 	}
 	return nil, "", uid
 }
 
-func (q *queues) getMinPriority(userID string, querierID string) int64 {
-	// TODO: justinjung04 reserved querier
-	// check list of queriers and QueryPriority config
-	// from QueryPriority config, establish map of
-	return 0
+func (q *queues) createUserQueue(userID string) *userQueue {
+	uq := &userQueue{
+		seed:  util.ShuffleShardSeed(userID, ""),
+		index: -1,
+	}
+
+	uq.queue = q.createUserRequestQueue(userID)
+	q.userQueues[userID] = uq
+
+	// Add user to the list of users... find first free spot, and put it there.
+	for ix, u := range q.users {
+		if u == "" {
+			uq.index = ix
+			q.users[ix] = userID
+			break
+		}
+	}
+
+	// ... or add to the end.
+	if uq.index < 0 {
+		uq.index = len(q.users)
+		q.users = append(q.users, userID)
+	}
+
+	return uq
+}
+
+func (q *queues) createUserRequestQueue(userID string) userRequestQueue {
+	if q.limits.QueryPriority(userID).Enabled {
+		return NewPriorityRequestQueue(util.NewPriorityQueue(nil))
+	}
+
+	return NewFIFORequestQueue(make(chan Request, q.getQueueSize(userID)))
+}
+
+func (q *queues) getQueueSize(userID string) int {
+	queueSize := q.limits.MaxOutstandingPerTenant(userID)
+
+	// 0 is the default value of the flag. If the old flag is set
+	// then we use its value for compatibility reason.
+	if q.maxUserQueueSize != 0 {
+		queueSize = q.maxUserQueueSize
+	}
+
+	return queueSize
+}
+
+func (q *queues) updateUserQueuesAttributes(uq *userQueue, userID string, maxQueriers int, priorityList []int64, priorityEnabled bool) {
+	if uq.maxQueriers != maxQueriers {
+		uq.maxQueriers = maxQueriers
+		uq.queriers = shuffleQueriersForUser(uq.seed, maxQueriers, q.sortedQueriers, nil)
+	}
+
+	// if query priority is newly enabled/disabled, transfer the requests to the new queue
+	if uq.priorityEnabled != priorityEnabled {
+		tmpQueue := q.createUserRequestQueue(userID)
+		uq.queue.closeQueue()
+
+		for uq.queue.length() > 0 {
+			tmpQueue.enqueueRequest(uq.queue.dequeueRequest(0, false))
+		}
+		uq.queue = tmpQueue
+	}
+
+	if priorityEnabled && !reflect.DeepEqual(uq.priorityList, priorityList) {
+		reservedQueriers := make(map[string]int64)
+
+		i := 0
+		for querierID := range uq.queriers {
+			reservedQueriers[querierID] = priorityList[i]
+			i++
+			if i == len(priorityList) {
+				break
+			}
+		}
+
+		uq.reservedQueriers = reservedQueriers
+		uq.priorityList = priorityList
+		uq.priorityEnabled = priorityEnabled
+	}
+}
+
+func (q *queues) getMinPriority(userID string, querierID string) (int64, bool) {
+	if priority, ok := q.userQueues[userID].reservedQueriers[querierID]; ok {
+		return priority, true
+	}
+	return 0, false
 }
 
 func (q *queues) addQuerierConnection(querierID string) {
@@ -349,7 +394,7 @@ type MockLimits struct {
 	queryPriority      validation.QueryPriority
 }
 
-func (l MockLimits) MaxQueriersPerUser(user string) float64 {
+func (l MockLimits) MaxQueriersPerUser(_ string) float64 {
 	return l.maxQueriersPerUser
 }
 
diff --git a/pkg/scheduler/queue/user_queues_test.go b/pkg/scheduler/queue/user_queues_test.go
index 729ae9699d..4b141dbf88 100644
--- a/pkg/scheduler/queue/user_queues_test.go
+++ b/pkg/scheduler/queue/user_queues_test.go
@@ -156,7 +156,7 @@ func TestQueuesConsistency(t *testing.T) {
 			conns := map[string]int{}
 
 			for i := 0; i < 10000; i++ {
-				queue := uq.getOrAddQueue(generateTenant(r), 3)
+				queue := uq.getOrAddQueue(generateTenant(r), 3, []int64{}, true)
 				switch r.Int() % 6 {
 				case 0:
 					assert.NotNil(t, queue)
@@ -352,6 +352,10 @@ func TestQueues_ForgetDelay_ShouldCorrectlyHandleQuerierReconnectingBeforeForget
 	}
 }
 
+func TestQueuesUpdatedConfig(t *testing.T) {
+	// TODO: justinjung04
+}
+
 func generateTenant(r *rand.Rand) string {
 	return fmt.Sprint("tenant-", r.Int()%5)
 }
@@ -361,10 +365,10 @@ func generateQuerier(r *rand.Rand) string {
 }
 
 func getOrAdd(t *testing.T, uq *queues, tenant string, maxQueriers int) userRequestQueue {
-	q := uq.getOrAddQueue(tenant, maxQueriers)
+	q := uq.getOrAddQueue(tenant, maxQueriers, []int64{}, true)
 	assert.NotNil(t, q)
 	assert.NoError(t, isConsistent(uq))
-	assert.Equal(t, q, uq.getOrAddQueue(tenant, maxQueriers))
+	assert.Equal(t, q, uq.getOrAddQueue(tenant, maxQueriers, []int64{}, true))
 	return q
 }
 
diff --git a/pkg/scheduler/queue/user_request_queue.go b/pkg/scheduler/queue/user_request_queue.go
index b8ee1ecaf7..77afa31806 100644
--- a/pkg/scheduler/queue/user_request_queue.go
+++ b/pkg/scheduler/queue/user_request_queue.go
@@ -4,7 +4,7 @@ import "github.com/cortexproject/cortex/pkg/util"
 
 type userRequestQueue interface {
 	enqueueRequest(Request)
-	dequeueRequest(minPriority int64) Request
+	dequeueRequest(minPriority int64, checkMinPriority bool) Request
 	length() int
 	closeQueue()
 }
@@ -21,7 +21,7 @@ func (f *FIFORequestQueue) enqueueRequest(r Request) {
 	f.queue <- r
 }
 
-func (f *FIFORequestQueue) dequeueRequest(_ int64) Request {
+func (f *FIFORequestQueue) dequeueRequest(_ int64, _ bool) Request {
 	return <-f.queue
 }
 
@@ -45,8 +45,8 @@ func (f *PriorityRequestQueue) enqueueRequest(r Request) {
 	f.queue.Enqueue(r)
 }
 
-func (f *PriorityRequestQueue) dequeueRequest(minPriority int64) Request {
-	if f.queue.Peek().Priority() < minPriority {
+func (f *PriorityRequestQueue) dequeueRequest(minPriority int64, checkMinPriority bool) Request {
+	if checkMinPriority && f.queue.Peek().Priority() < minPriority {
 		return nil
 	}
 	return f.queue.Dequeue()
@@ -57,5 +57,5 @@ func (f *PriorityRequestQueue) length() int {
 }
 
 func (f *PriorityRequestQueue) closeQueue() {
-	f.queue.DiscardAndClose()
+	f.queue.Close()
 }
diff --git a/pkg/scheduler/queue/user_request_queue_test.go b/pkg/scheduler/queue/user_request_queue_test.go
index aeaad7fb2a..ec4c728f89 100644
--- a/pkg/scheduler/queue/user_request_queue_test.go
+++ b/pkg/scheduler/queue/user_request_queue_test.go
@@ -1,9 +1,11 @@
 package queue
 
 import (
-	"github.com/cortexproject/cortex/pkg/util"
-	"github.com/stretchr/testify/assert"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/cortexproject/cortex/pkg/util"
 )
 
 func TestFIFORequestQueue(t *testing.T) {
@@ -20,9 +22,9 @@ func TestFIFORequestQueue(t *testing.T) {
 	queue.enqueueRequest(request1)
 	queue.enqueueRequest(request2)
 	assert.Equal(t, 2, queue.length())
-	assert.Equal(t, request1, queue.dequeueRequest(0))
+	assert.Equal(t, request1, queue.dequeueRequest(0, false))
 	assert.Equal(t, 1, queue.length())
-	assert.Equal(t, request2, queue.dequeueRequest(0))
+	assert.Equal(t, request2, queue.dequeueRequest(0, false))
 	assert.Equal(t, 0, queue.length())
 	queue.closeQueue()
 	assert.Panics(t, func() { queue.enqueueRequest(request1) })
@@ -42,16 +44,17 @@ func TestPriorityRequestQueue(t *testing.T) {
 	queue.enqueueRequest(request1)
 	queue.enqueueRequest(request2)
 	assert.Equal(t, 2, queue.length())
-	assert.Equal(t, request2, queue.dequeueRequest(0))
+	assert.Equal(t, request2, queue.dequeueRequest(0, true))
 	assert.Equal(t, 1, queue.length())
-	assert.Equal(t, request1, queue.dequeueRequest(0))
+	assert.Equal(t, request1, queue.dequeueRequest(0, true))
 	assert.Equal(t, 0, queue.length())
 
 	queue.enqueueRequest(request1)
 	queue.enqueueRequest(request2)
 	assert.Equal(t, 2, queue.length())
-	assert.Equal(t, request2, queue.dequeueRequest(2))
-	
+	assert.Equal(t, request2, queue.dequeueRequest(2, true))
+	assert.Equal(t, request1, queue.dequeueRequest(2, false))
+
 	queue.closeQueue()
 	assert.Panics(t, func() { queue.enqueueRequest(request1) })
 }
diff --git a/pkg/util/priority_queue.go b/pkg/util/priority_queue.go
index 26e0b13832..543192e126 100644
--- a/pkg/util/priority_queue.go
+++ b/pkg/util/priority_queue.go
@@ -64,7 +64,11 @@ func (pq *PriorityQueue) Length() int {
 func (pq *PriorityQueue) Close() {
 	pq.lock.Lock()
 	defer pq.lock.Unlock()
-	pq.closing = true
+	if len(pq.queue) > 0 {
+		pq.closing = true
+	} else {
+		pq.closed = true
+	}
 	pq.cond.Broadcast()
 }
 
diff --git a/pkg/util/query/priority.go b/pkg/util/query/priority.go
index e37dd8f0c9..e32c5df89b 100644
--- a/pkg/util/query/priority.go
+++ b/pkg/util/query/priority.go
@@ -3,6 +3,7 @@ package query
 import (
 	"math"
 	"net/url"
+	"regexp"
 	"strconv"
 	"time"
 
@@ -11,7 +12,7 @@ import (
 	"github.com/cortexproject/cortex/pkg/util/validation"
 )
 
-func GetPriority(requestParams url.Values, now time.Time, queryPriority validation.QueryPriority) int64 {
+func GetPriority(requestParams url.Values, now time.Time, queryPriority *validation.QueryPriority, queryPriorityChanged bool) int64 {
 	queryParam := requestParams.Get("query")
 	timeParam := requestParams.Get("time")
 	startParam := requestParams.Get("start")
@@ -21,8 +22,17 @@ func GetPriority(requestParams url.Values, now time.Time, queryPriority validati
 		return queryPriority.DefaultPriority
 	}
 
-	for _, priority := range queryPriority.Priorities {
-		for _, attribute := range priority.QueryAttributes {
+	for i, priority := range queryPriority.Priorities {
+		for j, attribute := range priority.QueryAttributes {
+			if queryPriorityChanged {
+				compiledRegex, err := regexp.Compile(attribute.Regex)
+				if err != nil {
+					continue
+				}
+
+				queryPriority.Priorities[i].QueryAttributes[j].CompiledRegex = compiledRegex
+			}
+
 			if attribute.CompiledRegex != nil && !attribute.CompiledRegex.MatchString(queryParam) {
 				continue
 			}
diff --git a/pkg/util/query/priority_test.go b/pkg/util/query/priority_test.go
index 696fb33af2..0a93ac355b 100644
--- a/pkg/util/query/priority_test.go
+++ b/pkg/util/query/priority_test.go
@@ -1,12 +1,14 @@
 package query
 
 import (
-	"github.com/cortexproject/cortex/pkg/util/validation"
-	"github.com/stretchr/testify/assert"
 	"net/url"
 	"strconv"
 	"testing"
 	"time"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/cortexproject/cortex/pkg/util/validation"
 )
 
 func Test_GetPriorityShouldReturnDefaultPriorityIfNotEnabledOrEmptyQueryString(t *testing.T) {
@@ -23,21 +25,20 @@ func Test_GetPriorityShouldReturnDefaultPriorityIfNotEnabledOrEmptyQueryString(t
 			},
 		},
 	}
+	queryPriority := validation.QueryPriority{
+		Priorities: priorities,
+	}
 
 	assert.Equal(t, int64(0), GetPriority(url.Values{
 		"query": []string{"sum(up)"},
 		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
-	}, now, validation.QueryPriority{
-		Priorities: priorities,
-	}))
+	}, now, &queryPriority, true))
 
+	queryPriority.Enabled = true
 	assert.Equal(t, int64(0), GetPriority(url.Values{
 		"query": []string{""},
 		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
-	}, now, validation.QueryPriority{
-		Enabled:    true,
-		Priorities: priorities,
-	}))
+	}, now, &queryPriority, true))
 }
 
 func Test_GetPriorityShouldConsiderRegex(t *testing.T) {
@@ -54,72 +55,52 @@ func Test_GetPriorityShouldConsiderRegex(t *testing.T) {
 			},
 		},
 	}
-	limits, _ := validation.NewOverrides(validation.Limits{
-		QueryPriority: validation.QueryPriority{
-			Enabled:    true,
-			Priorities: priorities,
-		},
-	}, nil)
+	queryPriority := validation.QueryPriority{
+		Enabled:    true,
+		Priorities: priorities,
+	}
 
 	assert.Equal(t, int64(1), GetPriority(url.Values{
 		"query": []string{"sum(up)"},
 		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
-	}, now, limits.QueryPriority("")))
+	}, now, &queryPriority, true))
 	assert.Equal(t, int64(0), GetPriority(url.Values{
 		"query": []string{"count(up)"},
 		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
-	}, now, limits.QueryPriority("")))
+	}, now, &queryPriority, false))
 
-	priorities[0].QueryAttributes[0].Regex = "(^sum$|c(.+)t)"
-	limits, _ = validation.NewOverrides(validation.Limits{
-		QueryPriority: validation.QueryPriority{
-			Enabled:    true,
-			Priorities: priorities,
-		},
-	}, nil)
+	queryPriority.Priorities[0].QueryAttributes[0].Regex = "(^sum$|c(.+)t)"
 
 	assert.Equal(t, int64(0), GetPriority(url.Values{
 		"query": []string{"sum(up)"},
 		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
-	}, now, limits.QueryPriority("")))
+	}, now, &queryPriority, true))
 	assert.Equal(t, int64(1), GetPriority(url.Values{
 		"query": []string{"count(up)"},
 		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
-	}, now, limits.QueryPriority("")))
+	}, now, &queryPriority, false))
 
-	priorities[0].QueryAttributes[0].Regex = ".*"
-	limits, _ = validation.NewOverrides(validation.Limits{
-		QueryPriority: validation.QueryPriority{
-			Enabled:    true,
-			Priorities: priorities,
-		},
-	}, nil)
+	queryPriority.Priorities[0].QueryAttributes[0].Regex = ".*"
 
 	assert.Equal(t, int64(1), GetPriority(url.Values{
 		"query": []string{"sum(up)"},
 		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
-	}, now, limits.QueryPriority("")))
+	}, now, &queryPriority, true))
 	assert.Equal(t, int64(1), GetPriority(url.Values{
 		"query": []string{"count(up)"},
 		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
-	}, now, limits.QueryPriority("")))
+	}, now, &queryPriority, false))
 
-	priorities[0].QueryAttributes[0].Regex = ""
-	limits, _ = validation.NewOverrides(validation.Limits{
-		QueryPriority: validation.QueryPriority{
-			Enabled:    true,
-			Priorities: priorities,
-		},
-	}, nil)
+	queryPriority.Priorities[0].QueryAttributes[0].Regex = ""
 
 	assert.Equal(t, int64(1), GetPriority(url.Values{
 		"query": []string{"sum(up)"},
 		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
-	}, now, limits.QueryPriority("")))
+	}, now, &queryPriority, true))
 	assert.Equal(t, int64(1), GetPriority(url.Values{
 		"query": []string{"count(up)"},
 		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
-	}, now, limits.QueryPriority("")))
+	}, now, &queryPriority, false))
 }
 
 func Test_GetPriorityShouldConsiderStartAndEndTime(t *testing.T) {
@@ -135,57 +116,55 @@ func Test_GetPriorityShouldConsiderStartAndEndTime(t *testing.T) {
 			},
 		},
 	}
-	limits, _ := validation.NewOverrides(validation.Limits{
-		QueryPriority: validation.QueryPriority{
-			Enabled:    true,
-			Priorities: priorities,
-		},
-	}, nil)
+	queryPriority := validation.QueryPriority{
+		Enabled:    true,
+		Priorities: priorities,
+	}
 
 	assert.Equal(t, int64(0), GetPriority(url.Values{
 		"query": []string{"sum(up)"},
 		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
-	}, now, limits.QueryPriority("")))
+	}, now, &queryPriority, true))
 	assert.Equal(t, int64(1), GetPriority(url.Values{
 		"query": []string{"sum(up)"},
 		"time":  []string{strconv.FormatInt(now.Add(-30*time.Minute).Unix(), 10)},
-	}, now, limits.QueryPriority("")))
+	}, now, &queryPriority, false))
 	assert.Equal(t, int64(0), GetPriority(url.Values{
 		"query": []string{"sum(up)"},
 		"time":  []string{strconv.FormatInt(now.Add(-60*time.Minute).Unix(), 10)},
-	}, now, limits.QueryPriority("")))
+	}, now, &queryPriority, false))
 	assert.Equal(t, int64(1), GetPriority(url.Values{
 		"query": []string{"sum(up)"},
 		"time":  []string{strconv.FormatInt(now.Add(-45*time.Minute).Unix(), 10)},
-	}, now, limits.QueryPriority("")))
+	}, now, &queryPriority, false))
 	assert.Equal(t, int64(1), GetPriority(url.Values{
 		"query": []string{"sum(up)"},
 		"time":  []string{strconv.FormatInt(now.Add(-15*time.Minute).Unix(), 10)},
-	}, now, limits.QueryPriority("")))
+	}, now, &queryPriority, false))
 
 	assert.Equal(t, int64(1), GetPriority(url.Values{
 		"query": []string{"sum(up)"},
 		"start": []string{strconv.FormatInt(now.Add(-45*time.Minute).Unix(), 10)},
 		"end":   []string{strconv.FormatInt(now.Add(-15*time.Minute).Unix(), 10)},
-	}, now, limits.QueryPriority("")))
+	}, now, &queryPriority, false))
 	assert.Equal(t, int64(0), GetPriority(url.Values{
 		"query": []string{"sum(up)"},
 		"start": []string{strconv.FormatInt(now.Add(-50*time.Minute).Unix(), 10)},
 		"end":   []string{strconv.FormatInt(now.Add(-15*time.Minute).Unix(), 10)},
-	}, now, limits.QueryPriority("")))
+	}, now, &queryPriority, false))
 	assert.Equal(t, int64(0), GetPriority(url.Values{
 		"query": []string{"sum(up)"},
 		"start": []string{strconv.FormatInt(now.Add(-45*time.Minute).Unix(), 10)},
 		"end":   []string{strconv.FormatInt(now.Add(-10*time.Minute).Unix(), 10)},
-	}, now, limits.QueryPriority("")))
+	}, now, &queryPriority, false))
 	assert.Equal(t, int64(0), GetPriority(url.Values{
 		"query": []string{"sum(up)"},
 		"start": []string{strconv.FormatInt(now.Add(-60*time.Minute).Unix(), 10)},
 		"end":   []string{strconv.FormatInt(now.Add(-45*time.Minute).Unix(), 10)},
-	}, now, limits.QueryPriority("")))
+	}, now, &queryPriority, false))
 	assert.Equal(t, int64(0), GetPriority(url.Values{
 		"query": []string{"sum(up)"},
 		"start": []string{strconv.FormatInt(now.Add(-15*time.Minute).Unix(), 10)},
 		"end":   []string{strconv.FormatInt(now.Add(-1*time.Minute).Unix(), 10)},
-	}, now, limits.QueryPriority("")))
+	}, now, &queryPriority, false))
 }
diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go
index 10db7b9bc9..0403d0c495 100644
--- a/pkg/util/validation/limits.go
+++ b/pkg/util/validation/limits.go
@@ -52,7 +52,6 @@ type QueryPriority struct {
 	Enabled         bool          `yaml:"enabled" doc:"nocli|description=Whether queries are assigned with priorities.|default=false"`
 	DefaultPriority int64         `yaml:"default_priority" doc:"nocli|description=Priority assigned to all queries by default. Must be a unique value. Use this as a baseline to make certain queries higher/lower priority.|default=0"`
 	Priorities      []PriorityDef `yaml:"priorities" doc:"nocli|description=List of priority definitions."`
-	RegexCompiled   bool          `yaml:"-" doc:"nocli"`
 }
 
 type PriorityDef struct {
@@ -530,23 +529,6 @@ func (o *Overrides) MaxOutstandingPerTenant(userID string) int {
 
 // QueryPriority returns the query priority config for the tenant, including different priorities and their attributes
 func (o *Overrides) QueryPriority(userID string) QueryPriority {
-	queryPriority := o.GetOverridesForUser(userID).QueryPriority
-
-	if !queryPriority.RegexCompiled {
-		priorities := queryPriority.Priorities
-		for i, priority := range priorities {
-			for j, attributes := range priority.QueryAttributes {
-				if attributes.Regex == "" || attributes.Regex == ".*" || attributes.Regex == ".+" {
-					// if it matches all, don't use regex
-					o.GetOverridesForUser(userID).QueryPriority.Priorities[i].QueryAttributes[j].CompiledRegex = nil
-				} else {
-					o.GetOverridesForUser(userID).QueryPriority.Priorities[i].QueryAttributes[j].CompiledRegex, _ = regexp.Compile(attributes.Regex)
-				}
-			}
-		}
-		o.GetOverridesForUser(userID).QueryPriority.RegexCompiled = true
-	}
-
 	return o.GetOverridesForUser(userID).QueryPriority
 }
 
diff --git a/pkg/util/validation/limits_test.go b/pkg/util/validation/limits_test.go
index f2de1300e6..c3b8c7e422 100644
--- a/pkg/util/validation/limits_test.go
+++ b/pkg/util/validation/limits_test.go
@@ -194,7 +194,6 @@ func TestQueryPriority(t *testing.T) {
 		} else {
 			assert.NotNil(t, queryPriority.Priorities[0].QueryAttributes[0].CompiledRegex)
 		}
-		assert.True(t, queryPriority.RegexCompiled)
 	}
 }
 

From 067cc836cd5660ff62f5497078114b605f9bb9e1 Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Thu, 9 Nov 2023 04:14:09 -0800
Subject: [PATCH 26/49] Fix tests

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 pkg/scheduler/queue/queue.go                  |  2 +-
 pkg/scheduler/queue/user_queues.go            |  8 ++-
 pkg/scheduler/queue/user_queues_test.go       |  3 +-
 .../queue/user_request_queue_test.go          |  3 +
 pkg/util/query/priority.go                    |  3 +-
 pkg/util/query/priority_test.go               | 13 +++++
 pkg/util/validation/limits_test.go            | 58 -------------------
 7 files changed, 26 insertions(+), 64 deletions(-)

diff --git a/pkg/scheduler/queue/queue.go b/pkg/scheduler/queue/queue.go
index 3fc0fd16d3..f316229499 100644
--- a/pkg/scheduler/queue/queue.go
+++ b/pkg/scheduler/queue/queue.go
@@ -107,7 +107,7 @@ func (q *RequestQueue) EnqueueRequest(userID string, req Request, maxQueriers fl
 		return errors.New("no queue found")
 	}
 
-	q.totalRequests.WithLabelValues(userID).Inc()
+	q.totalRequests.WithLabelValues(userID).Inc() // TODO: justinjung04
 
 	if queue.length() >= maxOutstandingRequests {
 		q.discardedRequests.WithLabelValues(userID).Inc()
diff --git a/pkg/scheduler/queue/user_queues.go b/pkg/scheduler/queue/user_queues.go
index 7bf0f8c80b..9c1e2452f2 100644
--- a/pkg/scheduler/queue/user_queues.go
+++ b/pkg/scheduler/queue/user_queues.go
@@ -238,11 +238,11 @@ func (q *queues) updateUserQueuesAttributes(uq *userQueue, userID string, maxQue
 
 		i := 0
 		for querierID := range uq.queriers {
-			reservedQueriers[querierID] = priorityList[i]
-			i++
 			if i == len(priorityList) {
 				break
 			}
+			reservedQueriers[querierID] = priorityList[i]
+			i++
 		}
 
 		uq.reservedQueriers = reservedQueriers
@@ -370,6 +370,10 @@ func (q *queues) recomputeUserQueriers() {
 // In that case *all* queriers should be used.
 // Scratchpad is used for shuffling, to avoid new allocations. If nil, new slice is allocated.
 func shuffleQueriersForUser(userSeed int64, queriersToSelect int, allSortedQueriers []string, scratchpad []string) map[string]struct{} {
+	if queriersToSelect == 0 || len(allSortedQueriers) <= queriersToSelect {
+		return nil
+	}
+
 	queriers := make(map[string]struct{}, queriersToSelect)
 	rnd := rand.New(rand.NewSource(userSeed))
 
diff --git a/pkg/scheduler/queue/user_queues_test.go b/pkg/scheduler/queue/user_queues_test.go
index 4b141dbf88..dd4cc3f5e1 100644
--- a/pkg/scheduler/queue/user_queues_test.go
+++ b/pkg/scheduler/queue/user_queues_test.go
@@ -156,10 +156,9 @@ func TestQueuesConsistency(t *testing.T) {
 			conns := map[string]int{}
 
 			for i := 0; i < 10000; i++ {
-				queue := uq.getOrAddQueue(generateTenant(r), 3, []int64{}, true)
 				switch r.Int() % 6 {
 				case 0:
-					assert.NotNil(t, queue)
+					assert.NotNil(t, uq.getOrAddQueue(generateTenant(r), 3, []int64{}, false))
 				case 1:
 					qid := generateQuerier(r)
 					_, _, luid := uq.getNextQueueForQuerier(lastUserIndexes[qid], qid)
diff --git a/pkg/scheduler/queue/user_request_queue_test.go b/pkg/scheduler/queue/user_request_queue_test.go
index ec4c728f89..84a4ce208e 100644
--- a/pkg/scheduler/queue/user_request_queue_test.go
+++ b/pkg/scheduler/queue/user_request_queue_test.go
@@ -26,6 +26,7 @@ func TestFIFORequestQueue(t *testing.T) {
 	assert.Equal(t, 1, queue.length())
 	assert.Equal(t, request2, queue.dequeueRequest(0, false))
 	assert.Equal(t, 0, queue.length())
+
 	queue.closeQueue()
 	assert.Panics(t, func() { queue.enqueueRequest(request1) })
 }
@@ -53,7 +54,9 @@ func TestPriorityRequestQueue(t *testing.T) {
 	queue.enqueueRequest(request2)
 	assert.Equal(t, 2, queue.length())
 	assert.Equal(t, request2, queue.dequeueRequest(2, true))
+	assert.Equal(t, 1, queue.length())
 	assert.Equal(t, request1, queue.dequeueRequest(2, false))
+	assert.Equal(t, 0, queue.length())
 
 	queue.closeQueue()
 	assert.Panics(t, func() { queue.enqueueRequest(request1) })
diff --git a/pkg/util/query/priority.go b/pkg/util/query/priority.go
index e32c5df89b..fa63cd3c30 100644
--- a/pkg/util/query/priority.go
+++ b/pkg/util/query/priority.go
@@ -30,7 +30,8 @@ func GetPriority(requestParams url.Values, now time.Time, queryPriority *validat
 					continue
 				}
 
-				queryPriority.Priorities[i].QueryAttributes[j].CompiledRegex = compiledRegex
+				attribute.CompiledRegex = compiledRegex
+				queryPriority.Priorities[i].QueryAttributes[j] = attribute
 			}
 
 			if attribute.CompiledRegex != nil && !attribute.CompiledRegex.MatchString(queryParam) {
diff --git a/pkg/util/query/priority_test.go b/pkg/util/query/priority_test.go
index 0a93ac355b..0d1193a2f3 100644
--- a/pkg/util/query/priority_test.go
+++ b/pkg/util/query/priority_test.go
@@ -60,10 +60,12 @@ func Test_GetPriorityShouldConsiderRegex(t *testing.T) {
 		Priorities: priorities,
 	}
 
+	assert.Nil(t, queryPriority.Priorities[0].QueryAttributes[0].CompiledRegex)
 	assert.Equal(t, int64(1), GetPriority(url.Values{
 		"query": []string{"sum(up)"},
 		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
 	}, now, &queryPriority, true))
+	assert.NotNil(t, queryPriority.Priorities[0].QueryAttributes[0].CompiledRegex)
 	assert.Equal(t, int64(0), GetPriority(url.Values{
 		"query": []string{"count(up)"},
 		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
@@ -91,6 +93,17 @@ func Test_GetPriorityShouldConsiderRegex(t *testing.T) {
 		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
 	}, now, &queryPriority, false))
 
+	queryPriority.Priorities[0].QueryAttributes[0].Regex = ".+"
+
+	assert.Equal(t, int64(1), GetPriority(url.Values{
+		"query": []string{"sum(up)"},
+		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
+	}, now, &queryPriority, true))
+	assert.Equal(t, int64(1), GetPriority(url.Values{
+		"query": []string{"count(up)"},
+		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
+	}, now, &queryPriority, false))
+
 	queryPriority.Priorities[0].QueryAttributes[0].Regex = ""
 
 	assert.Equal(t, int64(1), GetPriority(url.Values{
diff --git a/pkg/util/validation/limits_test.go b/pkg/util/validation/limits_test.go
index c3b8c7e422..01c18225c2 100644
--- a/pkg/util/validation/limits_test.go
+++ b/pkg/util/validation/limits_test.go
@@ -154,64 +154,6 @@ func TestOverridesManager_GetOverrides(t *testing.T) {
 	require.Equal(t, 0, ov.MaxLabelsSizeBytes("user2"))
 }
 
-func TestQueryPriority(t *testing.T) {
-	type testCase struct {
-		regex            string
-		compiledRegexNil bool
-	}
-	testCases := []testCase{
-		{
-			regex:            "",
-			compiledRegexNil: true,
-		},
-		{
-			regex:            ".*",
-			compiledRegexNil: true,
-		},
-		{
-			regex:            ".+",
-			compiledRegexNil: true,
-		},
-		{
-			regex:            "some_metric",
-			compiledRegexNil: false,
-		},
-	}
-
-	for _, tc := range testCases {
-		overrides, err := NewOverrides(Limits{
-			QueryPriority: QueryPriority{
-				Enabled:    true,
-				Priorities: getPriorities(tc.regex, 1*time.Hour, 0*time.Hour),
-			},
-		}, nil)
-		queryPriority := overrides.QueryPriority("")
-
-		assert.NoError(t, err)
-		assert.Equal(t, 1, len(queryPriority.Priorities[0].QueryAttributes))
-		if tc.compiledRegexNil {
-			assert.Nil(t, queryPriority.Priorities[0].QueryAttributes[0].CompiledRegex)
-		} else {
-			assert.NotNil(t, queryPriority.Priorities[0].QueryAttributes[0].CompiledRegex)
-		}
-	}
-}
-
-func getPriorities(regex string, startTime, endTime time.Duration) []PriorityDef {
-	return []PriorityDef{
-		{
-			Priority: 1,
-			QueryAttributes: []QueryAttribute{
-				{
-					Regex:     regex,
-					StartTime: startTime,
-					EndTime:   endTime,
-				},
-			},
-		},
-	}
-}
-
 func TestLimitsLoadingFromYaml(t *testing.T) {
 	SetDefaultLimitsForYAMLUnmarshalling(Limits{
 		MaxLabelNameLength: 100,

From e1b996f8dd8e1a88782606332eb7c1303bd02888 Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Thu, 9 Nov 2023 05:19:00 -0800
Subject: [PATCH 27/49] Add priority label to request and queue metrics

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 .../single-process-config-blocks-local.yaml         | 13 +------------
 pkg/frontend/v1/frontend.go                         |  4 ++--
 pkg/scheduler/queue/queue.go                        | 12 +++++++-----
 pkg/scheduler/queue/queue_test.go                   |  5 +++++
 pkg/scheduler/queue/user_queues.go                  |  3 ---
 pkg/scheduler/queue/user_queues_test.go             |  4 ----
 pkg/scheduler/queue/user_request_queue.go           |  9 ---------
 pkg/scheduler/scheduler.go                          |  4 ++--
 8 files changed, 17 insertions(+), 37 deletions(-)

diff --git a/docs/configuration/single-process-config-blocks-local.yaml b/docs/configuration/single-process-config-blocks-local.yaml
index c04e9cc07c..ce21bc3ce4 100644
--- a/docs/configuration/single-process-config-blocks-local.yaml
+++ b/docs/configuration/single-process-config-blocks-local.yaml
@@ -87,15 +87,4 @@ ruler:
 ruler_storage:
   backend: local
   local:
-    directory: /tmp/cortex/rules
-
-limits:
-  query_priority:
-    enabled: true
-    default_priority: 1
-    priorities:
-      - priority: 2
-        reserved_queriers: 1
-        query_attributes:
-          - start_time: 2h
-            end_time: 0s
\ No newline at end of file
+    directory: /tmp/cortex/rules
\ No newline at end of file
diff --git a/pkg/frontend/v1/frontend.go b/pkg/frontend/v1/frontend.go
index 7b952fff2b..c98cda732f 100644
--- a/pkg/frontend/v1/frontend.go
+++ b/pkg/frontend/v1/frontend.go
@@ -119,11 +119,11 @@ func New(cfg Config, limits Limits, log log.Logger, registerer prometheus.Regist
 		queueLength: promauto.With(registerer).NewGaugeVec(prometheus.GaugeOpts{
 			Name: "cortex_query_frontend_queue_length",
 			Help: "Number of queries in the queue.",
-		}, []string{"user"}),
+		}, []string{"user", "priority"}),
 		discardedRequests: promauto.With(registerer).NewCounterVec(prometheus.CounterOpts{
 			Name: "cortex_query_frontend_discarded_requests_total",
 			Help: "Total number of query requests discarded.",
-		}, []string{"user"}),
+		}, []string{"user", "priority"}),
 		queueDuration: promauto.With(registerer).NewHistogram(prometheus.HistogramOpts{
 			Name:    "cortex_query_frontend_queue_duration_seconds",
 			Help:    "Time spend by requests queued.",
diff --git a/pkg/scheduler/queue/queue.go b/pkg/scheduler/queue/queue.go
index f316229499..ab0296e678 100644
--- a/pkg/scheduler/queue/queue.go
+++ b/pkg/scheduler/queue/queue.go
@@ -2,6 +2,7 @@ package queue
 
 import (
 	"context"
+	"strconv"
 	"sync"
 	"time"
 
@@ -74,7 +75,7 @@ func NewRequestQueue(maxOutstandingPerTenant int, forgetDelay time.Duration, que
 		totalRequests: promauto.With(registerer).NewCounterVec(prometheus.CounterOpts{
 			Name: "cortex_request_queue_requests_total",
 			Help: "Total number of query requests going to the request queue.",
-		}, []string{"user"}),
+		}, []string{"user", "priority"}),
 		discardedRequests: discardedRequests,
 	}
 
@@ -101,21 +102,22 @@ func (q *RequestQueue) EnqueueRequest(userID string, req Request, maxQueriers fl
 	priorityList, priorityEnabled := q.getPriorityList(userID)
 	queue := q.queues.getOrAddQueue(userID, shardSize, priorityList, priorityEnabled)
 	maxOutstandingRequests := q.queues.limits.MaxOutstandingPerTenant(userID)
+	priority := strconv.FormatInt(req.Priority(), 10)
 
 	if queue == nil {
 		// This can only happen if userID is "".
 		return errors.New("no queue found")
 	}
 
-	q.totalRequests.WithLabelValues(userID).Inc() // TODO: justinjung04
+	q.totalRequests.WithLabelValues(userID, priority).Inc()
 
 	if queue.length() >= maxOutstandingRequests {
-		q.discardedRequests.WithLabelValues(userID).Inc()
+		q.discardedRequests.WithLabelValues(userID, priority).Inc()
 		return ErrTooManyRequests
 	}
 
 	queue.enqueueRequest(req)
-	q.queueLength.WithLabelValues(userID).Inc()
+	q.queueLength.WithLabelValues(userID, priority).Inc()
 	q.cond.Broadcast()
 	// Call this function while holding a lock. This guarantees that no querier can fetch the request before function returns.
 	if successFn != nil {
@@ -186,7 +188,7 @@ FindQueue:
 				q.queues.deleteQueue(userID)
 			}
 
-			q.queueLength.WithLabelValues(userID).Dec()
+			q.queueLength.WithLabelValues(userID, strconv.FormatInt(request.Priority(), 10)).Dec()
 
 			// Tell close() we've processed a request.
 			q.cond.Broadcast()
diff --git a/pkg/scheduler/queue/queue_test.go b/pkg/scheduler/queue/queue_test.go
index 535647bc7a..b2de565db3 100644
--- a/pkg/scheduler/queue/queue_test.go
+++ b/pkg/scheduler/queue/queue_test.go
@@ -228,6 +228,11 @@ func TestRequestQueue_ReservedQueriersShouldOnlyGetHighPriorityQueries(t *testin
 	//assert.Equal(t, 1, queue.queues.getTotalQueueSize("userID"))
 }
 
+func TestRequestQueue_EnqueueRequest(t *testing.T) {
+	// TODO: justinjung04
+	// check updated limit as well
+}
+
 type MockRequest struct {
 	id       string
 	priority int64
diff --git a/pkg/scheduler/queue/user_queues.go b/pkg/scheduler/queue/user_queues.go
index 9c1e2452f2..b44c1e5a72 100644
--- a/pkg/scheduler/queue/user_queues.go
+++ b/pkg/scheduler/queue/user_queues.go
@@ -100,7 +100,6 @@ func (q *queues) deleteQueue(userID string) {
 		return
 	}
 
-	uq.queue.closeQueue()
 	delete(q.userQueues, userID)
 	q.users[uq.index] = ""
 
@@ -225,7 +224,6 @@ func (q *queues) updateUserQueuesAttributes(uq *userQueue, userID string, maxQue
 	// if query priority is newly enabled/disabled, transfer the requests to the new queue
 	if uq.priorityEnabled != priorityEnabled {
 		tmpQueue := q.createUserRequestQueue(userID)
-		uq.queue.closeQueue()
 
 		for uq.queue.length() > 0 {
 			tmpQueue.enqueueRequest(uq.queue.dequeueRequest(0, false))
@@ -361,7 +359,6 @@ func (q *queues) recomputeUserQueriers() {
 	scratchpad := make([]string, 0, len(q.sortedQueriers))
 
 	for _, uq := range q.userQueues {
-		//userID := q.users[uq.index]
 		uq.queriers = shuffleQueriersForUser(uq.seed, uq.maxQueriers, q.sortedQueriers, scratchpad)
 	}
 }
diff --git a/pkg/scheduler/queue/user_queues_test.go b/pkg/scheduler/queue/user_queues_test.go
index dd4cc3f5e1..c6c6e8cfc2 100644
--- a/pkg/scheduler/queue/user_queues_test.go
+++ b/pkg/scheduler/queue/user_queues_test.go
@@ -351,10 +351,6 @@ func TestQueues_ForgetDelay_ShouldCorrectlyHandleQuerierReconnectingBeforeForget
 	}
 }
 
-func TestQueuesUpdatedConfig(t *testing.T) {
-	// TODO: justinjung04
-}
-
 func generateTenant(r *rand.Rand) string {
 	return fmt.Sprint("tenant-", r.Int()%5)
 }
diff --git a/pkg/scheduler/queue/user_request_queue.go b/pkg/scheduler/queue/user_request_queue.go
index 77afa31806..9f95c9e8c5 100644
--- a/pkg/scheduler/queue/user_request_queue.go
+++ b/pkg/scheduler/queue/user_request_queue.go
@@ -6,7 +6,6 @@ type userRequestQueue interface {
 	enqueueRequest(Request)
 	dequeueRequest(minPriority int64, checkMinPriority bool) Request
 	length() int
-	closeQueue()
 }
 
 type FIFORequestQueue struct {
@@ -29,10 +28,6 @@ func (f *FIFORequestQueue) length() int {
 	return len(f.queue)
 }
 
-func (f *FIFORequestQueue) closeQueue() {
-	close(f.queue)
-}
-
 type PriorityRequestQueue struct {
 	queue *util.PriorityQueue
 }
@@ -55,7 +50,3 @@ func (f *PriorityRequestQueue) dequeueRequest(minPriority int64, checkMinPriorit
 func (f *PriorityRequestQueue) length() int {
 	return f.queue.Length()
 }
-
-func (f *PriorityRequestQueue) closeQueue() {
-	f.queue.Close()
-}
diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go
index 4c3a0afcf8..2d7c61dac1 100644
--- a/pkg/scheduler/scheduler.go
+++ b/pkg/scheduler/scheduler.go
@@ -105,12 +105,12 @@ func NewScheduler(cfg Config, limits Limits, log log.Logger, registerer promethe
 	s.queueLength = promauto.With(registerer).NewGaugeVec(prometheus.GaugeOpts{
 		Name: "cortex_query_scheduler_queue_length",
 		Help: "Number of queries in the queue.",
-	}, []string{"user"})
+	}, []string{"user", "priority"})
 
 	s.discardedRequests = promauto.With(registerer).NewCounterVec(prometheus.CounterOpts{
 		Name: "cortex_query_scheduler_discarded_requests_total",
 		Help: "Total number of query requests discarded.",
-	}, []string{"user"})
+	}, []string{"user", "priority"})
 
 	s.requestQueue = queue.NewRequestQueue(cfg.MaxOutstandingPerTenant, cfg.QuerierForgetDelay, s.queueLength, s.discardedRequests, s.limits, registerer)
 

From 2f7947c334af7b737dd8ae2f38aaaef4cc96c768 Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Thu, 9 Nov 2023 08:39:11 -0800
Subject: [PATCH 28/49] Fix tests

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 .../configuration/single-process-config-blocks-local.yaml | 2 +-
 pkg/frontend/v1/frontend.go                               | 8 ++++++--
 pkg/frontend/v1/frontend_test.go                          | 2 +-
 pkg/scheduler/scheduler.go                                | 8 ++++++--
 pkg/scheduler/scheduler_test.go                           | 6 +++---
 5 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/docs/configuration/single-process-config-blocks-local.yaml b/docs/configuration/single-process-config-blocks-local.yaml
index ce21bc3ce4..a5eb711d97 100644
--- a/docs/configuration/single-process-config-blocks-local.yaml
+++ b/docs/configuration/single-process-config-blocks-local.yaml
@@ -87,4 +87,4 @@ ruler:
 ruler_storage:
   backend: local
   local:
-    directory: /tmp/cortex/rules
\ No newline at end of file
+    directory: /tmp/cortex/rules
diff --git a/pkg/frontend/v1/frontend.go b/pkg/frontend/v1/frontend.go
index c98cda732f..dbe5c28ae1 100644
--- a/pkg/frontend/v1/frontend.go
+++ b/pkg/frontend/v1/frontend.go
@@ -176,8 +176,12 @@ func (f *Frontend) stopping(_ error) error {
 }
 
 func (f *Frontend) cleanupInactiveUserMetrics(user string) {
-	f.queueLength.DeleteLabelValues(user)
-	f.discardedRequests.DeleteLabelValues(user)
+	f.queueLength.DeletePartialMatch(prometheus.Labels{
+		"user": user,
+	})
+	f.discardedRequests.DeletePartialMatch(prometheus.Labels{
+		"user": user,
+	})
 }
 
 // RoundTripGRPC round trips a proto (instead of a HTTP request).
diff --git a/pkg/frontend/v1/frontend_test.go b/pkg/frontend/v1/frontend_test.go
index 7951dd1d60..316c8c25fe 100644
--- a/pkg/frontend/v1/frontend_test.go
+++ b/pkg/frontend/v1/frontend_test.go
@@ -211,7 +211,7 @@ func TestFrontendMetricsCleanup(t *testing.T) {
 			require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(`
 				# HELP cortex_query_frontend_queue_length Number of queries in the queue.
 				# TYPE cortex_query_frontend_queue_length gauge
-				cortex_query_frontend_queue_length{user="1"} 0
+				cortex_query_frontend_queue_length{priority="0",user="1"} 0
 			`), "cortex_query_frontend_queue_length"))
 
 			fr.cleanupInactiveUserMetrics("1")
diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go
index 2d7c61dac1..b96b659e4b 100644
--- a/pkg/scheduler/scheduler.go
+++ b/pkg/scheduler/scheduler.go
@@ -520,8 +520,12 @@ func (s *Scheduler) stopping(_ error) error {
 }
 
 func (s *Scheduler) cleanupMetricsForInactiveUser(user string) {
-	s.queueLength.DeleteLabelValues(user)
-	s.discardedRequests.DeleteLabelValues(user)
+	s.queueLength.DeletePartialMatch(prometheus.Labels{
+		"user": user,
+	})
+	s.discardedRequests.DeletePartialMatch(prometheus.Labels{
+		"user": user,
+	})
 }
 
 func (s *Scheduler) getConnectedFrontendClientsMetric() float64 {
diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go
index c309ff195e..b055754133 100644
--- a/pkg/scheduler/scheduler_test.go
+++ b/pkg/scheduler/scheduler_test.go
@@ -430,8 +430,8 @@ func TestSchedulerMetrics(t *testing.T) {
 	require.NoError(t, promtest.GatherAndCompare(reg, strings.NewReader(`
 		# HELP cortex_query_scheduler_queue_length Number of queries in the queue.
 		# TYPE cortex_query_scheduler_queue_length gauge
-		cortex_query_scheduler_queue_length{user="another"} 1
-		cortex_query_scheduler_queue_length{user="test"} 1
+		cortex_query_scheduler_queue_length{priority="0",user="another"} 1
+		cortex_query_scheduler_queue_length{priority="0",user="test"} 1
 	`), "cortex_query_scheduler_queue_length"))
 
 	scheduler.cleanupMetricsForInactiveUser("test")
@@ -439,7 +439,7 @@ func TestSchedulerMetrics(t *testing.T) {
 	require.NoError(t, promtest.GatherAndCompare(reg, strings.NewReader(`
 		# HELP cortex_query_scheduler_queue_length Number of queries in the queue.
 		# TYPE cortex_query_scheduler_queue_length gauge
-		cortex_query_scheduler_queue_length{user="another"} 1
+		cortex_query_scheduler_queue_length{priority="0",user="another"} 1
 	`), "cortex_query_scheduler_queue_length"))
 }
 

From f9df45396673eefde8a1c43acd13a48a16b2878e Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Thu, 9 Nov 2023 09:15:42 -0800
Subject: [PATCH 29/49] Fix tests

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 pkg/scheduler/queue/user_request_queue_test.go | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/pkg/scheduler/queue/user_request_queue_test.go b/pkg/scheduler/queue/user_request_queue_test.go
index 84a4ce208e..6001f8966c 100644
--- a/pkg/scheduler/queue/user_request_queue_test.go
+++ b/pkg/scheduler/queue/user_request_queue_test.go
@@ -26,9 +26,6 @@ func TestFIFORequestQueue(t *testing.T) {
 	assert.Equal(t, 1, queue.length())
 	assert.Equal(t, request2, queue.dequeueRequest(0, false))
 	assert.Equal(t, 0, queue.length())
-
-	queue.closeQueue()
-	assert.Panics(t, func() { queue.enqueueRequest(request1) })
 }
 
 func TestPriorityRequestQueue(t *testing.T) {
@@ -57,7 +54,4 @@ func TestPriorityRequestQueue(t *testing.T) {
 	assert.Equal(t, 1, queue.length())
 	assert.Equal(t, request1, queue.dequeueRequest(2, false))
 	assert.Equal(t, 0, queue.length())
-
-	queue.closeQueue()
-	assert.Panics(t, func() { queue.enqueueRequest(request1) })
 }

From a510f77db41339bb5e547a5c1088b241d3c304c7 Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Thu, 9 Nov 2023 11:53:39 -0800
Subject: [PATCH 30/49] Refactor

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 pkg/scheduler/queue/queue.go            |  54 ++++----
 pkg/scheduler/queue/user_queues.go      | 177 ++++++++++++------------
 pkg/scheduler/queue/user_queues_test.go |   6 +-
 3 files changed, 118 insertions(+), 119 deletions(-)

diff --git a/pkg/scheduler/queue/queue.go b/pkg/scheduler/queue/queue.go
index ab0296e678..0502e9dd46 100644
--- a/pkg/scheduler/queue/queue.go
+++ b/pkg/scheduler/queue/queue.go
@@ -62,9 +62,9 @@ type RequestQueue struct {
 	queues  *queues
 	stopped bool
 
-	queueLength       *prometheus.GaugeVec   // Per user and reason.
-	totalRequests     *prometheus.CounterVec // Per user.
-	discardedRequests *prometheus.CounterVec // Per user.
+	queueLength       *prometheus.GaugeVec   // Per user and priority.
+	totalRequests     *prometheus.CounterVec // Per user and priority.
+	discardedRequests *prometheus.CounterVec // Per user and priority.
 }
 
 func NewRequestQueue(maxOutstandingPerTenant int, forgetDelay time.Duration, queueLength *prometheus.GaugeVec, discardedRequests *prometheus.CounterVec, limits Limits, registerer prometheus.Registerer) *RequestQueue {
@@ -98,9 +98,8 @@ func (q *RequestQueue) EnqueueRequest(userID string, req Request, maxQueriers fl
 		return ErrStopped
 	}
 
-	shardSize := util.DynamicShardSize(maxQueriers, len(q.queues.queriers))
-	priorityList, priorityEnabled := q.getPriorityList(userID)
-	queue := q.queues.getOrAddQueue(userID, shardSize, priorityList, priorityEnabled)
+	maxQuerierCount := util.DynamicShardSize(maxQueriers, len(q.queues.queriers))
+	queue := q.queues.getOrAddQueue(userID, maxQuerierCount)
 	maxOutstandingRequests := q.queues.limits.MaxOutstandingPerTenant(userID)
 	priority := strconv.FormatInt(req.Priority(), 10)
 
@@ -109,15 +108,19 @@ func (q *RequestQueue) EnqueueRequest(userID string, req Request, maxQueriers fl
 		return errors.New("no queue found")
 	}
 
-	q.totalRequests.WithLabelValues(userID, priority).Inc()
+	metricLabels := prometheus.Labels{
+		"user":     userID,
+		"priority": priority,
+	}
+	q.totalRequests.With(metricLabels).Inc()
 
 	if queue.length() >= maxOutstandingRequests {
-		q.discardedRequests.WithLabelValues(userID, priority).Inc()
+		q.discardedRequests.With(metricLabels).Inc()
 		return ErrTooManyRequests
 	}
 
 	queue.enqueueRequest(req)
-	q.queueLength.WithLabelValues(userID, priority).Inc()
+	q.queueLength.With(metricLabels).Inc()
 	q.cond.Broadcast()
 	// Call this function while holding a lock. This guarantees that no querier can fetch the request before function returns.
 	if successFn != nil {
@@ -126,24 +129,6 @@ func (q *RequestQueue) EnqueueRequest(userID string, req Request, maxQueriers fl
 	return nil
 }
 
-func (q *RequestQueue) getPriorityList(userID string) ([]int64, bool) {
-	var priorityList []int64
-
-	queryPriority := q.queues.limits.QueryPriority(userID)
-
-	if queryPriority.Enabled {
-		for _, priority := range queryPriority.Priorities {
-			reservedQuerierShardSize := util.DynamicShardSize(priority.ReservedQueriers, len(q.queues.queriers))
-
-			for i := 0; i < reservedQuerierShardSize; i++ {
-				priorityList = append(priorityList, priority.Priority)
-			}
-		}
-	}
-
-	return priorityList, queryPriority.Enabled
-}
-
 // GetNextRequestForQuerier find next user queue and takes the next request off of it. Will block if there are no requests.
 // By passing user index from previous call of this method, querier guarantees that it iterates over all users fairly.
 // If querier finds that request from the user is already expired, it can get a request for the same user by using UserIndex.ReuseLastUser.
@@ -177,7 +162,7 @@ FindQueue:
 
 		// Pick next request from the queue.
 		for {
-			minPriority, checkMinPriority := q.queues.getMinPriority(userID, querierID)
+			minPriority, checkMinPriority := q.getMinPriorityForQuerier(userID, querierID)
 			request := queue.dequeueRequest(minPriority, checkMinPriority)
 			if request == nil {
 				// the queue does not contain request with the min priority, break to wait for more requests
@@ -188,7 +173,11 @@ FindQueue:
 				q.queues.deleteQueue(userID)
 			}
 
-			q.queueLength.WithLabelValues(userID, strconv.FormatInt(request.Priority(), 10)).Dec()
+			metricLabels := prometheus.Labels{
+				"user":     userID,
+				"priority": strconv.FormatInt(request.Priority(), 10),
+			}
+			q.queueLength.With(metricLabels).Dec()
 
 			// Tell close() we've processed a request.
 			q.cond.Broadcast()
@@ -203,6 +192,13 @@ FindQueue:
 	goto FindQueue
 }
 
+func (q *RequestQueue) getMinPriorityForQuerier(userID string, querierID string) (int64, bool) {
+	if priority, ok := q.queues.userQueues[userID].reservedQueriers[querierID]; ok {
+		return priority, true
+	}
+	return 0, false
+}
+
 func (q *RequestQueue) forgetDisconnectedQueriers(_ context.Context) error {
 	q.mtx.Lock()
 	defer q.mtx.Unlock()
diff --git a/pkg/scheduler/queue/user_queues.go b/pkg/scheduler/queue/user_queues.go
index b44c1e5a72..46c0bb598a 100644
--- a/pkg/scheduler/queue/user_queues.go
+++ b/pkg/scheduler/queue/user_queues.go
@@ -65,10 +65,11 @@ type userQueue struct {
 	// If not nil, only these queriers can handle user requests. If nil, all queriers can.
 	// We set this to nil if number of available queriers <= maxQueriers.
 	queriers         map[string]struct{}
+	maxQueriers      int
+	maxOutstanding   int
 	reservedQueriers map[string]int64
 	priorityList     []int64
 	priorityEnabled  bool
-	maxQueriers      int
 
 	// Seed for shuffle sharding of queriers. This seed is based on userID only and is therefore consistent
 	// between different frontends.
@@ -113,7 +114,8 @@ func (q *queues) deleteQueue(userID string) {
 // MaxQueriers is used to compute which queriers should handle requests for this user.
 // If maxQueriers is <= 0, all queriers can handle this user's requests.
 // If maxQueriers has changed since the last call, queriers for this are recomputed.
-func (q *queues) getOrAddQueue(userID string, maxQueriers int, priorityList []int64, priorityEnabled bool) userRequestQueue {
+// It's also responsible to store user configs and update the attributes if related configs have changed.
+func (q *queues) getOrAddQueue(userID string, maxQueriers int) userRequestQueue {
 	// Empty user is not allowed, as that would break our users list ("" is used for free spot).
 	if userID == "" {
 		return nil
@@ -124,75 +126,69 @@ func (q *queues) getOrAddQueue(userID string, maxQueriers int, priorityList []in
 	}
 
 	uq := q.userQueues[userID]
+	priorityEnabled := q.limits.QueryPriority(userID).Enabled
+	maxOutstanding := q.limits.MaxOutstandingPerTenant(userID)
+	priorityList := getPriorityList(q.limits.QueryPriority(userID), len(q.queriers))
 
 	if uq == nil {
-		uq = q.createUserQueue(userID)
-	}
-
-	q.updateUserQueuesAttributes(uq, userID, maxQueriers, priorityList, priorityEnabled)
+		uq = &userQueue{
+			seed:  util.ShuffleShardSeed(userID, ""),
+			index: -1,
+		}
 
-	return uq.queue
-}
+		uq.queue = q.createUserRequestQueue(userID)
+		uq.maxOutstanding = q.limits.MaxOutstandingPerTenant(userID)
+		q.userQueues[userID] = uq
 
-// Finds next queue for the querier. To support fair scheduling between users, client is expected
-// to pass last user index returned by this function as argument. Is there was no previous
-// last user index, use -1.
-func (q *queues) getNextQueueForQuerier(lastUserIndex int, querierID string) (userRequestQueue, string, int) {
-	uid := lastUserIndex
-
-	for iters := 0; iters < len(q.users); iters++ {
-		uid = uid + 1
-
-		// Don't use "mod len(q.users)", as that could skip users at the beginning of the list
-		// for example when q.users has shrunk since last call.
-		if uid >= len(q.users) {
-			uid = 0
+		// Add user to the list of users... find first free spot, and put it there.
+		for ix, u := range q.users {
+			if u == "" {
+				uq.index = ix
+				q.users[ix] = userID
+				break
+			}
 		}
 
-		u := q.users[uid]
-		if u == "" {
-			continue
+		// ... or add to the end.
+		if uq.index < 0 {
+			uq.index = len(q.users)
+			q.users = append(q.users, userID)
 		}
+	} else if (uq.priorityEnabled != priorityEnabled) || (uq.maxOutstanding != maxOutstanding && priorityEnabled) {
+		tmpQueue := q.createUserRequestQueue(userID)
 
-		uq := q.userQueues[u]
-
-		if uq.queriers != nil {
-			if _, ok := uq.queriers[querierID]; !ok {
-				// This querier is not handling the user.
-				continue
-			}
+		// flush to new queue
+		for uq.queue.length() > 0 {
+			tmpQueue.enqueueRequest(uq.queue.dequeueRequest(0, false))
 		}
 
-		return uq.queue, u, uid
+		uq.queue = tmpQueue
+		uq.maxOutstanding = q.limits.MaxOutstandingPerTenant(userID)
 	}
-	return nil, "", uid
-}
 
-func (q *queues) createUserQueue(userID string) *userQueue {
-	uq := &userQueue{
-		seed:  util.ShuffleShardSeed(userID, ""),
-		index: -1,
+	if uq.maxQueriers != maxQueriers {
+		uq.maxQueriers = maxQueriers
+		uq.queriers = shuffleQueriersForUser(uq.seed, maxQueriers, q.sortedQueriers, nil)
 	}
 
-	uq.queue = q.createUserRequestQueue(userID)
-	q.userQueues[userID] = uq
+	if priorityEnabled && !reflect.DeepEqual(uq.priorityList, priorityList) {
+		reservedQueriers := make(map[string]int64)
 
-	// Add user to the list of users... find first free spot, and put it there.
-	for ix, u := range q.users {
-		if u == "" {
-			uq.index = ix
-			q.users[ix] = userID
-			break
+		i := 0
+		for querierID := range uq.queriers {
+			if i == len(priorityList) {
+				break
+			}
+			reservedQueriers[querierID] = priorityList[i]
+			i++
 		}
-	}
 
-	// ... or add to the end.
-	if uq.index < 0 {
-		uq.index = len(q.users)
-		q.users = append(q.users, userID)
+		uq.reservedQueriers = reservedQueriers
+		uq.priorityList = priorityList
+		uq.priorityEnabled = priorityEnabled
 	}
 
-	return uq
+	return uq.queue
 }
 
 func (q *queues) createUserRequestQueue(userID string) userRequestQueue {
@@ -200,10 +196,6 @@ func (q *queues) createUserRequestQueue(userID string) userRequestQueue {
 		return NewPriorityRequestQueue(util.NewPriorityQueue(nil))
 	}
 
-	return NewFIFORequestQueue(make(chan Request, q.getQueueSize(userID)))
-}
-
-func (q *queues) getQueueSize(userID string) int {
 	queueSize := q.limits.MaxOutstandingPerTenant(userID)
 
 	// 0 is the default value of the flag. If the old flag is set
@@ -212,48 +204,41 @@ func (q *queues) getQueueSize(userID string) int {
 		queueSize = q.maxUserQueueSize
 	}
 
-	return queueSize
+	return NewFIFORequestQueue(make(chan Request, queueSize))
 }
 
-func (q *queues) updateUserQueuesAttributes(uq *userQueue, userID string, maxQueriers int, priorityList []int64, priorityEnabled bool) {
-	if uq.maxQueriers != maxQueriers {
-		uq.maxQueriers = maxQueriers
-		uq.queriers = shuffleQueriersForUser(uq.seed, maxQueriers, q.sortedQueriers, nil)
-	}
+// Finds next queue for the querier. To support fair scheduling between users, client is expected
+// to pass last user index returned by this function as argument. Is there was no previous
+// last user index, use -1.
+func (q *queues) getNextQueueForQuerier(lastUserIndex int, querierID string) (userRequestQueue, string, int) {
+	uid := lastUserIndex
 
-	// if query priority is newly enabled/disabled, transfer the requests to the new queue
-	if uq.priorityEnabled != priorityEnabled {
-		tmpQueue := q.createUserRequestQueue(userID)
+	for iters := 0; iters < len(q.users); iters++ {
+		uid = uid + 1
 
-		for uq.queue.length() > 0 {
-			tmpQueue.enqueueRequest(uq.queue.dequeueRequest(0, false))
+		// Don't use "mod len(q.users)", as that could skip users at the beginning of the list
+		// for example when q.users has shrunk since last call.
+		if uid >= len(q.users) {
+			uid = 0
 		}
-		uq.queue = tmpQueue
-	}
 
-	if priorityEnabled && !reflect.DeepEqual(uq.priorityList, priorityList) {
-		reservedQueriers := make(map[string]int64)
+		u := q.users[uid]
+		if u == "" {
+			continue
+		}
 
-		i := 0
-		for querierID := range uq.queriers {
-			if i == len(priorityList) {
-				break
+		uq := q.userQueues[u]
+
+		if uq.queriers != nil {
+			if _, ok := uq.queriers[querierID]; !ok {
+				// This querier is not handling the user.
+				continue
 			}
-			reservedQueriers[querierID] = priorityList[i]
-			i++
 		}
 
-		uq.reservedQueriers = reservedQueriers
-		uq.priorityList = priorityList
-		uq.priorityEnabled = priorityEnabled
-	}
-}
-
-func (q *queues) getMinPriority(userID string, querierID string) (int64, bool) {
-	if priority, ok := q.userQueues[userID].reservedQueriers[querierID]; ok {
-		return priority, true
+		return uq.queue, u, uid
 	}
-	return 0, false
+	return nil, "", uid
 }
 
 func (q *queues) addQuerierConnection(querierID string) {
@@ -388,6 +373,24 @@ func shuffleQueriersForUser(userSeed int64, queriersToSelect int, allSortedQueri
 	return queriers
 }
 
+// getPriorityList returns a list of priorities, each priority repeated as much as number of reserved queriers.
+// This is used when creating map of reserved queriers.
+func getPriorityList(queryPriority validation.QueryPriority, totalQuerierCount int) []int64 {
+	var priorityList []int64
+
+	if queryPriority.Enabled {
+		for _, priority := range queryPriority.Priorities {
+			reservedQuerierShardSize := util.DynamicShardSize(priority.ReservedQueriers, totalQuerierCount)
+
+			for i := 0; i < reservedQuerierShardSize; i++ {
+				priorityList = append(priorityList, priority.Priority)
+			}
+		}
+	}
+
+	return priorityList
+}
+
 // MockLimits implements the Limits interface. Used in tests only.
 type MockLimits struct {
 	MaxOutstanding     int
diff --git a/pkg/scheduler/queue/user_queues_test.go b/pkg/scheduler/queue/user_queues_test.go
index c6c6e8cfc2..235f1cb187 100644
--- a/pkg/scheduler/queue/user_queues_test.go
+++ b/pkg/scheduler/queue/user_queues_test.go
@@ -158,7 +158,7 @@ func TestQueuesConsistency(t *testing.T) {
 			for i := 0; i < 10000; i++ {
 				switch r.Int() % 6 {
 				case 0:
-					assert.NotNil(t, uq.getOrAddQueue(generateTenant(r), 3, []int64{}, false))
+					assert.NotNil(t, uq.getOrAddQueue(generateTenant(r), 3))
 				case 1:
 					qid := generateQuerier(r)
 					_, _, luid := uq.getNextQueueForQuerier(lastUserIndexes[qid], qid)
@@ -360,10 +360,10 @@ func generateQuerier(r *rand.Rand) string {
 }
 
 func getOrAdd(t *testing.T, uq *queues, tenant string, maxQueriers int) userRequestQueue {
-	q := uq.getOrAddQueue(tenant, maxQueriers, []int64{}, true)
+	q := uq.getOrAddQueue(tenant, maxQueriers)
 	assert.NotNil(t, q)
 	assert.NoError(t, isConsistent(uq))
-	assert.Equal(t, q, uq.getOrAddQueue(tenant, maxQueriers, []int64{}, true))
+	assert.Equal(t, q, uq.getOrAddQueue(tenant, maxQueriers))
 	return q
 }
 

From 717d746d5b99a354f6fd5deef917a82311354a1b Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Fri, 10 Nov 2023 10:26:46 -0800
Subject: [PATCH 31/49] Add more tests

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 pkg/frontend/v1/frontend.go                   |   2 +-
 pkg/scheduler/queue/queue.go                  |  20 ++-
 pkg/scheduler/queue/queue_test.go             | 147 ++++++++++++------
 pkg/scheduler/queue/user_queues.go            |  24 +--
 pkg/scheduler/queue/user_queues_test.go       | 111 +++++++++++++
 .../queue/user_request_queue_test.go          |   1 +
 pkg/scheduler/scheduler.go                    |   2 +-
 pkg/util/priority_queue.go                    |   2 +-
 pkg/util/priority_queue_test.go               |  18 +++
 pkg/util/query/priority_test.go               |  36 +++++
 10 files changed, 301 insertions(+), 62 deletions(-)

diff --git a/pkg/frontend/v1/frontend.go b/pkg/frontend/v1/frontend.go
index dbe5c28ae1..de5d2d0647 100644
--- a/pkg/frontend/v1/frontend.go
+++ b/pkg/frontend/v1/frontend.go
@@ -119,7 +119,7 @@ func New(cfg Config, limits Limits, log log.Logger, registerer prometheus.Regist
 		queueLength: promauto.With(registerer).NewGaugeVec(prometheus.GaugeOpts{
 			Name: "cortex_query_frontend_queue_length",
 			Help: "Number of queries in the queue.",
-		}, []string{"user", "priority"}),
+		}, []string{"user", "priority", "type"}),
 		discardedRequests: promauto.With(registerer).NewCounterVec(prometheus.CounterOpts{
 			Name: "cortex_query_frontend_discarded_requests_total",
 			Help: "Total number of query requests discarded.",
diff --git a/pkg/scheduler/queue/queue.go b/pkg/scheduler/queue/queue.go
index 0502e9dd46..aa29e061f8 100644
--- a/pkg/scheduler/queue/queue.go
+++ b/pkg/scheduler/queue/queue.go
@@ -62,7 +62,7 @@ type RequestQueue struct {
 	queues  *queues
 	stopped bool
 
-	queueLength       *prometheus.GaugeVec   // Per user and priority.
+	queueLength       *prometheus.GaugeVec   // Per user, priority and type of the queue (fifo or priority).
 	totalRequests     *prometheus.CounterVec // Per user and priority.
 	discardedRequests *prometheus.CounterVec // Per user and priority.
 }
@@ -108,6 +108,10 @@ func (q *RequestQueue) EnqueueRequest(userID string, req Request, maxQueriers fl
 		return errors.New("no queue found")
 	}
 
+	queueType := "fifo"
+	if q.queues.limits.QueryPriority(userID).Enabled {
+		queueType = "priority"
+	}
 	metricLabels := prometheus.Labels{
 		"user":     userID,
 		"priority": priority,
@@ -120,7 +124,11 @@ func (q *RequestQueue) EnqueueRequest(userID string, req Request, maxQueriers fl
 	}
 
 	queue.enqueueRequest(req)
-	q.queueLength.With(metricLabels).Inc()
+	q.queueLength.With(prometheus.Labels{
+		"user":     userID,
+		"priority": priority,
+		"type":     queueType,
+	}).Inc()
 	q.cond.Broadcast()
 	// Call this function while holding a lock. This guarantees that no querier can fetch the request before function returns.
 	if successFn != nil {
@@ -166,16 +174,22 @@ FindQueue:
 			request := queue.dequeueRequest(minPriority, checkMinPriority)
 			if request == nil {
 				// the queue does not contain request with the min priority, break to wait for more requests
-				break
+				querierWait = true
+				goto FindQueue
 			}
 
 			if queue.length() == 0 {
 				q.queues.deleteQueue(userID)
 			}
 
+			queueType := "fifo"
+			if q.queues.limits.QueryPriority(userID).Enabled {
+				queueType = "priority"
+			}
 			metricLabels := prometheus.Labels{
 				"user":     userID,
 				"priority": strconv.FormatInt(request.Priority(), 10),
+				"type":     queueType,
 			}
 			q.queueLength.With(metricLabels).Dec()
 
diff --git a/pkg/scheduler/queue/queue_test.go b/pkg/scheduler/queue/queue_test.go
index b2de565db3..02317338d0 100644
--- a/pkg/scheduler/queue/queue_test.go
+++ b/pkg/scheduler/queue/queue_test.go
@@ -25,8 +25,8 @@ func BenchmarkGetNextRequest(b *testing.B) {
 
 	for n := 0; n < b.N; n++ {
 		queue := NewRequestQueue(maxOutstandingPerTenant, 0,
-			prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user"}),
-			prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user"}),
+			prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user", "priority", "type"}),
+			prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user", "priority"}),
 			MockLimits{MaxOutstanding: 100},
 			nil,
 		)
@@ -119,8 +119,8 @@ func TestRequestQueue_GetNextRequestForQuerier_ShouldGetRequestAfterReshardingBe
 	const forgetDelay = 3 * time.Second
 
 	queue := NewRequestQueue(1, forgetDelay,
-		prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user"}),
-		prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user"}),
+		prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user", "priority", "type"}),
+		prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user", "priority"}),
 		MockLimits{MaxOutstanding: 100},
 		nil,
 	)
@@ -160,11 +160,11 @@ func TestRequestQueue_GetNextRequestForQuerier_ShouldGetRequestAfterReshardingBe
 	assert.GreaterOrEqual(t, waitTime.Milliseconds(), forgetDelay.Milliseconds())
 }
 
-func TestRequestQueue_QueriersShouldGetHighPriorityQueryFirst(t *testing.T) {
+func TestQueriersShouldGetHighPriorityQueryFirst(t *testing.T) {
 	queue := NewRequestQueue(0, 0,
-		prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user"}),
-		prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user"}),
-		MockLimits{MaxOutstanding: 3, queryPriority: validation.QueryPriority{Enabled: true}},
+		prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user", "priority", "type"}),
+		prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user", "priority"}),
+		MockLimits{MaxOutstanding: 3, QueryPriorityVal: validation.QueryPriority{Enabled: true}},
 		nil,
 	)
 	ctx := context.Background()
@@ -190,47 +190,100 @@ func TestRequestQueue_QueriersShouldGetHighPriorityQueryFirst(t *testing.T) {
 	assert.Equal(t, highPriorityRequest, nextRequest) // high priority request returned, although it was enqueued the last
 }
 
-func TestRequestQueue_ReservedQueriersShouldOnlyGetHighPriorityQueries(t *testing.T) {
-	// TODO: justinjung04
-	//queue := NewRequestQueue(0, 0,
-	//	prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user"}),
-	//	prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user"}),
-	//	MockLimits{MaxOutstanding: 3},
-	//	nil,
-	//)
-	//ctx := context.Background()
-	//
-	//queue.RegisterQuerierConnection("querier-1")
-	//
-	//normalRequest := MockRequest{
-	//	id:             "normal query",
-	//	isHighPriority: false,
-	//}
-	//highPriorityRequest := MockRequest{
-	//	id:             "high priority query",
-	//	isHighPriority: true,
-	//}
-	//
-	//assert.NoError(t, queue.EnqueueRequest("userID", normalRequest, 1, func() {}))
-	//assert.NoError(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {}))
-	//
-	//nextRequest, _, _ := queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1")
-	//assert.Equal(t, highPriorityRequest, nextRequest)
-	//
-	//ctxTimeout, cancel := context.WithTimeout(ctx, 1*time.Second)
-	//defer cancel()
-	//
-	//time.AfterFunc(2*time.Second, func() {
-	//	queue.cond.Broadcast()
-	//})
-	//nextRequest, _, _ = queue.GetNextRequestForQuerier(ctxTimeout, FirstUser(), "querier-1")
-	//assert.Nil(t, nextRequest)
-	//assert.Equal(t, 1, queue.queues.getTotalQueueSize("userID"))
+func TestReservedQueriersShouldOnlyGetHighPriorityQueries(t *testing.T) {
+	queue := NewRequestQueue(0, 0,
+		prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user", "priority", "type"}),
+		prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user", "priority"}),
+		MockLimits{
+			MaxOutstanding: 3,
+			QueryPriorityVal: validation.QueryPriority{
+				Enabled: true,
+				Priorities: []validation.PriorityDef{
+					{
+						Priority:         1,
+						ReservedQueriers: 1,
+					},
+				},
+			},
+		},
+		nil,
+	)
+	ctx := context.Background()
+
+	queue.RegisterQuerierConnection("querier-1")
+
+	normalRequest := MockRequest{
+		id: "normal query",
+	}
+	priority1Request := MockRequest{
+		id:       "priority 1",
+		priority: 1,
+	}
+	priority2Request := MockRequest{
+		id:       "priority 2",
+		priority: 2,
+	}
+
+	assert.NoError(t, queue.EnqueueRequest("userID", normalRequest, 1, func() {}))
+	assert.NoError(t, queue.EnqueueRequest("userID", priority1Request, 1, func() {}))
+	assert.NoError(t, queue.EnqueueRequest("userID", priority2Request, 1, func() {}))
+
+	nextRequest, _, _ := queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1")
+	assert.Equal(t, priority2Request, nextRequest)
+
+	nextRequest, _, _ = queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1")
+	assert.Equal(t, priority1Request, nextRequest)
+
+	ctxTimeout, cancel := context.WithTimeout(ctx, 1*time.Second)
+	defer cancel()
+
+	time.AfterFunc(2*time.Second, func() {
+		queue.cond.Broadcast()
+	})
+	nextRequest, _, _ = queue.GetNextRequestForQuerier(ctxTimeout, FirstUser(), "querier-1")
+	assert.Nil(t, nextRequest)
+	assert.Equal(t, 1, queue.queues.userQueues["userID"].queue.length())
 }
 
-func TestRequestQueue_EnqueueRequest(t *testing.T) {
-	// TODO: justinjung04
-	// check updated limit as well
+func TestExitingRequestsShouldPersistEvenIfTheConfigHasChanged(t *testing.T) {
+	limits := MockLimits{
+		MaxOutstanding: 3,
+	}
+	queue := NewRequestQueue(0, 0,
+		prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user", "priority", "type"}),
+		prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user", "priority"}),
+		limits,
+		nil,
+	)
+
+	ctx := context.Background()
+
+	queue.RegisterQuerierConnection("querier-1")
+
+	normalRequest := MockRequest{
+		id: "normal query",
+	}
+	highPriorityRequest := MockRequest{
+		id:       "high priority query",
+		priority: 1,
+	}
+
+	assert.NoError(t, queue.EnqueueRequest("userID", normalRequest, 1, func() {}))
+	assert.NoError(t, queue.EnqueueRequest("userID", highPriorityRequest, 1, func() {}))
+	assert.NoError(t, queue.EnqueueRequest("userID", normalRequest, 1, func() {}))
+
+	limits.MaxOutstanding = 4
+	limits.QueryPriorityVal = validation.QueryPriority{Enabled: true}
+	queue.queues.limits = limits
+
+	assert.NoError(t, queue.EnqueueRequest("userID", normalRequest, 1, func() {}))
+
+	nextRequest, _, _ := queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1")
+	assert.Equal(t, highPriorityRequest, nextRequest)
+
+	nextRequest, _, _ = queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1")
+	assert.Equal(t, normalRequest, nextRequest)
+	assert.Equal(t, 2, queue.queues.userQueues["userID"].queue.length())
 }
 
 type MockRequest struct {
diff --git a/pkg/scheduler/queue/user_queues.go b/pkg/scheduler/queue/user_queues.go
index 46c0bb598a..816f87aae8 100644
--- a/pkg/scheduler/queue/user_queues.go
+++ b/pkg/scheduler/queue/user_queues.go
@@ -154,7 +154,7 @@ func (q *queues) getOrAddQueue(userID string, maxQueriers int) userRequestQueue
 			uq.index = len(q.users)
 			q.users = append(q.users, userID)
 		}
-	} else if (uq.priorityEnabled != priorityEnabled) || (uq.maxOutstanding != maxOutstanding && priorityEnabled) {
+	} else if (uq.priorityEnabled != priorityEnabled) || (!priorityEnabled && uq.maxOutstanding != maxOutstanding) {
 		tmpQueue := q.createUserRequestQueue(userID)
 
 		// flush to new queue
@@ -175,12 +175,14 @@ func (q *queues) getOrAddQueue(userID string, maxQueriers int) userRequestQueue
 		reservedQueriers := make(map[string]int64)
 
 		i := 0
-		for querierID := range uq.queriers {
+		for _, querierID := range q.sortedQueriers {
 			if i == len(priorityList) {
 				break
 			}
-			reservedQueriers[querierID] = priorityList[i]
-			i++
+			if _, ok := uq.queriers[querierID]; ok || uq.queriers == nil {
+				reservedQueriers[querierID] = priorityList[i]
+				i++
+			}
 		}
 
 		uq.reservedQueriers = reservedQueriers
@@ -388,18 +390,22 @@ func getPriorityList(queryPriority validation.QueryPriority, totalQuerierCount i
 		}
 	}
 
+	if len(priorityList) > totalQuerierCount {
+		return []int64{}
+	}
+
 	return priorityList
 }
 
 // MockLimits implements the Limits interface. Used in tests only.
 type MockLimits struct {
-	MaxOutstanding     int
-	maxQueriersPerUser float64
-	queryPriority      validation.QueryPriority
+	MaxOutstanding        int
+	MaxQueriersPerUserVal float64
+	QueryPriorityVal      validation.QueryPriority
 }
 
 func (l MockLimits) MaxQueriersPerUser(_ string) float64 {
-	return l.maxQueriersPerUser
+	return l.MaxQueriersPerUserVal
 }
 
 func (l MockLimits) MaxOutstandingPerTenant(_ string) int {
@@ -407,5 +413,5 @@ func (l MockLimits) MaxOutstandingPerTenant(_ string) int {
 }
 
 func (l MockLimits) QueryPriority(_ string) validation.QueryPriority {
-	return l.queryPriority
+	return l.QueryPriorityVal
 }
diff --git a/pkg/scheduler/queue/user_queues_test.go b/pkg/scheduler/queue/user_queues_test.go
index 235f1cb187..3f0495b9ce 100644
--- a/pkg/scheduler/queue/user_queues_test.go
+++ b/pkg/scheduler/queue/user_queues_test.go
@@ -2,6 +2,7 @@ package queue
 
 import (
 	"fmt"
+	"github.com/cortexproject/cortex/pkg/util/validation"
 	"math"
 	"math/rand"
 	"sort"
@@ -351,6 +352,69 @@ func TestQueues_ForgetDelay_ShouldCorrectlyHandleQuerierReconnectingBeforeForget
 	}
 }
 
+func TestGetQueue(t *testing.T) {
+	limits := MockLimits{
+		MaxOutstanding: 3,
+	}
+	q := newUserQueues(0, 0, limits)
+	q.addQuerierConnection("q-1")
+	q.addQuerierConnection("q-2")
+	q.addQuerierConnection("q-3")
+	q.addQuerierConnection("q-4")
+	q.addQuerierConnection("q-5")
+
+	queue := q.getOrAddQueue("userID", 2)
+	queue.enqueueRequest(MockRequest{})
+
+	assert.NotNil(t, q.userQueues["userID"])
+	assert.Equal(t, 3, q.userQueues["userID"].maxOutstanding)
+	assert.Equal(t, 2, q.userQueues["userID"].maxQueriers)
+	assert.Equal(t, 5, len(q.queriers))
+	assert.Equal(t, 2, len(q.userQueues["userID"].queriers))
+	assert.Subset(t, getKeys(q.queriers), getKeys(q.userQueues["userID"].queriers))
+	assert.IsType(t, &FIFORequestQueue{}, queue)
+	assert.Equal(t, 1, queue.length())
+
+	limits.MaxOutstanding = 10
+	q.limits = limits
+	queue = q.getOrAddQueue("userID", 0)
+
+	assert.Equal(t, 10, q.userQueues["userID"].maxOutstanding)
+	assert.Equal(t, 0, q.userQueues["userID"].maxQueriers)
+	assert.Nil(t, q.userQueues["userID"].queriers)
+	assert.IsType(t, &FIFORequestQueue{}, queue)
+	assert.Equal(t, 1, queue.length())
+
+	limits.QueryPriorityVal = validation.QueryPriority{Enabled: true, Priorities: []validation.PriorityDef{
+		{
+			Priority:         1,
+			ReservedQueriers: 2,
+		},
+	}}
+	q.limits = limits
+	queue = q.getOrAddQueue("userID", 3)
+
+	assert.Equal(t, 10, q.userQueues["userID"].maxOutstanding)
+	assert.Equal(t, 3, q.userQueues["userID"].maxQueriers)
+	assert.Equal(t, 5, len(q.queriers))
+	assert.Equal(t, 3, len(q.userQueues["userID"].queriers))
+	assert.IsType(t, &PriorityRequestQueue{}, queue)
+	assert.Equal(t, 1, queue.length())
+	assert.ElementsMatch(t, []int64{1, 1}, q.userQueues["userID"].priorityList)
+	assert.True(t, q.userQueues["userID"].priorityEnabled)
+	assert.Subset(t, getKeys(q.queriers), getKeys(q.userQueues["userID"].queriers))
+	assert.Subset(t, getKeys(q.userQueues["userID"].queriers), getKeys(q.userQueues["userID"].reservedQueriers))
+
+	// check the queriers and reservedQueriers map are consistent
+	for i := 0; i < 10; i++ {
+		queriers := q.userQueues["userID"].queriers
+		reservedQueriers := q.userQueues["userID"].reservedQueriers
+		queue = q.getOrAddQueue("userID", 3)
+		assert.Equal(t, queriers, q.userQueues["userID"].queriers)
+		assert.Equal(t, reservedQueriers, q.userQueues["userID"].reservedQueriers)
+	}
+}
+
 func generateTenant(r *rand.Rand) string {
 	return fmt.Sprint("tenant-", r.Int()%5)
 }
@@ -437,6 +501,27 @@ func getUsersByQuerier(queues *queues, querierID string) []string {
 	return userIDs
 }
 
+func getKeys(x interface{}) []string {
+	var keys []string
+
+	switch i := x.(type) {
+	case map[string]struct{}:
+		for k := range i {
+			keys = append(keys, k)
+		}
+	case map[string]*querier:
+		for k := range i {
+			keys = append(keys, k)
+		}
+	case map[string]int64:
+		for k := range i {
+			keys = append(keys, k)
+		}
+	}
+
+	return keys
+}
+
 func TestShuffleQueriers(t *testing.T) {
 	allQueriers := []string{"a", "b", "c", "d", "e"}
 
@@ -552,3 +637,29 @@ func TestShuffleQueriers_WithReservedQueriers_Correctness(t *testing.T) {
 	//	prevReservedQueriers = reservedQueriers
 	//}
 }
+
+func TestGetPriorityList(t *testing.T) {
+	queryPriority := validation.QueryPriority{
+		Enabled: true,
+		Priorities: []validation.PriorityDef{
+			{
+				Priority:         1,
+				ReservedQueriers: 2,
+			},
+			{
+				Priority:         2,
+				ReservedQueriers: 3,
+			},
+		},
+	}
+
+	assert.EqualValues(t, []int64{1, 1, 2, 2, 2}, getPriorityList(queryPriority, 10))
+	assert.EqualValues(t, []int64{}, getPriorityList(queryPriority, 1))
+
+	queryPriority.Priorities[0].ReservedQueriers = 0.4
+	queryPriority.Priorities[1].ReservedQueriers = 0.6
+	assert.EqualValues(t, []int64{1, 1, 1, 1, 2, 2, 2, 2, 2, 2}, getPriorityList(queryPriority, 10))
+
+	queryPriority.Enabled = false
+	assert.Nil(t, getPriorityList(queryPriority, 10))
+}
diff --git a/pkg/scheduler/queue/user_request_queue_test.go b/pkg/scheduler/queue/user_request_queue_test.go
index 6001f8966c..aeaa88440d 100644
--- a/pkg/scheduler/queue/user_request_queue_test.go
+++ b/pkg/scheduler/queue/user_request_queue_test.go
@@ -52,6 +52,7 @@ func TestPriorityRequestQueue(t *testing.T) {
 	assert.Equal(t, 2, queue.length())
 	assert.Equal(t, request2, queue.dequeueRequest(2, true))
 	assert.Equal(t, 1, queue.length())
+	assert.Nil(t, queue.dequeueRequest(2, true))
 	assert.Equal(t, request1, queue.dequeueRequest(2, false))
 	assert.Equal(t, 0, queue.length())
 }
diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go
index b96b659e4b..65235540a3 100644
--- a/pkg/scheduler/scheduler.go
+++ b/pkg/scheduler/scheduler.go
@@ -105,7 +105,7 @@ func NewScheduler(cfg Config, limits Limits, log log.Logger, registerer promethe
 	s.queueLength = promauto.With(registerer).NewGaugeVec(prometheus.GaugeOpts{
 		Name: "cortex_query_scheduler_queue_length",
 		Help: "Number of queries in the queue.",
-	}, []string{"user", "priority"})
+	}, []string{"user", "priority", "type"})
 
 	s.discardedRequests = promauto.With(registerer).NewCounterVec(prometheus.CounterOpts{
 		Name: "cortex_query_scheduler_discarded_requests_total",
diff --git a/pkg/util/priority_queue.go b/pkg/util/priority_queue.go
index 543192e126..9937c231c9 100644
--- a/pkg/util/priority_queue.go
+++ b/pkg/util/priority_queue.go
@@ -86,7 +86,7 @@ func (pq *PriorityQueue) Enqueue(op PriorityOp) {
 	pq.lock.Lock()
 	defer pq.lock.Unlock()
 
-	if pq.closed {
+	if pq.closing || pq.closed {
 		panic("enqueue on closed queue")
 	}
 
diff --git a/pkg/util/priority_queue_test.go b/pkg/util/priority_queue_test.go
index d7ee8bc202..168ed96137 100644
--- a/pkg/util/priority_queue_test.go
+++ b/pkg/util/priority_queue_test.go
@@ -33,9 +33,11 @@ func TestPriorityQueuePriorities(t *testing.T) {
 	queue := NewPriorityQueue(nil)
 	queue.Enqueue(simpleItem(1))
 	queue.Enqueue(simpleItem(2))
+	queue.Enqueue(simpleItem(2))
 
 	assert.Equal(t, simpleItem(2), queue.Peek().(simpleItem), "Expected to peek simpleItem(2)")
 	assert.Equal(t, simpleItem(2), queue.Dequeue().(simpleItem), "Expected to dequeue simpleItem(2)")
+	assert.Equal(t, simpleItem(2), queue.Dequeue().(simpleItem), "Expected to dequeue simpleItem(2)")
 	assert.Equal(t, simpleItem(1), queue.Dequeue().(simpleItem), "Expected to dequeue simpleItem(1)")
 
 	queue.Close()
@@ -46,9 +48,11 @@ func TestPriorityQueuePriorities2(t *testing.T) {
 	queue := NewPriorityQueue(nil)
 	queue.Enqueue(simpleItem(2))
 	queue.Enqueue(simpleItem(1))
+	queue.Enqueue(simpleItem(2))
 
 	assert.Equal(t, simpleItem(2), queue.Peek().(simpleItem), "Expected to peek simpleItem(2)")
 	assert.Equal(t, simpleItem(2), queue.Dequeue().(simpleItem), "Expected to dequeue simpleItem(2)")
+	assert.Equal(t, simpleItem(2), queue.Dequeue().(simpleItem), "Expected to dequeue simpleItem(2)")
 	assert.Equal(t, simpleItem(1), queue.Dequeue().(simpleItem), "Expected to dequeue simpleItem(1)")
 
 	queue.Close()
@@ -72,3 +76,17 @@ func TestPriorityQueueWait(t *testing.T) {
 		t.Fatal("Close didn't unblock Dequeue.")
 	}
 }
+
+func TestPriorityQueueClose(t *testing.T) {
+	queue := NewPriorityQueue(nil)
+	queue.Enqueue(simpleItem(1))
+	queue.Close()
+	assert.Panics(t, func() { queue.Enqueue(simpleItem(2)) })
+	assert.Equal(t, simpleItem(1), queue.Dequeue().(simpleItem))
+
+	queue = NewPriorityQueue(nil)
+	queue.Enqueue(simpleItem(1))
+	queue.DiscardAndClose()
+	assert.Panics(t, func() { queue.Enqueue(simpleItem(2)) })
+	assert.Nil(t, queue.Dequeue())
+}
diff --git a/pkg/util/query/priority_test.go b/pkg/util/query/priority_test.go
index 0d1193a2f3..f1c9f4d185 100644
--- a/pkg/util/query/priority_test.go
+++ b/pkg/util/query/priority_test.go
@@ -116,6 +116,42 @@ func Test_GetPriorityShouldConsiderRegex(t *testing.T) {
 	}, now, &queryPriority, false))
 }
 
+func Test_GetPriorityShouldNotRecompileRegexIfQueryPriorityChangedIsTrue(t *testing.T) {
+	now := time.Now()
+	priorities := []validation.PriorityDef{
+		{
+			Priority: 1,
+			QueryAttributes: []validation.QueryAttribute{
+				{
+					Regex:     "sum",
+					StartTime: 2 * time.Hour,
+					EndTime:   0 * time.Hour,
+				},
+			},
+		},
+	}
+	queryPriority := validation.QueryPriority{
+		Enabled:    true,
+		Priorities: priorities,
+	}
+
+	assert.Equal(t, int64(1), GetPriority(url.Values{
+		"query": []string{"sum(up)"},
+		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
+	}, now, &queryPriority, true))
+
+	queryPriority.Priorities[0].QueryAttributes[0].Regex = "count"
+
+	assert.Equal(t, int64(0), GetPriority(url.Values{
+		"query": []string{"count(up)"},
+		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
+	}, now, &queryPriority, false))
+	assert.Equal(t, int64(1), GetPriority(url.Values{
+		"query": []string{"count(up)"},
+		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
+	}, now, &queryPriority, true))
+}
+
 func Test_GetPriorityShouldConsiderStartAndEndTime(t *testing.T) {
 	now := time.Now()
 	priorities := []validation.PriorityDef{

From f3a977292d128e4937b41a957e4f75cb1bb2664b Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Fri, 10 Nov 2023 11:01:28 -0800
Subject: [PATCH 32/49] Lint

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 pkg/frontend/v1/frontend_test.go        |  2 +-
 pkg/scheduler/queue/user_queues_test.go | 12 ++++++++----
 pkg/scheduler/scheduler_test.go         |  6 +++---
 3 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/pkg/frontend/v1/frontend_test.go b/pkg/frontend/v1/frontend_test.go
index 316c8c25fe..1206969a93 100644
--- a/pkg/frontend/v1/frontend_test.go
+++ b/pkg/frontend/v1/frontend_test.go
@@ -211,7 +211,7 @@ func TestFrontendMetricsCleanup(t *testing.T) {
 			require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(`
 				# HELP cortex_query_frontend_queue_length Number of queries in the queue.
 				# TYPE cortex_query_frontend_queue_length gauge
-				cortex_query_frontend_queue_length{priority="0",user="1"} 0
+				cortex_query_frontend_queue_length{priority="0",type="fifo",user="1"} 0
 			`), "cortex_query_frontend_queue_length"))
 
 			fr.cleanupInactiveUserMetrics("1")
diff --git a/pkg/scheduler/queue/user_queues_test.go b/pkg/scheduler/queue/user_queues_test.go
index 3f0495b9ce..df4fce2b86 100644
--- a/pkg/scheduler/queue/user_queues_test.go
+++ b/pkg/scheduler/queue/user_queues_test.go
@@ -2,7 +2,6 @@ package queue
 
 import (
 	"fmt"
-	"github.com/cortexproject/cortex/pkg/util/validation"
 	"math"
 	"math/rand"
 	"sort"
@@ -11,6 +10,8 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	"github.com/cortexproject/cortex/pkg/util/validation"
 )
 
 func TestQueues(t *testing.T) {
@@ -352,7 +353,7 @@ func TestQueues_ForgetDelay_ShouldCorrectlyHandleQuerierReconnectingBeforeForget
 	}
 }
 
-func TestGetQueue(t *testing.T) {
+func TestGetOrAddQueueShouldUpdateProperties(t *testing.T) {
 	limits := MockLimits{
 		MaxOutstanding: 3,
 	}
@@ -406,10 +407,13 @@ func TestGetQueue(t *testing.T) {
 	assert.Subset(t, getKeys(q.userQueues["userID"].queriers), getKeys(q.userQueues["userID"].reservedQueriers))
 
 	// check the queriers and reservedQueriers map are consistent
-	for i := 0; i < 10; i++ {
+	for i := 0; i < 100; i++ {
 		queriers := q.userQueues["userID"].queriers
 		reservedQueriers := q.userQueues["userID"].reservedQueriers
-		queue = q.getOrAddQueue("userID", 3)
+		q.userQueues["userID"].maxQueriers = 0          // reset to trigger querier assignment
+		q.userQueues["userID"].priorityList = []int64{} // reset to trigger reserved querier assignment
+		_ = q.getOrAddQueue("userID", 3)
+
 		assert.Equal(t, queriers, q.userQueues["userID"].queriers)
 		assert.Equal(t, reservedQueriers, q.userQueues["userID"].reservedQueriers)
 	}
diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go
index b055754133..798843284e 100644
--- a/pkg/scheduler/scheduler_test.go
+++ b/pkg/scheduler/scheduler_test.go
@@ -430,8 +430,8 @@ func TestSchedulerMetrics(t *testing.T) {
 	require.NoError(t, promtest.GatherAndCompare(reg, strings.NewReader(`
 		# HELP cortex_query_scheduler_queue_length Number of queries in the queue.
 		# TYPE cortex_query_scheduler_queue_length gauge
-		cortex_query_scheduler_queue_length{priority="0",user="another"} 1
-		cortex_query_scheduler_queue_length{priority="0",user="test"} 1
+		cortex_query_scheduler_queue_length{priority="0",type="fifo",user="another"} 1
+		cortex_query_scheduler_queue_length{priority="0",type="fifo",user="test"} 1
 	`), "cortex_query_scheduler_queue_length"))
 
 	scheduler.cleanupMetricsForInactiveUser("test")
@@ -439,7 +439,7 @@ func TestSchedulerMetrics(t *testing.T) {
 	require.NoError(t, promtest.GatherAndCompare(reg, strings.NewReader(`
 		# HELP cortex_query_scheduler_queue_length Number of queries in the queue.
 		# TYPE cortex_query_scheduler_queue_length gauge
-		cortex_query_scheduler_queue_length{priority="0",user="another"} 1
+		cortex_query_scheduler_queue_length{priority="0",type="fifo",user="another"} 1
 	`), "cortex_query_scheduler_queue_length"))
 }
 

From 87c9a0ed6608fcc4da589dd9b660fc3d0d4bb6f9 Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Fri, 10 Nov 2023 12:15:52 -0800
Subject: [PATCH 33/49] Update doc

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 docs/configuration/config-file-reference.md | 9 +++------
 pkg/util/validation/limits.go               | 6 +++---
 2 files changed, 6 insertions(+), 9 deletions(-)

diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md
index 05b2b67103..c4765c2948 100644
--- a/docs/configuration/config-file-reference.md
+++ b/docs/configuration/config-file-reference.md
@@ -5061,16 +5061,13 @@ otel:
 ### `QueryAttribute`
 
 ```yaml
-# Query string regex. If evaluated true (on top of meeting all other criteria),
-# query is treated as a high priority.
+# Query string regex.
 [regex: <string> | default = ".*"]
 
-# If query range falls between the start_time and end_time (on top of meeting
-# all other criteria), query is treated as a high priority.
+# Query start time.
 [start_time: <duration> | default = 0s]
 
-# If query range falls between the start_time and end_time (on top of meeting
-# all other criteria), query is treated as a high priority.
+# Query end time.
 [end_time: <duration> | default = 0s]
 ```
 
diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go
index 0403d0c495..a4561823d2 100644
--- a/pkg/util/validation/limits.go
+++ b/pkg/util/validation/limits.go
@@ -61,10 +61,10 @@ type PriorityDef struct {
 }
 
 type QueryAttribute struct {
-	Regex         string         `yaml:"regex" doc:"nocli|description=Query string regex. If evaluated true (on top of meeting all other criteria), query is treated as a high priority.|default=.*"`
+	Regex         string         `yaml:"regex" doc:"nocli|description=Query string regex.|default=.*"`
 	CompiledRegex *regexp.Regexp `yaml:"-" doc:"nocli"`
-	StartTime     time.Duration  `yaml:"start_time" doc:"nocli|description=If query range falls between the start_time and end_time (on top of meeting all other criteria), query is treated as a high priority.|default=0s"`
-	EndTime       time.Duration  `yaml:"end_time" doc:"nocli|description=If query range falls between the start_time and end_time (on top of meeting all other criteria), query is treated as a high priority.|default=0s"`
+	StartTime     time.Duration  `yaml:"start_time" doc:"nocli|description=Query start time.|default=0s"`
+	EndTime       time.Duration  `yaml:"end_time" doc:"nocli|description=Query end time.|default=0s"`
 }
 
 // Limits describe all the limits for users; can be used to describe global default

From 506d5c29a3d07beeaefd5210e89311e5a35b64b2 Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Wed, 15 Nov 2023 04:39:52 -0800
Subject: [PATCH 34/49] Improve time comparison when assigning priority

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 pkg/util/query/priority.go | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/pkg/util/query/priority.go b/pkg/util/query/priority.go
index fa63cd3c30..2b5353266e 100644
--- a/pkg/util/query/priority.go
+++ b/pkg/util/query/priority.go
@@ -39,7 +39,15 @@ func GetPriority(requestParams url.Values, now time.Time, queryPriority *validat
 			}
 
 			startTimeThreshold := now.Add(-1 * attribute.StartTime.Abs()).Truncate(time.Second).UTC()
-			endTimeThreshold := now.Add(-1 * attribute.EndTime.Abs()).Round(time.Second).UTC()
+			endTimeThreshold := now.Add(-1 * attribute.EndTime.Abs()).Add(1 * time.Second).Truncate(time.Second).UTC()
+
+			if startTime, err := parseTime(startParam); err == nil {
+				if endTime, err := parseTime(endParam); err == nil {
+					if isBetweenThresholds(startTime, endTime, startTimeThreshold, endTimeThreshold) {
+						return priority.Priority
+					}
+				}
+			}
 
 			if instantTime, err := parseTime(timeParam); err == nil {
 				if isBetweenThresholds(instantTime, instantTime, startTimeThreshold, endTimeThreshold) {
@@ -47,11 +55,9 @@ func GetPriority(requestParams url.Values, now time.Time, queryPriority *validat
 				}
 			}
 
-			if startTime, err := parseTime(startParam); err == nil {
-				if endTime, err := parseTime(endParam); err == nil {
-					if isBetweenThresholds(startTime, endTime, startTimeThreshold, endTimeThreshold) {
-						return priority.Priority
-					}
+			if timeParam == "" {
+				if isBetweenThresholds(now, now, startTimeThreshold, endTimeThreshold) {
+					return priority.Priority
 				}
 			}
 		}

From c0f268d623e68a71677d4ff4da1003674f9bbc5e Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Wed, 15 Nov 2023 04:41:57 -0800
Subject: [PATCH 35/49] Make reserved querier to match exact priority + change
 query length gauge vector in user request queue

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 pkg/scheduler/queue/queue.go                  | 32 ++------
 pkg/scheduler/queue/queue_test.go             | 16 +++-
 pkg/scheduler/queue/user_queues.go            | 12 ++-
 pkg/scheduler/queue/user_queues_test.go       | 12 +--
 pkg/scheduler/queue/user_request_queue.go     | 64 ++++++++++++---
 .../queue/user_request_queue_test.go          | 79 ++++++++++++++++++-
 6 files changed, 160 insertions(+), 55 deletions(-)

diff --git a/pkg/scheduler/queue/queue.go b/pkg/scheduler/queue/queue.go
index aa29e061f8..6580068900 100644
--- a/pkg/scheduler/queue/queue.go
+++ b/pkg/scheduler/queue/queue.go
@@ -62,16 +62,14 @@ type RequestQueue struct {
 	queues  *queues
 	stopped bool
 
-	queueLength       *prometheus.GaugeVec   // Per user, priority and type of the queue (fifo or priority).
 	totalRequests     *prometheus.CounterVec // Per user and priority.
 	discardedRequests *prometheus.CounterVec // Per user and priority.
 }
 
 func NewRequestQueue(maxOutstandingPerTenant int, forgetDelay time.Duration, queueLength *prometheus.GaugeVec, discardedRequests *prometheus.CounterVec, limits Limits, registerer prometheus.Registerer) *RequestQueue {
 	q := &RequestQueue{
-		queues:                  newUserQueues(maxOutstandingPerTenant, forgetDelay, limits),
+		queues:                  newUserQueues(maxOutstandingPerTenant, forgetDelay, limits, queueLength),
 		connectedQuerierWorkers: atomic.NewInt32(0),
-		queueLength:             queueLength,
 		totalRequests: promauto.With(registerer).NewCounterVec(prometheus.CounterOpts{
 			Name: "cortex_request_queue_requests_total",
 			Help: "Total number of query requests going to the request queue.",
@@ -108,10 +106,6 @@ func (q *RequestQueue) EnqueueRequest(userID string, req Request, maxQueriers fl
 		return errors.New("no queue found")
 	}
 
-	queueType := "fifo"
-	if q.queues.limits.QueryPriority(userID).Enabled {
-		queueType = "priority"
-	}
 	metricLabels := prometheus.Labels{
 		"user":     userID,
 		"priority": priority,
@@ -124,11 +118,6 @@ func (q *RequestQueue) EnqueueRequest(userID string, req Request, maxQueriers fl
 	}
 
 	queue.enqueueRequest(req)
-	q.queueLength.With(prometheus.Labels{
-		"user":     userID,
-		"priority": priority,
-		"type":     queueType,
-	}).Inc()
 	q.cond.Broadcast()
 	// Call this function while holding a lock. This guarantees that no querier can fetch the request before function returns.
 	if successFn != nil {
@@ -170,10 +159,10 @@ FindQueue:
 
 		// Pick next request from the queue.
 		for {
-			minPriority, checkMinPriority := q.getMinPriorityForQuerier(userID, querierID)
-			request := queue.dequeueRequest(minPriority, checkMinPriority)
+			priority, matchPriority := q.getPriorityForQuerier(userID, querierID)
+			request := queue.dequeueRequest(priority, matchPriority)
 			if request == nil {
-				// the queue does not contain request with the min priority, break to wait for more requests
+				// the queue does not contain request with the min priority, wait for more requests
 				querierWait = true
 				goto FindQueue
 			}
@@ -182,17 +171,6 @@ FindQueue:
 				q.queues.deleteQueue(userID)
 			}
 
-			queueType := "fifo"
-			if q.queues.limits.QueryPriority(userID).Enabled {
-				queueType = "priority"
-			}
-			metricLabels := prometheus.Labels{
-				"user":     userID,
-				"priority": strconv.FormatInt(request.Priority(), 10),
-				"type":     queueType,
-			}
-			q.queueLength.With(metricLabels).Dec()
-
 			// Tell close() we've processed a request.
 			q.cond.Broadcast()
 
@@ -206,7 +184,7 @@ FindQueue:
 	goto FindQueue
 }
 
-func (q *RequestQueue) getMinPriorityForQuerier(userID string, querierID string) (int64, bool) {
+func (q *RequestQueue) getPriorityForQuerier(userID string, querierID string) (int64, bool) {
 	if priority, ok := q.queues.userQueues[userID].reservedQueriers[querierID]; ok {
 		return priority, true
 	}
diff --git a/pkg/scheduler/queue/queue_test.go b/pkg/scheduler/queue/queue_test.go
index 02317338d0..8c7c1f0794 100644
--- a/pkg/scheduler/queue/queue_test.go
+++ b/pkg/scheduler/queue/queue_test.go
@@ -226,10 +226,10 @@ func TestReservedQueriersShouldOnlyGetHighPriorityQueries(t *testing.T) {
 
 	assert.NoError(t, queue.EnqueueRequest("userID", normalRequest, 1, func() {}))
 	assert.NoError(t, queue.EnqueueRequest("userID", priority1Request, 1, func() {}))
-	assert.NoError(t, queue.EnqueueRequest("userID", priority2Request, 1, func() {}))
+	assert.NoError(t, queue.EnqueueRequest("userID", priority1Request, 1, func() {}))
 
 	nextRequest, _, _ := queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1")
-	assert.Equal(t, priority2Request, nextRequest)
+	assert.Equal(t, priority1Request, nextRequest)
 
 	nextRequest, _, _ = queue.GetNextRequestForQuerier(ctx, FirstUser(), "querier-1")
 	assert.Equal(t, priority1Request, nextRequest)
@@ -243,6 +243,18 @@ func TestReservedQueriersShouldOnlyGetHighPriorityQueries(t *testing.T) {
 	nextRequest, _, _ = queue.GetNextRequestForQuerier(ctxTimeout, FirstUser(), "querier-1")
 	assert.Nil(t, nextRequest)
 	assert.Equal(t, 1, queue.queues.userQueues["userID"].queue.length())
+
+	assert.NoError(t, queue.EnqueueRequest("userID", priority2Request, 1, func() {}))
+
+	ctxTimeout, cancel = context.WithTimeout(ctx, 1*time.Second)
+	defer cancel()
+
+	time.AfterFunc(2*time.Second, func() {
+		queue.cond.Broadcast()
+	})
+	nextRequest, _, _ = queue.GetNextRequestForQuerier(ctxTimeout, FirstUser(), "querier-1")
+	assert.Nil(t, nextRequest)
+	assert.Equal(t, 2, queue.queues.userQueues["userID"].queue.length())
 }
 
 func TestExitingRequestsShouldPersistEvenIfTheConfigHasChanged(t *testing.T) {
diff --git a/pkg/scheduler/queue/user_queues.go b/pkg/scheduler/queue/user_queues.go
index 816f87aae8..a42b3f80c6 100644
--- a/pkg/scheduler/queue/user_queues.go
+++ b/pkg/scheduler/queue/user_queues.go
@@ -6,9 +6,10 @@ import (
 	"sort"
 	"time"
 
-	"github.com/cortexproject/cortex/pkg/util/validation"
+	"github.com/prometheus/client_golang/prometheus"
 
 	"github.com/cortexproject/cortex/pkg/util"
+	"github.com/cortexproject/cortex/pkg/util/validation"
 )
 
 // Limits needed for the Query Scheduler - interface used for decoupling.
@@ -57,6 +58,8 @@ type queues struct {
 	sortedQueriers []string
 
 	limits Limits
+
+	queueLength *prometheus.GaugeVec // Per user, type and priority.
 }
 
 type userQueue struct {
@@ -79,7 +82,7 @@ type userQueue struct {
 	index int
 }
 
-func newUserQueues(maxUserQueueSize int, forgetDelay time.Duration, limits Limits) *queues {
+func newUserQueues(maxUserQueueSize int, forgetDelay time.Duration, limits Limits, queueLength *prometheus.GaugeVec) *queues {
 	return &queues{
 		userQueues:       map[string]*userQueue{},
 		users:            nil,
@@ -88,6 +91,7 @@ func newUserQueues(maxUserQueueSize int, forgetDelay time.Duration, limits Limit
 		queriers:         map[string]*querier{},
 		sortedQueriers:   nil,
 		limits:           limits,
+		queueLength:      queueLength,
 	}
 }
 
@@ -195,7 +199,7 @@ func (q *queues) getOrAddQueue(userID string, maxQueriers int) userRequestQueue
 
 func (q *queues) createUserRequestQueue(userID string) userRequestQueue {
 	if q.limits.QueryPriority(userID).Enabled {
-		return NewPriorityRequestQueue(util.NewPriorityQueue(nil))
+		return NewPriorityRequestQueue(util.NewPriorityQueue(nil), userID, q.queueLength)
 	}
 
 	queueSize := q.limits.MaxOutstandingPerTenant(userID)
@@ -206,7 +210,7 @@ func (q *queues) createUserRequestQueue(userID string) userRequestQueue {
 		queueSize = q.maxUserQueueSize
 	}
 
-	return NewFIFORequestQueue(make(chan Request, queueSize))
+	return NewFIFORequestQueue(make(chan Request, queueSize), userID, q.queueLength)
 }
 
 // Finds next queue for the querier. To support fair scheduling between users, client is expected
diff --git a/pkg/scheduler/queue/user_queues_test.go b/pkg/scheduler/queue/user_queues_test.go
index df4fce2b86..75c876c516 100644
--- a/pkg/scheduler/queue/user_queues_test.go
+++ b/pkg/scheduler/queue/user_queues_test.go
@@ -15,7 +15,7 @@ import (
 )
 
 func TestQueues(t *testing.T) {
-	uq := newUserQueues(0, 0, MockLimits{})
+	uq := newUserQueues(0, 0, MockLimits{}, nil)
 	assert.NotNil(t, uq)
 	assert.NoError(t, isConsistent(uq))
 
@@ -70,7 +70,7 @@ func TestQueues(t *testing.T) {
 }
 
 func TestQueuesWithQueriers(t *testing.T) {
-	uq := newUserQueues(0, 0, MockLimits{})
+	uq := newUserQueues(0, 0, MockLimits{}, nil)
 	assert.NotNil(t, uq)
 	assert.NoError(t, isConsistent(uq))
 
@@ -147,7 +147,7 @@ func TestQueuesConsistency(t *testing.T) {
 
 	for testName, testData := range tests {
 		t.Run(testName, func(t *testing.T) {
-			uq := newUserQueues(0, testData.forgetDelay, MockLimits{})
+			uq := newUserQueues(0, testData.forgetDelay, MockLimits{}, nil)
 			assert.NotNil(t, uq)
 			assert.NoError(t, isConsistent(uq))
 
@@ -196,7 +196,7 @@ func TestQueues_ForgetDelay(t *testing.T) {
 	)
 
 	now := time.Now()
-	uq := newUserQueues(0, forgetDelay, MockLimits{})
+	uq := newUserQueues(0, forgetDelay, MockLimits{}, nil)
 	assert.NotNil(t, uq)
 	assert.NoError(t, isConsistent(uq))
 
@@ -288,7 +288,7 @@ func TestQueues_ForgetDelay_ShouldCorrectlyHandleQuerierReconnectingBeforeForget
 	)
 
 	now := time.Now()
-	uq := newUserQueues(0, forgetDelay, MockLimits{})
+	uq := newUserQueues(0, forgetDelay, MockLimits{}, nil)
 	assert.NotNil(t, uq)
 	assert.NoError(t, isConsistent(uq))
 
@@ -357,7 +357,7 @@ func TestGetOrAddQueueShouldUpdateProperties(t *testing.T) {
 	limits := MockLimits{
 		MaxOutstanding: 3,
 	}
-	q := newUserQueues(0, 0, limits)
+	q := newUserQueues(0, 0, limits, nil)
 	q.addQuerierConnection("q-1")
 	q.addQuerierConnection("q-2")
 	q.addQuerierConnection("q-3")
diff --git a/pkg/scheduler/queue/user_request_queue.go b/pkg/scheduler/queue/user_request_queue.go
index 9f95c9e8c5..5325e94d9b 100644
--- a/pkg/scheduler/queue/user_request_queue.go
+++ b/pkg/scheduler/queue/user_request_queue.go
@@ -1,27 +1,50 @@
 package queue
 
-import "github.com/cortexproject/cortex/pkg/util"
+import (
+	"strconv"
+
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/cortexproject/cortex/pkg/util"
+)
 
 type userRequestQueue interface {
 	enqueueRequest(Request)
-	dequeueRequest(minPriority int64, checkMinPriority bool) Request
+	dequeueRequest(int64, bool) Request
 	length() int
 }
 
 type FIFORequestQueue struct {
-	queue chan Request
+	queue       chan Request
+	userID      string
+	queueLength *prometheus.GaugeVec
 }
 
-func NewFIFORequestQueue(queue chan Request) *FIFORequestQueue {
-	return &FIFORequestQueue{queue: queue}
+func NewFIFORequestQueue(queue chan Request, userID string, queueLength *prometheus.GaugeVec) *FIFORequestQueue {
+	return &FIFORequestQueue{queue: queue, userID: userID, queueLength: queueLength}
 }
 
 func (f *FIFORequestQueue) enqueueRequest(r Request) {
 	f.queue <- r
+	if f.queueLength != nil {
+		f.queueLength.With(prometheus.Labels{
+			"user":     f.userID,
+			"priority": strconv.FormatInt(r.Priority(), 10),
+			"type":     "fifo",
+		}).Inc()
+	}
 }
 
 func (f *FIFORequestQueue) dequeueRequest(_ int64, _ bool) Request {
-	return <-f.queue
+	r := <-f.queue
+	if f.queueLength != nil {
+		f.queueLength.With(prometheus.Labels{
+			"user":     f.userID,
+			"priority": strconv.FormatInt(r.Priority(), 10),
+			"type":     "fifo",
+		}).Dec()
+	}
+	return r
 }
 
 func (f *FIFORequestQueue) length() int {
@@ -29,22 +52,39 @@ func (f *FIFORequestQueue) length() int {
 }
 
 type PriorityRequestQueue struct {
-	queue *util.PriorityQueue
+	queue       *util.PriorityQueue
+	userID      string
+	queueLength *prometheus.GaugeVec
 }
 
-func NewPriorityRequestQueue(queue *util.PriorityQueue) *PriorityRequestQueue {
-	return &PriorityRequestQueue{queue: queue}
+func NewPriorityRequestQueue(queue *util.PriorityQueue, userID string, queueLength *prometheus.GaugeVec) *PriorityRequestQueue {
+	return &PriorityRequestQueue{queue: queue, userID: userID, queueLength: queueLength}
 }
 
 func (f *PriorityRequestQueue) enqueueRequest(r Request) {
 	f.queue.Enqueue(r)
+	if f.queueLength != nil {
+		f.queueLength.With(prometheus.Labels{
+			"user":     f.userID,
+			"priority": strconv.FormatInt(r.Priority(), 10),
+			"type":     "priority",
+		}).Inc()
+	}
 }
 
-func (f *PriorityRequestQueue) dequeueRequest(minPriority int64, checkMinPriority bool) Request {
-	if checkMinPriority && f.queue.Peek().Priority() < minPriority {
+func (f *PriorityRequestQueue) dequeueRequest(priority int64, matchPriority bool) Request {
+	if matchPriority && f.queue.Peek().Priority() != priority {
 		return nil
 	}
-	return f.queue.Dequeue()
+	r := f.queue.Dequeue()
+	if f.queueLength != nil {
+		f.queueLength.With(prometheus.Labels{
+			"user":     f.userID,
+			"priority": strconv.FormatInt(r.Priority(), 10),
+			"type":     "priority",
+		}).Dec()
+	}
+	return r
 }
 
 func (f *PriorityRequestQueue) length() int {
diff --git a/pkg/scheduler/queue/user_request_queue_test.go b/pkg/scheduler/queue/user_request_queue_test.go
index aeaa88440d..cabe792e06 100644
--- a/pkg/scheduler/queue/user_request_queue_test.go
+++ b/pkg/scheduler/queue/user_request_queue_test.go
@@ -1,15 +1,25 @@
 package queue
 
 import (
+	"strings"
 	"testing"
 
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	promtest "github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/stretchr/testify/assert"
 
 	"github.com/cortexproject/cortex/pkg/util"
 )
 
 func TestFIFORequestQueue(t *testing.T) {
-	queue := NewFIFORequestQueue(make(chan Request, 2))
+	reg := prometheus.NewPedanticRegistry()
+	queueLength := promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{
+		Name: "cortex_query_scheduler_queue_length",
+		Help: "Number of queries in the queue.",
+	}, []string{"user", "priority", "type"})
+
+	queue := NewFIFORequestQueue(make(chan Request, 2), "userID", queueLength)
 	request1 := MockRequest{
 		id:       "request 1",
 		priority: 1,
@@ -21,15 +31,40 @@ func TestFIFORequestQueue(t *testing.T) {
 
 	queue.enqueueRequest(request1)
 	queue.enqueueRequest(request2)
+
+	assert.NoError(t, promtest.GatherAndCompare(reg, strings.NewReader(`
+		# HELP cortex_query_scheduler_queue_length Number of queries in the queue.
+		# TYPE cortex_query_scheduler_queue_length gauge
+		cortex_query_scheduler_queue_length{priority="1",type="fifo",user="userID"} 1
+		cortex_query_scheduler_queue_length{priority="2",type="fifo",user="userID"} 1
+	`), "cortex_query_scheduler_queue_length"))
 	assert.Equal(t, 2, queue.length())
 	assert.Equal(t, request1, queue.dequeueRequest(0, false))
 	assert.Equal(t, 1, queue.length())
+	assert.NoError(t, promtest.GatherAndCompare(reg, strings.NewReader(`
+		# HELP cortex_query_scheduler_queue_length Number of queries in the queue.
+		# TYPE cortex_query_scheduler_queue_length gauge
+		cortex_query_scheduler_queue_length{priority="1",type="fifo",user="userID"} 0
+		cortex_query_scheduler_queue_length{priority="2",type="fifo",user="userID"} 1
+	`), "cortex_query_scheduler_queue_length"))
 	assert.Equal(t, request2, queue.dequeueRequest(0, false))
 	assert.Equal(t, 0, queue.length())
+	assert.NoError(t, promtest.GatherAndCompare(reg, strings.NewReader(`
+		# HELP cortex_query_scheduler_queue_length Number of queries in the queue.
+		# TYPE cortex_query_scheduler_queue_length gauge
+		cortex_query_scheduler_queue_length{priority="1",type="fifo",user="userID"} 0
+		cortex_query_scheduler_queue_length{priority="2",type="fifo",user="userID"} 0
+	`), "cortex_query_scheduler_queue_length"))
 }
 
 func TestPriorityRequestQueue(t *testing.T) {
-	queue := NewPriorityRequestQueue(util.NewPriorityQueue(nil))
+	reg := prometheus.NewPedanticRegistry()
+	queueLength := promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{
+		Name: "cortex_query_scheduler_queue_length",
+		Help: "Number of queries in the queue.",
+	}, []string{"user", "priority", "type"})
+
+	queue := NewPriorityRequestQueue(util.NewPriorityQueue(nil), "userID", queueLength)
 	request1 := MockRequest{
 		id:       "request 1",
 		priority: 1,
@@ -41,18 +76,54 @@ func TestPriorityRequestQueue(t *testing.T) {
 
 	queue.enqueueRequest(request1)
 	queue.enqueueRequest(request2)
+	assert.NoError(t, promtest.GatherAndCompare(reg, strings.NewReader(`
+		# HELP cortex_query_scheduler_queue_length Number of queries in the queue.
+		# TYPE cortex_query_scheduler_queue_length gauge
+		cortex_query_scheduler_queue_length{priority="1",type="priority",user="userID"} 1
+		cortex_query_scheduler_queue_length{priority="2",type="priority",user="userID"} 1
+	`), "cortex_query_scheduler_queue_length"))
 	assert.Equal(t, 2, queue.length())
-	assert.Equal(t, request2, queue.dequeueRequest(0, true))
+	assert.Equal(t, request2, queue.dequeueRequest(0, false))
 	assert.Equal(t, 1, queue.length())
-	assert.Equal(t, request1, queue.dequeueRequest(0, true))
+	assert.NoError(t, promtest.GatherAndCompare(reg, strings.NewReader(`
+		# HELP cortex_query_scheduler_queue_length Number of queries in the queue.
+		# TYPE cortex_query_scheduler_queue_length gauge
+		cortex_query_scheduler_queue_length{priority="1",type="priority",user="userID"} 1
+		cortex_query_scheduler_queue_length{priority="2",type="priority",user="userID"} 0
+	`), "cortex_query_scheduler_queue_length"))
+	assert.Equal(t, request1, queue.dequeueRequest(0, false))
 	assert.Equal(t, 0, queue.length())
+	assert.NoError(t, promtest.GatherAndCompare(reg, strings.NewReader(`
+		# HELP cortex_query_scheduler_queue_length Number of queries in the queue.
+		# TYPE cortex_query_scheduler_queue_length gauge
+		cortex_query_scheduler_queue_length{priority="1",type="priority",user="userID"} 0
+		cortex_query_scheduler_queue_length{priority="2",type="priority",user="userID"} 0
+	`), "cortex_query_scheduler_queue_length"))
 
 	queue.enqueueRequest(request1)
 	queue.enqueueRequest(request2)
+	assert.NoError(t, promtest.GatherAndCompare(reg, strings.NewReader(`
+		# HELP cortex_query_scheduler_queue_length Number of queries in the queue.
+		# TYPE cortex_query_scheduler_queue_length gauge
+		cortex_query_scheduler_queue_length{priority="1",type="priority",user="userID"} 1
+		cortex_query_scheduler_queue_length{priority="2",type="priority",user="userID"} 1
+	`), "cortex_query_scheduler_queue_length"))
 	assert.Equal(t, 2, queue.length())
 	assert.Equal(t, request2, queue.dequeueRequest(2, true))
 	assert.Equal(t, 1, queue.length())
 	assert.Nil(t, queue.dequeueRequest(2, true))
+	assert.NoError(t, promtest.GatherAndCompare(reg, strings.NewReader(`
+		# HELP cortex_query_scheduler_queue_length Number of queries in the queue.
+		# TYPE cortex_query_scheduler_queue_length gauge
+		cortex_query_scheduler_queue_length{priority="1",type="priority",user="userID"} 1
+		cortex_query_scheduler_queue_length{priority="2",type="priority",user="userID"} 0
+	`), "cortex_query_scheduler_queue_length"))
 	assert.Equal(t, request1, queue.dequeueRequest(2, false))
 	assert.Equal(t, 0, queue.length())
+	assert.NoError(t, promtest.GatherAndCompare(reg, strings.NewReader(`
+		# HELP cortex_query_scheduler_queue_length Number of queries in the queue.
+		# TYPE cortex_query_scheduler_queue_length gauge
+		cortex_query_scheduler_queue_length{priority="1",type="priority",user="userID"} 0
+		cortex_query_scheduler_queue_length{priority="2",type="priority",user="userID"} 0
+	`), "cortex_query_scheduler_queue_length"))
 }

From 0b987730e8151b9fd7d7a11d576220315707f695 Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Wed, 15 Nov 2023 06:28:25 -0800
Subject: [PATCH 36/49] Bug fix

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 pkg/scheduler/queue/user_queues.go      | 1 +
 pkg/scheduler/queue/user_queues_test.go | 5 +++++
 2 files changed, 6 insertions(+)

diff --git a/pkg/scheduler/queue/user_queues.go b/pkg/scheduler/queue/user_queues.go
index a42b3f80c6..722539ef9f 100644
--- a/pkg/scheduler/queue/user_queues.go
+++ b/pkg/scheduler/queue/user_queues.go
@@ -168,6 +168,7 @@ func (q *queues) getOrAddQueue(userID string, maxQueriers int) userRequestQueue
 
 		uq.queue = tmpQueue
 		uq.maxOutstanding = q.limits.MaxOutstandingPerTenant(userID)
+		uq.priorityEnabled = priorityEnabled
 	}
 
 	if uq.maxQueriers != maxQueriers {
diff --git a/pkg/scheduler/queue/user_queues_test.go b/pkg/scheduler/queue/user_queues_test.go
index 75c876c516..ae812aac6a 100644
--- a/pkg/scheduler/queue/user_queues_test.go
+++ b/pkg/scheduler/queue/user_queues_test.go
@@ -406,6 +406,11 @@ func TestGetOrAddQueueShouldUpdateProperties(t *testing.T) {
 	assert.Subset(t, getKeys(q.queriers), getKeys(q.userQueues["userID"].queriers))
 	assert.Subset(t, getKeys(q.userQueues["userID"].queriers), getKeys(q.userQueues["userID"].reservedQueriers))
 
+	limits.QueryPriorityVal.Enabled = false
+	q.limits = limits
+	queue = q.getOrAddQueue("userID", 3)
+	assert.IsType(t, &FIFORequestQueue{}, queue)
+
 	// check the queriers and reservedQueriers map are consistent
 	for i := 0; i < 100; i++ {
 		queriers := q.userQueues["userID"].queriers

From 3fffc3fe85f42d30a0525b8f8fbbce0603ca60cd Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Wed, 15 Nov 2023 15:21:15 -0800
Subject: [PATCH 37/49] Add comments

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 pkg/frontend/v1/frontend.go        |  5 ++++-
 pkg/frontend/v2/frontend.go        |  5 ++++-
 pkg/scheduler/queue/queue.go       |  2 +-
 pkg/scheduler/queue/user_queues.go | 14 +++++++++-----
 pkg/util/query/priority.go         |  1 +
 pkg/util/validation/limits.go      |  1 +
 6 files changed, 20 insertions(+), 8 deletions(-)

diff --git a/pkg/frontend/v1/frontend.go b/pkg/frontend/v1/frontend.go
index de5d2d0647..54e8ada594 100644
--- a/pkg/frontend/v1/frontend.go
+++ b/pkg/frontend/v1/frontend.go
@@ -80,7 +80,10 @@ type Frontend struct {
 	requestQueue *queue.RequestQueue
 	activeUsers  *util.ActiveUsersCleanupService
 
-	queryPriority         validation.QueryPriority
+	// Used to check whether query priority config has changed
+	queryPriority validation.QueryPriority
+
+	// Populate and reuse compiled regex until query priority config changes
 	compiledQueryPriority validation.QueryPriority
 
 	// Subservices manager.
diff --git a/pkg/frontend/v2/frontend.go b/pkg/frontend/v2/frontend.go
index c99980100e..b05af6277f 100644
--- a/pkg/frontend/v2/frontend.go
+++ b/pkg/frontend/v2/frontend.go
@@ -76,7 +76,10 @@ type Frontend struct {
 
 	lastQueryID atomic.Uint64
 
-	queryPriority         validation.QueryPriority
+	// Used to check whether query priority config has changed
+	queryPriority validation.QueryPriority
+
+	// Populate and reuse compiled regex until query priority config changes
 	compiledQueryPriority validation.QueryPriority
 
 	// frontend workers will read from this channel, and send request to scheduler.
diff --git a/pkg/scheduler/queue/queue.go b/pkg/scheduler/queue/queue.go
index 6580068900..57f85bfb12 100644
--- a/pkg/scheduler/queue/queue.go
+++ b/pkg/scheduler/queue/queue.go
@@ -162,7 +162,7 @@ FindQueue:
 			priority, matchPriority := q.getPriorityForQuerier(userID, querierID)
 			request := queue.dequeueRequest(priority, matchPriority)
 			if request == nil {
-				// the queue does not contain request with the min priority, wait for more requests
+				// The queue does not contain request with the priority, wait for more requests
 				querierWait = true
 				goto FindQueue
 			}
diff --git a/pkg/scheduler/queue/user_queues.go b/pkg/scheduler/queue/user_queues.go
index 722539ef9f..b0ae8069ac 100644
--- a/pkg/scheduler/queue/user_queues.go
+++ b/pkg/scheduler/queue/user_queues.go
@@ -67,12 +67,16 @@ type userQueue struct {
 
 	// If not nil, only these queriers can handle user requests. If nil, all queriers can.
 	// We set this to nil if number of available queriers <= maxQueriers.
-	queriers         map[string]struct{}
-	maxQueriers      int
-	maxOutstanding   int
+	queriers map[string]struct{}
+
+	// Contains assigned priority for querier ID
 	reservedQueriers map[string]int64
-	priorityList     []int64
-	priorityEnabled  bool
+
+	// Stores last limit config for the user. When changed, re-populate queriers and reservedQueriers
+	maxQueriers     int
+	maxOutstanding  int
+	priorityList    []int64
+	priorityEnabled bool
 
 	// Seed for shuffle sharding of queriers. This seed is based on userID only and is therefore consistent
 	// between different frontends.
diff --git a/pkg/util/query/priority.go b/pkg/util/query/priority.go
index 2b5353266e..79e08c1ad8 100644
--- a/pkg/util/query/priority.go
+++ b/pkg/util/query/priority.go
@@ -24,6 +24,7 @@ func GetPriority(requestParams url.Values, now time.Time, queryPriority *validat
 
 	for i, priority := range queryPriority.Priorities {
 		for j, attribute := range priority.QueryAttributes {
+			// If query priority config changed, re-populate the compiled regex
 			if queryPriorityChanged {
 				compiledRegex, err := regexp.Compile(attribute.Regex)
 				if err != nil {
diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go
index a4561823d2..b1bebf1d42 100644
--- a/pkg/util/validation/limits.go
+++ b/pkg/util/validation/limits.go
@@ -250,6 +250,7 @@ func (l *Limits) Validate(shardByAllLabels bool) error {
 		return errMaxGlobalSeriesPerUserValidation
 	}
 
+	// If query priority is enabled, do not allow duplicate priority values
 	if l.QueryPriority.Enabled {
 		queryPriority := l.QueryPriority
 		prioritySet := map[int64]struct{}{}

From 2abeffbec992f38fb1842b4f68d8319a7b5d3f9e Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Fri, 17 Nov 2023 08:58:58 -0800
Subject: [PATCH 38/49] Address comments

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 pkg/frontend/v1/frontend.go               |  35 +++++---
 pkg/frontend/v2/frontend.go               |  34 +++++---
 pkg/frontend/v2/frontend_test.go          |   3 +-
 pkg/scheduler/queue/user_request_queue.go |  24 +----
 pkg/util/query/priority.go                |  64 ++++++--------
 pkg/util/query/priority_test.go           | 101 ++++++++--------------
 6 files changed, 115 insertions(+), 146 deletions(-)

diff --git a/pkg/frontend/v1/frontend.go b/pkg/frontend/v1/frontend.go
index 54e8ada594..8ea66fbd58 100644
--- a/pkg/frontend/v1/frontend.go
+++ b/pkg/frontend/v1/frontend.go
@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"net/url"
 	"reflect"
+	"sync"
 	"time"
 
 	"github.com/go-kit/log"
@@ -81,10 +82,11 @@ type Frontend struct {
 	activeUsers  *util.ActiveUsersCleanupService
 
 	// Used to check whether query priority config has changed
-	queryPriority validation.QueryPriority
+	queryPriority    map[string]validation.QueryPriority
+	queryPriorityMtx map[string]*sync.RWMutex
 
 	// Populate and reuse compiled regex until query priority config changes
-	compiledQueryPriority validation.QueryPriority
+	compiledQueryPriority map[string]validation.QueryPriority
 
 	// Subservices manager.
 	subservices        *services.Manager
@@ -217,16 +219,29 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest,
 			response: make(chan *httpgrpc.HTTPResponse, 1),
 		}
 
-		queryPriority := f.limits.QueryPriority(userID)
+		if reqParams != nil {
+			queryPriority := f.limits.QueryPriority(userID)
 
-		if reqParams != nil && queryPriority.Enabled {
-			queryPriorityChanged := !reflect.DeepEqual(f.queryPriority, queryPriority)
-			if queryPriorityChanged {
-				f.queryPriority = queryPriority
-				f.compiledQueryPriority = queryPriority
-			}
+			if queryPriority.Enabled {
+				if _, exists := f.queryPriorityMtx[userID]; !exists {
+					f.queryPriorityMtx[userID] = &sync.RWMutex{}
+				}
+
+				f.queryPriorityMtx[userID].RLock()
+				queryPriorityChanged := !reflect.DeepEqual(f.queryPriority[userID], queryPriority)
+				f.queryPriorityMtx[userID].RUnlock()
 
-			request.priority = util_query.GetPriority(reqParams, ts, &f.compiledQueryPriority, queryPriorityChanged)
+				if queryPriorityChanged {
+					f.queryPriorityMtx[userID].Lock()
+					f.queryPriority[userID] = queryPriority
+					f.compiledQueryPriority[userID] = util_query.GetCompileQueryPriority(queryPriority)
+					f.queryPriorityMtx[userID].Unlock()
+				}
+
+				f.queryPriorityMtx[userID].RLock()
+				request.priority = util_query.GetPriority(reqParams, ts, f.compiledQueryPriority[userID])
+				f.queryPriorityMtx[userID].Unlock()
+			}
 		}
 
 		if err := f.queueRequest(ctx, &request); err != nil {
diff --git a/pkg/frontend/v2/frontend.go b/pkg/frontend/v2/frontend.go
index b05af6277f..41dd4e7749 100644
--- a/pkg/frontend/v2/frontend.go
+++ b/pkg/frontend/v2/frontend.go
@@ -77,10 +77,11 @@ type Frontend struct {
 	lastQueryID atomic.Uint64
 
 	// Used to check whether query priority config has changed
-	queryPriority validation.QueryPriority
+	queryPriority    map[string]validation.QueryPriority
+	queryPriorityMtx map[string]*sync.RWMutex
 
 	// Populate and reuse compiled regex until query priority config changes
-	compiledQueryPriority validation.QueryPriority
+	compiledQueryPriority map[string]validation.QueryPriority
 
 	// frontend workers will read from this channel, and send request to scheduler.
 	requestsCh chan *frontendRequest
@@ -218,16 +219,29 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest,
 			retryOnTooManyOutstandingRequests: f.cfg.RetryOnTooManyOutstandingRequests && f.schedulerWorkers.getWorkersCount() > 1,
 		}
 
-		queryPriority := f.limits.QueryPriority(userID)
+		if reqParams != nil {
+			queryPriority := f.limits.QueryPriority(userID)
 
-		if reqParams != nil && queryPriority.Enabled {
-			queryPriorityChanged := !reflect.DeepEqual(f.queryPriority, queryPriority)
-			if queryPriorityChanged {
-				f.queryPriority = queryPriority
-				f.compiledQueryPriority = queryPriority
-			}
+			if queryPriority.Enabled {
+				if _, exists := f.queryPriorityMtx[userID]; !exists {
+					f.queryPriorityMtx[userID] = &sync.RWMutex{}
+				}
+
+				f.queryPriorityMtx[userID].RLock()
+				queryPriorityChanged := !reflect.DeepEqual(f.queryPriority[userID], queryPriority)
+				f.queryPriorityMtx[userID].RUnlock()
 
-			freq.priority = util_query.GetPriority(reqParams, ts, &f.compiledQueryPriority, queryPriorityChanged)
+				if queryPriorityChanged {
+					f.queryPriorityMtx[userID].Lock()
+					f.queryPriority[userID] = queryPriority
+					f.compiledQueryPriority[userID] = util_query.GetCompileQueryPriority(queryPriority)
+					f.queryPriorityMtx[userID].Unlock()
+				}
+
+				f.queryPriorityMtx[userID].RLock()
+				freq.priority = util_query.GetPriority(reqParams, ts, f.compiledQueryPriority[userID])
+				f.queryPriorityMtx[userID].Unlock()
+			}
 		}
 
 		f.requests.put(freq)
diff --git a/pkg/frontend/v2/frontend_test.go b/pkg/frontend/v2/frontend_test.go
index a9cd226f59..4c40d88b66 100644
--- a/pkg/frontend/v2/frontend_test.go
+++ b/pkg/frontend/v2/frontend_test.go
@@ -10,8 +10,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/cortexproject/cortex/pkg/scheduler/queue"
-
 	"github.com/go-kit/log"
 	"github.com/stretchr/testify/require"
 	"github.com/weaveworks/common/httpgrpc"
@@ -22,6 +20,7 @@ import (
 	"github.com/cortexproject/cortex/pkg/frontend/transport"
 	"github.com/cortexproject/cortex/pkg/frontend/v2/frontendv2pb"
 	"github.com/cortexproject/cortex/pkg/querier/stats"
+	"github.com/cortexproject/cortex/pkg/scheduler/queue"
 	"github.com/cortexproject/cortex/pkg/scheduler/schedulerpb"
 	"github.com/cortexproject/cortex/pkg/util/flagext"
 	"github.com/cortexproject/cortex/pkg/util/services"
diff --git a/pkg/scheduler/queue/user_request_queue.go b/pkg/scheduler/queue/user_request_queue.go
index 5325e94d9b..828c9246fd 100644
--- a/pkg/scheduler/queue/user_request_queue.go
+++ b/pkg/scheduler/queue/user_request_queue.go
@@ -27,22 +27,14 @@ func NewFIFORequestQueue(queue chan Request, userID string, queueLength *prometh
 func (f *FIFORequestQueue) enqueueRequest(r Request) {
 	f.queue <- r
 	if f.queueLength != nil {
-		f.queueLength.With(prometheus.Labels{
-			"user":     f.userID,
-			"priority": strconv.FormatInt(r.Priority(), 10),
-			"type":     "fifo",
-		}).Inc()
+		f.queueLength.WithLabelValues(f.userID, strconv.FormatInt(r.Priority(), 10), "fifo").Inc()
 	}
 }
 
 func (f *FIFORequestQueue) dequeueRequest(_ int64, _ bool) Request {
 	r := <-f.queue
 	if f.queueLength != nil {
-		f.queueLength.With(prometheus.Labels{
-			"user":     f.userID,
-			"priority": strconv.FormatInt(r.Priority(), 10),
-			"type":     "fifo",
-		}).Dec()
+		f.queueLength.WithLabelValues(f.userID, strconv.FormatInt(r.Priority(), 10), "fifo").Dec()
 	}
 	return r
 }
@@ -64,11 +56,7 @@ func NewPriorityRequestQueue(queue *util.PriorityQueue, userID string, queueLeng
 func (f *PriorityRequestQueue) enqueueRequest(r Request) {
 	f.queue.Enqueue(r)
 	if f.queueLength != nil {
-		f.queueLength.With(prometheus.Labels{
-			"user":     f.userID,
-			"priority": strconv.FormatInt(r.Priority(), 10),
-			"type":     "priority",
-		}).Inc()
+		f.queueLength.WithLabelValues(f.userID, strconv.FormatInt(r.Priority(), 10), "priority").Inc()
 	}
 }
 
@@ -78,11 +66,7 @@ func (f *PriorityRequestQueue) dequeueRequest(priority int64, matchPriority bool
 	}
 	r := f.queue.Dequeue()
 	if f.queueLength != nil {
-		f.queueLength.With(prometheus.Labels{
-			"user":     f.userID,
-			"priority": strconv.FormatInt(r.Priority(), 10),
-			"type":     "priority",
-		}).Dec()
+		f.queueLength.WithLabelValues(f.userID, strconv.FormatInt(r.Priority(), 10), "priority").Dec()
 	}
 	return r
 }
diff --git a/pkg/util/query/priority.go b/pkg/util/query/priority.go
index 79e08c1ad8..503736f208 100644
--- a/pkg/util/query/priority.go
+++ b/pkg/util/query/priority.go
@@ -1,18 +1,32 @@
 package query
 
 import (
-	"math"
 	"net/url"
 	"regexp"
-	"strconv"
 	"time"
 
-	"github.com/pkg/errors"
-
+	"github.com/cortexproject/cortex/pkg/util"
 	"github.com/cortexproject/cortex/pkg/util/validation"
 )
 
-func GetPriority(requestParams url.Values, now time.Time, queryPriority *validation.QueryPriority, queryPriorityChanged bool) int64 {
+func GetCompileQueryPriority(queryPriority validation.QueryPriority) validation.QueryPriority {
+	compiledQueryPriority := queryPriority
+	for i, priority := range compiledQueryPriority.Priorities {
+		for j, attribute := range priority.QueryAttributes {
+			compiledRegex, err := regexp.Compile(attribute.Regex)
+			if err != nil {
+				continue
+			}
+
+			attribute.CompiledRegex = compiledRegex
+			compiledQueryPriority.Priorities[i].QueryAttributes[j] = attribute
+		}
+	}
+
+	return compiledQueryPriority
+}
+
+func GetPriority(requestParams url.Values, now time.Time, queryPriority validation.QueryPriority) int64 {
 	queryParam := requestParams.Get("query")
 	timeParam := requestParams.Get("time")
 	startParam := requestParams.Get("start")
@@ -22,19 +36,8 @@ func GetPriority(requestParams url.Values, now time.Time, queryPriority *validat
 		return queryPriority.DefaultPriority
 	}
 
-	for i, priority := range queryPriority.Priorities {
-		for j, attribute := range priority.QueryAttributes {
-			// If query priority config changed, re-populate the compiled regex
-			if queryPriorityChanged {
-				compiledRegex, err := regexp.Compile(attribute.Regex)
-				if err != nil {
-					continue
-				}
-
-				attribute.CompiledRegex = compiledRegex
-				queryPriority.Priorities[i].QueryAttributes[j] = attribute
-			}
-
+	for _, priority := range queryPriority.Priorities {
+		for _, attribute := range priority.QueryAttributes {
 			if attribute.CompiledRegex != nil && !attribute.CompiledRegex.MatchString(queryParam) {
 				continue
 			}
@@ -42,16 +45,16 @@ func GetPriority(requestParams url.Values, now time.Time, queryPriority *validat
 			startTimeThreshold := now.Add(-1 * attribute.StartTime.Abs()).Truncate(time.Second).UTC()
 			endTimeThreshold := now.Add(-1 * attribute.EndTime.Abs()).Add(1 * time.Second).Truncate(time.Second).UTC()
 
-			if startTime, err := parseTime(startParam); err == nil {
-				if endTime, err := parseTime(endParam); err == nil {
-					if isBetweenThresholds(startTime, endTime, startTimeThreshold, endTimeThreshold) {
+			if startTime, err := util.ParseTime(startParam); err == nil {
+				if endTime, err := util.ParseTime(endParam); err == nil {
+					if isBetweenThresholds(util.TimeFromMillis(startTime), util.TimeFromMillis(endTime), startTimeThreshold, endTimeThreshold) {
 						return priority.Priority
 					}
 				}
 			}
 
-			if instantTime, err := parseTime(timeParam); err == nil {
-				if isBetweenThresholds(instantTime, instantTime, startTimeThreshold, endTimeThreshold) {
+			if instantTime, err := util.ParseTime(timeParam); err == nil {
+				if isBetweenThresholds(util.TimeFromMillis(instantTime), util.TimeFromMillis(instantTime), startTimeThreshold, endTimeThreshold) {
 					return priority.Priority
 				}
 			}
@@ -67,21 +70,6 @@ func GetPriority(requestParams url.Values, now time.Time, queryPriority *validat
 	return queryPriority.DefaultPriority
 }
 
-func parseTime(s string) (time.Time, error) {
-	if s != "" {
-		if t, err := strconv.ParseFloat(s, 64); err == nil {
-			s, ns := math.Modf(t)
-			ns = math.Round(ns*1000) / 1000
-			return time.Unix(int64(s), int64(ns*float64(time.Second))).UTC(), nil
-		}
-		if t, err := time.Parse(time.RFC3339Nano, s); err == nil {
-			return t, nil
-		}
-	}
-
-	return time.Time{}, errors.Errorf("cannot parse %q to a valid timestamp", s)
-}
-
 func isBetweenThresholds(start, end, startThreshold, endThreshold time.Time) bool {
 	return (start.Equal(startThreshold) || start.After(startThreshold)) && (end.Equal(endThreshold) || end.Before(endThreshold))
 }
diff --git a/pkg/util/query/priority_test.go b/pkg/util/query/priority_test.go
index f1c9f4d185..646aed17c1 100644
--- a/pkg/util/query/priority_test.go
+++ b/pkg/util/query/priority_test.go
@@ -2,6 +2,7 @@ package query
 
 import (
 	"net/url"
+	"regexp"
 	"strconv"
 	"testing"
 	"time"
@@ -18,9 +19,10 @@ func Test_GetPriorityShouldReturnDefaultPriorityIfNotEnabledOrEmptyQueryString(t
 			Priority: 1,
 			QueryAttributes: []validation.QueryAttribute{
 				{
-					Regex:     ".*",
-					StartTime: 2 * time.Hour,
-					EndTime:   0 * time.Hour,
+					Regex:         ".*",
+					CompiledRegex: regexp.MustCompile(".*"),
+					StartTime:     2 * time.Hour,
+					EndTime:       0 * time.Hour,
 				},
 			},
 		},
@@ -32,13 +34,13 @@ func Test_GetPriorityShouldReturnDefaultPriorityIfNotEnabledOrEmptyQueryString(t
 	assert.Equal(t, int64(0), GetPriority(url.Values{
 		"query": []string{"sum(up)"},
 		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
-	}, now, &queryPriority, true))
+	}, now, queryPriority))
 
 	queryPriority.Enabled = true
 	assert.Equal(t, int64(0), GetPriority(url.Values{
 		"query": []string{""},
 		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
-	}, now, &queryPriority, true))
+	}, now, queryPriority))
 }
 
 func Test_GetPriorityShouldConsiderRegex(t *testing.T) {
@@ -48,9 +50,10 @@ func Test_GetPriorityShouldConsiderRegex(t *testing.T) {
 			Priority: 1,
 			QueryAttributes: []validation.QueryAttribute{
 				{
-					Regex:     "sum",
-					StartTime: 2 * time.Hour,
-					EndTime:   0 * time.Hour,
+					Regex:         "sum",
+					CompiledRegex: regexp.MustCompile("sum"),
+					StartTime:     2 * time.Hour,
+					EndTime:       0 * time.Hour,
 				},
 			},
 		},
@@ -60,96 +63,62 @@ func Test_GetPriorityShouldConsiderRegex(t *testing.T) {
 		Priorities: priorities,
 	}
 
-	assert.Nil(t, queryPriority.Priorities[0].QueryAttributes[0].CompiledRegex)
 	assert.Equal(t, int64(1), GetPriority(url.Values{
 		"query": []string{"sum(up)"},
 		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
-	}, now, &queryPriority, true))
-	assert.NotNil(t, queryPriority.Priorities[0].QueryAttributes[0].CompiledRegex)
+	}, now, queryPriority))
 	assert.Equal(t, int64(0), GetPriority(url.Values{
 		"query": []string{"count(up)"},
 		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
-	}, now, &queryPriority, false))
+	}, now, queryPriority))
 
 	queryPriority.Priorities[0].QueryAttributes[0].Regex = "(^sum$|c(.+)t)"
+	queryPriority.Priorities[0].QueryAttributes[0].CompiledRegex = regexp.MustCompile("(^sum$|c(.+)t)")
 
 	assert.Equal(t, int64(0), GetPriority(url.Values{
 		"query": []string{"sum(up)"},
 		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
-	}, now, &queryPriority, true))
+	}, now, queryPriority))
 	assert.Equal(t, int64(1), GetPriority(url.Values{
 		"query": []string{"count(up)"},
 		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
-	}, now, &queryPriority, false))
+	}, now, queryPriority))
 
 	queryPriority.Priorities[0].QueryAttributes[0].Regex = ".*"
+	queryPriority.Priorities[0].QueryAttributes[0].CompiledRegex = regexp.MustCompile(".*")
 
 	assert.Equal(t, int64(1), GetPriority(url.Values{
 		"query": []string{"sum(up)"},
 		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
-	}, now, &queryPriority, true))
+	}, now, queryPriority))
 	assert.Equal(t, int64(1), GetPriority(url.Values{
 		"query": []string{"count(up)"},
 		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
-	}, now, &queryPriority, false))
+	}, now, queryPriority))
 
 	queryPriority.Priorities[0].QueryAttributes[0].Regex = ".+"
+	queryPriority.Priorities[0].QueryAttributes[0].CompiledRegex = regexp.MustCompile(".+")
 
 	assert.Equal(t, int64(1), GetPriority(url.Values{
 		"query": []string{"sum(up)"},
 		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
-	}, now, &queryPriority, true))
+	}, now, queryPriority))
 	assert.Equal(t, int64(1), GetPriority(url.Values{
 		"query": []string{"count(up)"},
 		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
-	}, now, &queryPriority, false))
+	}, now, queryPriority))
 
 	queryPriority.Priorities[0].QueryAttributes[0].Regex = ""
+	queryPriority.Priorities[0].QueryAttributes[0].CompiledRegex = regexp.MustCompile("")
 
 	assert.Equal(t, int64(1), GetPriority(url.Values{
 		"query": []string{"sum(up)"},
 		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
-	}, now, &queryPriority, true))
+	}, now, queryPriority))
 	assert.Equal(t, int64(1), GetPriority(url.Values{
 		"query": []string{"count(up)"},
 		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
-	}, now, &queryPriority, false))
-}
-
-func Test_GetPriorityShouldNotRecompileRegexIfQueryPriorityChangedIsTrue(t *testing.T) {
-	now := time.Now()
-	priorities := []validation.PriorityDef{
-		{
-			Priority: 1,
-			QueryAttributes: []validation.QueryAttribute{
-				{
-					Regex:     "sum",
-					StartTime: 2 * time.Hour,
-					EndTime:   0 * time.Hour,
-				},
-			},
-		},
-	}
-	queryPriority := validation.QueryPriority{
-		Enabled:    true,
-		Priorities: priorities,
-	}
-
-	assert.Equal(t, int64(1), GetPriority(url.Values{
-		"query": []string{"sum(up)"},
-		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
-	}, now, &queryPriority, true))
-
-	queryPriority.Priorities[0].QueryAttributes[0].Regex = "count"
-
-	assert.Equal(t, int64(0), GetPriority(url.Values{
-		"query": []string{"count(up)"},
-		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
-	}, now, &queryPriority, false))
-	assert.Equal(t, int64(1), GetPriority(url.Values{
-		"query": []string{"count(up)"},
-		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
-	}, now, &queryPriority, true))
+	}, now, queryPriority))
 }
 
 func Test_GetPriorityShouldConsiderStartAndEndTime(t *testing.T) {
@@ -173,47 +142,47 @@ func Test_GetPriorityShouldConsiderStartAndEndTime(t *testing.T) {
 	assert.Equal(t, int64(0), GetPriority(url.Values{
 		"query": []string{"sum(up)"},
 		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
-	}, now, &queryPriority, true))
+	}, now, queryPriority))
 	assert.Equal(t, int64(1), GetPriority(url.Values{
 		"query": []string{"sum(up)"},
 		"time":  []string{strconv.FormatInt(now.Add(-30*time.Minute).Unix(), 10)},
-	}, now, &queryPriority, false))
+	}, now, queryPriority))
 	assert.Equal(t, int64(0), GetPriority(url.Values{
 		"query": []string{"sum(up)"},
 		"time":  []string{strconv.FormatInt(now.Add(-60*time.Minute).Unix(), 10)},
-	}, now, &queryPriority, false))
+	}, now, queryPriority))
 	assert.Equal(t, int64(1), GetPriority(url.Values{
 		"query": []string{"sum(up)"},
 		"time":  []string{strconv.FormatInt(now.Add(-45*time.Minute).Unix(), 10)},
-	}, now, &queryPriority, false))
+	}, now, queryPriority))
 	assert.Equal(t, int64(1), GetPriority(url.Values{
 		"query": []string{"sum(up)"},
 		"time":  []string{strconv.FormatInt(now.Add(-15*time.Minute).Unix(), 10)},
-	}, now, &queryPriority, false))
+	}, now, queryPriority))
 
 	assert.Equal(t, int64(1), GetPriority(url.Values{
 		"query": []string{"sum(up)"},
 		"start": []string{strconv.FormatInt(now.Add(-45*time.Minute).Unix(), 10)},
 		"end":   []string{strconv.FormatInt(now.Add(-15*time.Minute).Unix(), 10)},
-	}, now, &queryPriority, false))
+	}, now, queryPriority))
 	assert.Equal(t, int64(0), GetPriority(url.Values{
 		"query": []string{"sum(up)"},
 		"start": []string{strconv.FormatInt(now.Add(-50*time.Minute).Unix(), 10)},
 		"end":   []string{strconv.FormatInt(now.Add(-15*time.Minute).Unix(), 10)},
-	}, now, &queryPriority, false))
+	}, now, queryPriority))
 	assert.Equal(t, int64(0), GetPriority(url.Values{
 		"query": []string{"sum(up)"},
 		"start": []string{strconv.FormatInt(now.Add(-45*time.Minute).Unix(), 10)},
 		"end":   []string{strconv.FormatInt(now.Add(-10*time.Minute).Unix(), 10)},
-	}, now, &queryPriority, false))
+	}, now, queryPriority))
 	assert.Equal(t, int64(0), GetPriority(url.Values{
 		"query": []string{"sum(up)"},
 		"start": []string{strconv.FormatInt(now.Add(-60*time.Minute).Unix(), 10)},
 		"end":   []string{strconv.FormatInt(now.Add(-45*time.Minute).Unix(), 10)},
-	}, now, &queryPriority, false))
+	}, now, queryPriority))
 	assert.Equal(t, int64(0), GetPriority(url.Values{
 		"query": []string{"sum(up)"},
 		"start": []string{strconv.FormatInt(now.Add(-15*time.Minute).Unix(), 10)},
 		"end":   []string{strconv.FormatInt(now.Add(-1*time.Minute).Unix(), 10)},
-	}, now, &queryPriority, false))
+	}, now, queryPriority))
 }

From ab4b9c7902cf5f0a39f5a15b177bf76a84d645f0 Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Tue, 21 Nov 2023 10:52:49 -0800
Subject: [PATCH 39/49] Make reserved querier to handle priorities higher or
 equal

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 docs/configuration/config-file-reference.md    |  4 ++--
 pkg/scheduler/queue/queue.go                   |  4 ++--
 pkg/scheduler/queue/queue_test.go              |  6 +-----
 pkg/scheduler/queue/user_request_queue.go      |  4 ++--
 pkg/scheduler/queue/user_request_queue_test.go | 17 +++++++++++++++++
 pkg/util/validation/limits.go                  |  2 +-
 6 files changed, 25 insertions(+), 12 deletions(-)

diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md
index c4765c2948..1083b3fc6c 100644
--- a/docs/configuration/config-file-reference.md
+++ b/docs/configuration/config-file-reference.md
@@ -5050,8 +5050,8 @@ otel:
 # Priority level. Must be a unique value.
 [priority: <int> | default = 0]
 
-# Number of reserved queriers to handle this priority only. Value between 0 and
-# 1 will be used as a percentage.
+# Number of reserved queriers to handle priorities higher or equal to this value
+# only. Value between 0 and 1 will be used as a percentage.
 [reserved_queriers: <float> | default = 0]
 
 # List of query attributes to assign the priority.
diff --git a/pkg/scheduler/queue/queue.go b/pkg/scheduler/queue/queue.go
index 57f85bfb12..0f604fce8a 100644
--- a/pkg/scheduler/queue/queue.go
+++ b/pkg/scheduler/queue/queue.go
@@ -159,8 +159,8 @@ FindQueue:
 
 		// Pick next request from the queue.
 		for {
-			priority, matchPriority := q.getPriorityForQuerier(userID, querierID)
-			request := queue.dequeueRequest(priority, matchPriority)
+			minPriority, matchMinPriority := q.getPriorityForQuerier(userID, querierID)
+			request := queue.dequeueRequest(minPriority, matchMinPriority)
 			if request == nil {
 				// The queue does not contain request with the priority, wait for more requests
 				querierWait = true
diff --git a/pkg/scheduler/queue/queue_test.go b/pkg/scheduler/queue/queue_test.go
index 8c7c1f0794..36531b3886 100644
--- a/pkg/scheduler/queue/queue_test.go
+++ b/pkg/scheduler/queue/queue_test.go
@@ -219,10 +219,6 @@ func TestReservedQueriersShouldOnlyGetHighPriorityQueries(t *testing.T) {
 		id:       "priority 1",
 		priority: 1,
 	}
-	priority2Request := MockRequest{
-		id:       "priority 2",
-		priority: 2,
-	}
 
 	assert.NoError(t, queue.EnqueueRequest("userID", normalRequest, 1, func() {}))
 	assert.NoError(t, queue.EnqueueRequest("userID", priority1Request, 1, func() {}))
@@ -244,7 +240,7 @@ func TestReservedQueriersShouldOnlyGetHighPriorityQueries(t *testing.T) {
 	assert.Nil(t, nextRequest)
 	assert.Equal(t, 1, queue.queues.userQueues["userID"].queue.length())
 
-	assert.NoError(t, queue.EnqueueRequest("userID", priority2Request, 1, func() {}))
+	assert.NoError(t, queue.EnqueueRequest("userID", normalRequest, 1, func() {}))
 
 	ctxTimeout, cancel = context.WithTimeout(ctx, 1*time.Second)
 	defer cancel()
diff --git a/pkg/scheduler/queue/user_request_queue.go b/pkg/scheduler/queue/user_request_queue.go
index 828c9246fd..20588988c2 100644
--- a/pkg/scheduler/queue/user_request_queue.go
+++ b/pkg/scheduler/queue/user_request_queue.go
@@ -60,8 +60,8 @@ func (f *PriorityRequestQueue) enqueueRequest(r Request) {
 	}
 }
 
-func (f *PriorityRequestQueue) dequeueRequest(priority int64, matchPriority bool) Request {
-	if matchPriority && f.queue.Peek().Priority() != priority {
+func (f *PriorityRequestQueue) dequeueRequest(minPriority int64, checkMinPriority bool) Request {
+	if checkMinPriority && f.queue.Peek().Priority() < minPriority {
 		return nil
 	}
 	r := f.queue.Dequeue()
diff --git a/pkg/scheduler/queue/user_request_queue_test.go b/pkg/scheduler/queue/user_request_queue_test.go
index cabe792e06..f1f2314585 100644
--- a/pkg/scheduler/queue/user_request_queue_test.go
+++ b/pkg/scheduler/queue/user_request_queue_test.go
@@ -73,6 +73,10 @@ func TestPriorityRequestQueue(t *testing.T) {
 		id:       "request 2",
 		priority: 2,
 	}
+	request3 := MockRequest{
+		id:       "request 3",
+		priority: 3,
+	}
 
 	queue.enqueueRequest(request1)
 	queue.enqueueRequest(request2)
@@ -102,11 +106,22 @@ func TestPriorityRequestQueue(t *testing.T) {
 
 	queue.enqueueRequest(request1)
 	queue.enqueueRequest(request2)
+	queue.enqueueRequest(request3)
+	assert.NoError(t, promtest.GatherAndCompare(reg, strings.NewReader(`
+		# HELP cortex_query_scheduler_queue_length Number of queries in the queue.
+		# TYPE cortex_query_scheduler_queue_length gauge
+		cortex_query_scheduler_queue_length{priority="1",type="priority",user="userID"} 1
+		cortex_query_scheduler_queue_length{priority="2",type="priority",user="userID"} 1
+		cortex_query_scheduler_queue_length{priority="3",type="priority",user="userID"} 1
+	`), "cortex_query_scheduler_queue_length"))
+	assert.Equal(t, 3, queue.length())
+	assert.Equal(t, request3, queue.dequeueRequest(2, true))
 	assert.NoError(t, promtest.GatherAndCompare(reg, strings.NewReader(`
 		# HELP cortex_query_scheduler_queue_length Number of queries in the queue.
 		# TYPE cortex_query_scheduler_queue_length gauge
 		cortex_query_scheduler_queue_length{priority="1",type="priority",user="userID"} 1
 		cortex_query_scheduler_queue_length{priority="2",type="priority",user="userID"} 1
+		cortex_query_scheduler_queue_length{priority="3",type="priority",user="userID"} 0
 	`), "cortex_query_scheduler_queue_length"))
 	assert.Equal(t, 2, queue.length())
 	assert.Equal(t, request2, queue.dequeueRequest(2, true))
@@ -117,6 +132,7 @@ func TestPriorityRequestQueue(t *testing.T) {
 		# TYPE cortex_query_scheduler_queue_length gauge
 		cortex_query_scheduler_queue_length{priority="1",type="priority",user="userID"} 1
 		cortex_query_scheduler_queue_length{priority="2",type="priority",user="userID"} 0
+		cortex_query_scheduler_queue_length{priority="3",type="priority",user="userID"} 0
 	`), "cortex_query_scheduler_queue_length"))
 	assert.Equal(t, request1, queue.dequeueRequest(2, false))
 	assert.Equal(t, 0, queue.length())
@@ -125,5 +141,6 @@ func TestPriorityRequestQueue(t *testing.T) {
 		# TYPE cortex_query_scheduler_queue_length gauge
 		cortex_query_scheduler_queue_length{priority="1",type="priority",user="userID"} 0
 		cortex_query_scheduler_queue_length{priority="2",type="priority",user="userID"} 0
+		cortex_query_scheduler_queue_length{priority="3",type="priority",user="userID"} 0
 	`), "cortex_query_scheduler_queue_length"))
 }
diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go
index b1bebf1d42..d5adeb9085 100644
--- a/pkg/util/validation/limits.go
+++ b/pkg/util/validation/limits.go
@@ -56,7 +56,7 @@ type QueryPriority struct {
 
 type PriorityDef struct {
 	Priority         int64            `yaml:"priority" doc:"nocli|description=Priority level. Must be a unique value.|default=0"`
-	ReservedQueriers float64          `yaml:"reserved_queriers" doc:"nocli|description=Number of reserved queriers to handle this priority only. Value between 0 and 1 will be used as a percentage.|default=0"`
+	ReservedQueriers float64          `yaml:"reserved_queriers" doc:"nocli|description=Number of reserved queriers to handle priorities higher or equal to this value only. Value between 0 and 1 will be used as a percentage.|default=0"`
 	QueryAttributes  []QueryAttribute `yaml:"query_attributes" doc:"nocli|description=List of query attributes to assign the priority."`
 }
 

From 8086aee1b85c07ac8261d41e7d87a295c88953c4 Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Tue, 21 Nov 2023 11:11:26 -0800
Subject: [PATCH 40/49] Add benchmark tests

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 pkg/scheduler/queue/queue.go      |   8 +--
 pkg/scheduler/queue/queue_test.go | 103 +++++++++++++++++++++++++++++-
 2 files changed, 103 insertions(+), 8 deletions(-)

diff --git a/pkg/scheduler/queue/queue.go b/pkg/scheduler/queue/queue.go
index 0f604fce8a..0d634debed 100644
--- a/pkg/scheduler/queue/queue.go
+++ b/pkg/scheduler/queue/queue.go
@@ -106,14 +106,10 @@ func (q *RequestQueue) EnqueueRequest(userID string, req Request, maxQueriers fl
 		return errors.New("no queue found")
 	}
 
-	metricLabels := prometheus.Labels{
-		"user":     userID,
-		"priority": priority,
-	}
-	q.totalRequests.With(metricLabels).Inc()
+	q.totalRequests.WithLabelValues(userID, priority).Inc()
 
 	if queue.length() >= maxOutstandingRequests {
-		q.discardedRequests.With(metricLabels).Inc()
+		q.discardedRequests.WithLabelValues(userID, priority).Inc()
 		return ErrTooManyRequests
 	}
 
diff --git a/pkg/scheduler/queue/queue_test.go b/pkg/scheduler/queue/queue_test.go
index 36531b3886..52abe26270 100644
--- a/pkg/scheduler/queue/queue_test.go
+++ b/pkg/scheduler/queue/queue_test.go
@@ -84,8 +84,8 @@ func BenchmarkQueueRequest(b *testing.B) {
 
 	for n := 0; n < b.N; n++ {
 		q := NewRequestQueue(maxOutstandingPerTenant, 0,
-			prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user"}),
-			prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user"}),
+			prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user", "priority", "type"}),
+			prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user", "priority"}),
 			MockLimits{MaxOutstanding: 100},
 			nil,
 		)
@@ -115,6 +115,105 @@ func BenchmarkQueueRequest(b *testing.B) {
 	}
 }
 
+func BenchmarkGetNextRequestPriorityQueue(b *testing.B) {
+	const maxOutstandingPerTenant = 2
+	const numTenants = 50
+	const queriers = 5
+
+	queues := make([]*RequestQueue, 0, b.N)
+
+	for n := 0; n < b.N; n++ {
+		queue := NewRequestQueue(maxOutstandingPerTenant, 0,
+			prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user", "priority", "type"}),
+			prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user", "priority"}),
+			MockLimits{MaxOutstanding: 100, QueryPriorityVal: validation.QueryPriority{Enabled: true}},
+			nil,
+		)
+		queues = append(queues, queue)
+
+		for ix := 0; ix < queriers; ix++ {
+			queue.RegisterQuerierConnection(fmt.Sprintf("querier-%d", ix))
+		}
+
+		for i := 0; i < maxOutstandingPerTenant; i++ {
+			for j := 0; j < numTenants; j++ {
+				userID := strconv.Itoa(j)
+
+				err := queue.EnqueueRequest(userID, MockRequest{priority: int64(i)}, 0, nil)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		}
+	}
+
+	ctx := context.Background()
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		idx := FirstUser()
+		for j := 0; j < maxOutstandingPerTenant*numTenants; j++ {
+			querier := ""
+		b:
+			// Find querier with at least one request to avoid blocking in getNextRequestForQuerier.
+			for _, q := range queues[i].queues.userQueues {
+				for qid := range q.queriers {
+					querier = qid
+					break b
+				}
+			}
+
+			_, nidx, err := queues[i].GetNextRequestForQuerier(ctx, idx, querier)
+			if err != nil {
+				b.Fatal(err)
+			}
+			idx = nidx
+		}
+	}
+}
+
+func BenchmarkQueueRequestPriorityQueue(b *testing.B) {
+	const maxOutstandingPerTenant = 2
+	const numTenants = 50
+	const queriers = 5
+
+	queues := make([]*RequestQueue, 0, b.N)
+	users := make([]string, 0, numTenants)
+	requests := make([]MockRequest, 0, numTenants)
+
+	for n := 0; n < b.N; n++ {
+		q := NewRequestQueue(maxOutstandingPerTenant, 0,
+			prometheus.NewGaugeVec(prometheus.GaugeOpts{}, []string{"user", "priority", "type"}),
+			prometheus.NewCounterVec(prometheus.CounterOpts{}, []string{"user", "priority"}),
+			MockLimits{MaxOutstanding: 100, QueryPriorityVal: validation.QueryPriority{Enabled: true}},
+			nil,
+		)
+
+		for ix := 0; ix < queriers; ix++ {
+			q.RegisterQuerierConnection(fmt.Sprintf("querier-%d", ix))
+		}
+
+		queues = append(queues, q)
+
+		for j := 0; j < numTenants; j++ {
+			requests = append(requests, MockRequest{id: fmt.Sprintf("%d-%d", n, j), priority: int64(j)})
+			users = append(users, strconv.Itoa(j))
+		}
+	}
+
+	b.ResetTimer()
+	for n := 0; n < b.N; n++ {
+		for i := 0; i < maxOutstandingPerTenant; i++ {
+			for j := 0; j < numTenants; j++ {
+				err := queues[n].EnqueueRequest(users[j], requests[j], 0, nil)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		}
+	}
+}
+
 func TestRequestQueue_GetNextRequestForQuerier_ShouldGetRequestAfterReshardingBecauseQuerierHasBeenForgotten(t *testing.T) {
 	const forgetDelay = 3 * time.Second
 

From 584c6f94096ce09f486229605434eebe885fdbaa Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Wed, 22 Nov 2023 11:44:55 -0800
Subject: [PATCH 41/49] Update regex to be compiled upon unmarshal

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 pkg/frontend/transport/handler.go  |   5 ++
 pkg/frontend/v1/frontend.go        |  28 +------
 pkg/frontend/v2/frontend.go        |  28 +------
 pkg/util/query/priority.go         |  18 -----
 pkg/util/validation/limits.go      | 112 ++++++++++++++++++++-------
 pkg/util/validation/limits_test.go | 120 +++++++++++++++++++----------
 6 files changed, 169 insertions(+), 142 deletions(-)

diff --git a/pkg/frontend/transport/handler.go b/pkg/frontend/transport/handler.go
index d323a138cd..6bb21b92ba 100644
--- a/pkg/frontend/transport/handler.go
+++ b/pkg/frontend/transport/handler.go
@@ -201,6 +201,11 @@ func (f *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	}
 
 	startTime := time.Now()
+	// get config
+	// assign priority
+	// embed it to the http request, header?
+	// extract Decode to here, to make sure all requests pass here
+	// log the priority as well
 	resp, err := f.roundTripper.RoundTrip(r)
 	queryResponseTime := time.Since(startTime)
 
diff --git a/pkg/frontend/v1/frontend.go b/pkg/frontend/v1/frontend.go
index 8ea66fbd58..ef80fd07b1 100644
--- a/pkg/frontend/v1/frontend.go
+++ b/pkg/frontend/v1/frontend.go
@@ -6,8 +6,6 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
-	"reflect"
-	"sync"
 	"time"
 
 	"github.com/go-kit/log"
@@ -81,13 +79,6 @@ type Frontend struct {
 	requestQueue *queue.RequestQueue
 	activeUsers  *util.ActiveUsersCleanupService
 
-	// Used to check whether query priority config has changed
-	queryPriority    map[string]validation.QueryPriority
-	queryPriorityMtx map[string]*sync.RWMutex
-
-	// Populate and reuse compiled regex until query priority config changes
-	compiledQueryPriority map[string]validation.QueryPriority
-
 	// Subservices manager.
 	subservices        *services.Manager
 	subservicesWatcher *services.FailureWatcher
@@ -223,24 +214,7 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest,
 			queryPriority := f.limits.QueryPriority(userID)
 
 			if queryPriority.Enabled {
-				if _, exists := f.queryPriorityMtx[userID]; !exists {
-					f.queryPriorityMtx[userID] = &sync.RWMutex{}
-				}
-
-				f.queryPriorityMtx[userID].RLock()
-				queryPriorityChanged := !reflect.DeepEqual(f.queryPriority[userID], queryPriority)
-				f.queryPriorityMtx[userID].RUnlock()
-
-				if queryPriorityChanged {
-					f.queryPriorityMtx[userID].Lock()
-					f.queryPriority[userID] = queryPriority
-					f.compiledQueryPriority[userID] = util_query.GetCompileQueryPriority(queryPriority)
-					f.queryPriorityMtx[userID].Unlock()
-				}
-
-				f.queryPriorityMtx[userID].RLock()
-				request.priority = util_query.GetPriority(reqParams, ts, f.compiledQueryPriority[userID])
-				f.queryPriorityMtx[userID].Unlock()
+				request.priority = util_query.GetPriority(reqParams, ts, queryPriority)
 			}
 		}
 
diff --git a/pkg/frontend/v2/frontend.go b/pkg/frontend/v2/frontend.go
index 41dd4e7749..b11d8c04bc 100644
--- a/pkg/frontend/v2/frontend.go
+++ b/pkg/frontend/v2/frontend.go
@@ -7,7 +7,6 @@ import (
 	"math/rand"
 	"net/http"
 	"net/url"
-	"reflect"
 	"sync"
 	"time"
 
@@ -31,7 +30,6 @@ import (
 	util_log "github.com/cortexproject/cortex/pkg/util/log"
 	util_query "github.com/cortexproject/cortex/pkg/util/query"
 	"github.com/cortexproject/cortex/pkg/util/services"
-	"github.com/cortexproject/cortex/pkg/util/validation"
 )
 
 // Config for a Frontend.
@@ -76,13 +74,6 @@ type Frontend struct {
 
 	lastQueryID atomic.Uint64
 
-	// Used to check whether query priority config has changed
-	queryPriority    map[string]validation.QueryPriority
-	queryPriorityMtx map[string]*sync.RWMutex
-
-	// Populate and reuse compiled regex until query priority config changes
-	compiledQueryPriority map[string]validation.QueryPriority
-
 	// frontend workers will read from this channel, and send request to scheduler.
 	requestsCh chan *frontendRequest
 
@@ -223,24 +214,7 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest,
 			queryPriority := f.limits.QueryPriority(userID)
 
 			if queryPriority.Enabled {
-				if _, exists := f.queryPriorityMtx[userID]; !exists {
-					f.queryPriorityMtx[userID] = &sync.RWMutex{}
-				}
-
-				f.queryPriorityMtx[userID].RLock()
-				queryPriorityChanged := !reflect.DeepEqual(f.queryPriority[userID], queryPriority)
-				f.queryPriorityMtx[userID].RUnlock()
-
-				if queryPriorityChanged {
-					f.queryPriorityMtx[userID].Lock()
-					f.queryPriority[userID] = queryPriority
-					f.compiledQueryPriority[userID] = util_query.GetCompileQueryPriority(queryPriority)
-					f.queryPriorityMtx[userID].Unlock()
-				}
-
-				f.queryPriorityMtx[userID].RLock()
-				freq.priority = util_query.GetPriority(reqParams, ts, f.compiledQueryPriority[userID])
-				f.queryPriorityMtx[userID].Unlock()
+				freq.priority = util_query.GetPriority(reqParams, ts, queryPriority)
 			}
 		}
 
diff --git a/pkg/util/query/priority.go b/pkg/util/query/priority.go
index 503736f208..1a8b603d80 100644
--- a/pkg/util/query/priority.go
+++ b/pkg/util/query/priority.go
@@ -2,30 +2,12 @@ package query
 
 import (
 	"net/url"
-	"regexp"
 	"time"
 
 	"github.com/cortexproject/cortex/pkg/util"
 	"github.com/cortexproject/cortex/pkg/util/validation"
 )
 
-func GetCompileQueryPriority(queryPriority validation.QueryPriority) validation.QueryPriority {
-	compiledQueryPriority := queryPriority
-	for i, priority := range compiledQueryPriority.Priorities {
-		for j, attribute := range priority.QueryAttributes {
-			compiledRegex, err := regexp.Compile(attribute.Regex)
-			if err != nil {
-				continue
-			}
-
-			attribute.CompiledRegex = compiledRegex
-			compiledQueryPriority.Priorities[i].QueryAttributes[j] = attribute
-		}
-	}
-
-	return compiledQueryPriority
-}
-
 func GetPriority(requestParams url.Values, now time.Time, queryPriority validation.QueryPriority) int64 {
 	queryParam := requestParams.Get("query")
 	timeParam := requestParams.Get("time")
diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go
index d5adeb9085..35751c44fd 100644
--- a/pkg/util/validation/limits.go
+++ b/pkg/util/validation/limits.go
@@ -18,7 +18,8 @@ import (
 )
 
 var errMaxGlobalSeriesPerUserValidation = errors.New("The ingester.max-global-series-per-user limit is unsupported if distributor.shard-by-all-labels is disabled")
-var errDuplicateQueryPriorities = errors.New("There is a duplicate entry of priorities. Make sure they are all unique, including the default priority")
+var errDuplicateQueryPriorities = errors.New("duplicate entry of priorities found. Make sure they are all unique, including the default priority")
+var errCompilingQueryPriorityRegex = errors.New("error compiling query priority regex")
 
 // Supported values for enum limits
 const (
@@ -49,22 +50,22 @@ type DisabledRuleGroup struct {
 type DisabledRuleGroups []DisabledRuleGroup
 
 type QueryPriority struct {
-	Enabled         bool          `yaml:"enabled" doc:"nocli|description=Whether queries are assigned with priorities.|default=false"`
-	DefaultPriority int64         `yaml:"default_priority" doc:"nocli|description=Priority assigned to all queries by default. Must be a unique value. Use this as a baseline to make certain queries higher/lower priority.|default=0"`
-	Priorities      []PriorityDef `yaml:"priorities" doc:"nocli|description=List of priority definitions."`
+	Enabled         bool          `yaml:"enabled" json:"enabled" doc:"nocli|description=Whether queries are assigned with priorities.|default=false"`
+	DefaultPriority int64         `yaml:"default_priority" json:"default_priority" doc:"nocli|description=Priority assigned to all queries by default. Must be a unique value. Use this as a baseline to make certain queries higher/lower priority.|default=0"`
+	Priorities      []PriorityDef `yaml:"priorities" json:"priorities" doc:"nocli|description=List of priority definitions."`
 }
 
 type PriorityDef struct {
-	Priority         int64            `yaml:"priority" doc:"nocli|description=Priority level. Must be a unique value.|default=0"`
-	ReservedQueriers float64          `yaml:"reserved_queriers" doc:"nocli|description=Number of reserved queriers to handle priorities higher or equal to this value only. Value between 0 and 1 will be used as a percentage.|default=0"`
-	QueryAttributes  []QueryAttribute `yaml:"query_attributes" doc:"nocli|description=List of query attributes to assign the priority."`
+	Priority         int64            `yaml:"priority" json:"priority" doc:"nocli|description=Priority level. Must be a unique value.|default=0"`
+	ReservedQueriers float64          `yaml:"reserved_queriers" json:"reserved_queriers" doc:"nocli|description=Number of reserved queriers to handle priorities higher or equal to this value only. Value between 0 and 1 will be used as a percentage.|default=0"`
+	QueryAttributes  []QueryAttribute `yaml:"query_attributes" json:"query_attributes" doc:"nocli|description=List of query attributes to assign the priority."`
 }
 
 type QueryAttribute struct {
-	Regex         string         `yaml:"regex" doc:"nocli|description=Query string regex.|default=.*"`
-	CompiledRegex *regexp.Regexp `yaml:"-" doc:"nocli"`
-	StartTime     time.Duration  `yaml:"start_time" doc:"nocli|description=Query start time.|default=0s"`
-	EndTime       time.Duration  `yaml:"end_time" doc:"nocli|description=Query end time.|default=0s"`
+	Regex         string        `yaml:"regex" json:"regex" doc:"nocli|description=Query string regex.|default=.*"`
+	StartTime     time.Duration `yaml:"start_time" json:"start_time" doc:"nocli|description=Query start time.|default=0s"`
+	EndTime       time.Duration `yaml:"end_time" json:"end_time" doc:"nocli|description=Query end time.|default=0s"`
+	CompiledRegex *regexp.Regexp
 }
 
 // Limits describe all the limits for users; can be used to describe global default
@@ -122,8 +123,10 @@ type Limits struct {
 	QueryVerticalShardSize       int            `yaml:"query_vertical_shard_size" json:"query_vertical_shard_size" doc:"hidden"`
 
 	// Query Frontend / Scheduler enforced limits.
-	MaxOutstandingPerTenant int           `yaml:"max_outstanding_requests_per_tenant" json:"max_outstanding_requests_per_tenant"`
-	QueryPriority           QueryPriority `yaml:"query_priority" json:"query_priority" doc:"nocli|description=Configuration for query priority."`
+	MaxOutstandingPerTenant    int           `yaml:"max_outstanding_requests_per_tenant" json:"max_outstanding_requests_per_tenant"`
+	QueryPriority              QueryPriority `yaml:"query_priority" json:"query_priority" doc:"nocli|description=Configuration for query priority."`
+	queryPriorityRegexHash     string
+	queryPriorityCompiledRegex map[string]*regexp.Regexp
 
 	// Ruler defaults and limits.
 	RulerEvaluationDelay        model.Duration `yaml:"ruler_evaluation_delay_duration" json:"ruler_evaluation_delay_duration"`
@@ -250,20 +253,6 @@ func (l *Limits) Validate(shardByAllLabels bool) error {
 		return errMaxGlobalSeriesPerUserValidation
 	}
 
-	// If query priority is enabled, do not allow duplicate priority values
-	if l.QueryPriority.Enabled {
-		queryPriority := l.QueryPriority
-		prioritySet := map[int64]struct{}{}
-		prioritySet[queryPriority.DefaultPriority] = struct{}{}
-		for _, priority := range queryPriority.Priorities {
-			if _, exists := prioritySet[priority.Priority]; exists {
-				return errDuplicateQueryPriorities
-			}
-
-			prioritySet[priority.Priority] = struct{}{}
-		}
-	}
-
 	return nil
 }
 
@@ -280,7 +269,15 @@ func (l *Limits) UnmarshalYAML(unmarshal func(interface{}) error) error {
 		l.copyNotificationIntegrationLimits(defaultLimits.NotificationRateLimitPerIntegration)
 	}
 	type plain Limits
-	return unmarshal((*plain)(l))
+	if err := unmarshal((*plain)(l)); err != nil {
+		return err
+	}
+
+	if err := l.compileQueryPriorityRegex(); err != nil {
+		return err
+	}
+
+	return nil
 }
 
 // UnmarshalJSON implements the json.Unmarshaler interface.
@@ -298,7 +295,15 @@ func (l *Limits) UnmarshalJSON(data []byte) error {
 	dec := json.NewDecoder(bytes.NewReader(data))
 	dec.DisallowUnknownFields()
 
-	return dec.Decode((*plain)(l))
+	if err := dec.Decode((*plain)(l)); err != nil {
+		return err
+	}
+
+	if err := l.compileQueryPriorityRegex(); err != nil {
+		return err
+	}
+
+	return nil
 }
 
 func (l *Limits) copyNotificationIntegrationLimits(defaults NotificationRateLimitMap) {
@@ -308,6 +313,55 @@ func (l *Limits) copyNotificationIntegrationLimits(defaults NotificationRateLimi
 	}
 }
 
+func (l *Limits) hasQueryPriorityRegexChanged() bool {
+	var newHash string
+	for _, priority := range l.QueryPriority.Priorities {
+		for _, attribute := range priority.QueryAttributes {
+			newHash += attribute.Regex
+		}
+	}
+	if newHash != l.queryPriorityRegexHash {
+		l.queryPriorityRegexHash = newHash
+		return true
+	}
+	return false
+}
+
+func (l *Limits) compileQueryPriorityRegex() error {
+	if l.QueryPriority.Enabled {
+		hasQueryPriorityRegexChanged := l.hasQueryPriorityRegexChanged()
+		prioritySet := map[int64]struct{}{}
+		newQueryPriorityCompiledRegex := map[string]*regexp.Regexp{}
+
+		for i, priority := range l.QueryPriority.Priorities {
+			// Check for duplicate priority entry
+			if _, exists := prioritySet[priority.Priority]; exists {
+				return errDuplicateQueryPriorities
+			}
+			prioritySet[priority.Priority] = struct{}{}
+
+			for j, attribute := range priority.QueryAttributes {
+				if hasQueryPriorityRegexChanged {
+					compiledRegex, err := regexp.Compile(attribute.Regex)
+					if err != nil {
+						return errors.Join(errCompilingQueryPriorityRegex, err)
+					}
+					newQueryPriorityCompiledRegex[attribute.Regex] = compiledRegex
+					l.QueryPriority.Priorities[i].QueryAttributes[j].CompiledRegex = compiledRegex
+				} else {
+					l.QueryPriority.Priorities[i].QueryAttributes[j].CompiledRegex = l.queryPriorityCompiledRegex[attribute.Regex]
+				}
+			}
+		}
+
+		if hasQueryPriorityRegexChanged {
+			l.queryPriorityCompiledRegex = newQueryPriorityCompiledRegex
+		}
+	}
+
+	return nil
+}
+
 // When we load YAML from disk, we want the various per-customer limits
 // to default to any values specified on the command line, not default
 // command line values.  This global contains those values.  I (Tom) cannot
diff --git a/pkg/util/validation/limits_test.go b/pkg/util/validation/limits_test.go
index 01c18225c2..7d4542f491 100644
--- a/pkg/util/validation/limits_test.go
+++ b/pkg/util/validation/limits_test.go
@@ -3,6 +3,7 @@ package validation
 import (
 	"encoding/json"
 	"reflect"
+	"regexp"
 	"strings"
 	"testing"
 	"time"
@@ -61,47 +62,6 @@ func TestLimits_Validate(t *testing.T) {
 			shardByAllLabels: true,
 			expected:         nil,
 		},
-		"duplicate priority entries": {
-			limits: Limits{QueryPriority: QueryPriority{
-				Enabled: true,
-				Priorities: []PriorityDef{
-					{
-						Priority: 1,
-					},
-					{
-						Priority: 1,
-					},
-				},
-			}},
-			expected: errDuplicateQueryPriorities,
-		},
-		"priority matches default priority": {
-			limits: Limits{QueryPriority: QueryPriority{
-				Enabled:         true,
-				DefaultPriority: 1,
-				Priorities: []PriorityDef{
-					{
-						Priority: 1,
-					},
-				},
-			}},
-			expected: errDuplicateQueryPriorities,
-		},
-		"all priorities are unique": {
-			limits: Limits{QueryPriority: QueryPriority{
-				Enabled:         true,
-				DefaultPriority: 1,
-				Priorities: []PriorityDef{
-					{
-						Priority: 2,
-					},
-					{
-						Priority: 3,
-					},
-				},
-			}},
-			expected: nil,
-		},
 	}
 
 	for testName, testData := range tests {
@@ -637,3 +597,81 @@ tenant2:
 	require.Equal(t, 3, ov.MaxDownloadedBytesPerRequest("tenant2"))
 	require.Equal(t, 5, ov.MaxDownloadedBytesPerRequest("tenant3"))
 }
+
+func TestHasQueryPriorityRegexChanged(t *testing.T) {
+	l := Limits{
+		QueryPriority: QueryPriority{
+			Priorities: []PriorityDef{
+				{
+					Priority: 1,
+					QueryAttributes: []QueryAttribute{
+						{
+							Regex: "test",
+						},
+					},
+				},
+			},
+		},
+	}
+
+	require.True(t, l.hasQueryPriorityRegexChanged())
+
+	l.QueryPriority.Priorities[0].QueryAttributes[0].Regex = "new"
+
+	require.True(t, l.hasQueryPriorityRegexChanged())
+
+	l.QueryPriority.Priorities[0].QueryAttributes[0].StartTime = 2 * time.Hour
+
+	require.False(t, l.hasQueryPriorityRegexChanged())
+
+	l.QueryPriority.Priorities[0].QueryAttributes = append(l.QueryPriority.Priorities[0].QueryAttributes, QueryAttribute{Regex: "hi"})
+
+	require.True(t, l.hasQueryPriorityRegexChanged())
+
+	l.QueryPriority.Priorities[0].QueryAttributes = l.QueryPriority.Priorities[0].QueryAttributes[:1]
+
+	require.True(t, l.hasQueryPriorityRegexChanged())
+}
+
+func TestCompileQueryPriorityRegex(t *testing.T) {
+	l := Limits{
+		QueryPriority: QueryPriority{
+			Enabled: true,
+			Priorities: []PriorityDef{
+				{
+					Priority: 1,
+					QueryAttributes: []QueryAttribute{
+						{
+							Regex: "test",
+						},
+					},
+				},
+			},
+		},
+	}
+
+	require.Nil(t, l.QueryPriority.Priorities[0].QueryAttributes[0].CompiledRegex)
+
+	err := l.compileQueryPriorityRegex()
+	require.NoError(t, err)
+	require.Equal(t, regexp.MustCompile("test"), l.QueryPriority.Priorities[0].QueryAttributes[0].CompiledRegex)
+
+	l.QueryPriority.Priorities[0].QueryAttributes[0].Regex = "new"
+
+	err = l.compileQueryPriorityRegex()
+	require.NoError(t, err)
+	require.Equal(t, regexp.MustCompile("new"), l.QueryPriority.Priorities[0].QueryAttributes[0].CompiledRegex)
+
+	l.QueryPriority.Priorities[0].QueryAttributes[0].CompiledRegex = nil
+
+	err = l.compileQueryPriorityRegex()
+	require.NoError(t, err)
+	require.Equal(t, regexp.MustCompile("new"), l.QueryPriority.Priorities[0].QueryAttributes[0].CompiledRegex)
+
+	l.QueryPriority.Enabled = false
+	l.QueryPriority.Priorities[0].QueryAttributes[0].CompiledRegex = nil
+
+	err = l.compileQueryPriorityRegex()
+	require.NoError(t, err)
+	require.Nil(t, l.QueryPriority.Priorities[0].QueryAttributes[0].CompiledRegex)
+}

From 4001fe440a55eaeaa9b59155fddbd66ca7eddc1b Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Wed, 22 Nov 2023 14:13:16 -0800
Subject: [PATCH 42/49] Make start and end time check to be skipped if not
 specified

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 pkg/util/query/priority.go         | 33 +++++++++++++++-----
 pkg/util/query/priority_test.go    | 48 ++++++++++++++++++++++++++----
 pkg/util/validation/limits.go      |  6 ++--
 pkg/util/validation/limits_test.go |  2 +-
 4 files changed, 71 insertions(+), 18 deletions(-)

diff --git a/pkg/util/query/priority.go b/pkg/util/query/priority.go
index 1a8b603d80..2a7f61ba10 100644
--- a/pkg/util/query/priority.go
+++ b/pkg/util/query/priority.go
@@ -24,25 +24,22 @@ func GetPriority(requestParams url.Values, now time.Time, queryPriority validati
 				continue
 			}
 
-			startTimeThreshold := now.Add(-1 * attribute.StartTime.Abs()).Truncate(time.Second).UTC()
-			endTimeThreshold := now.Add(-1 * attribute.EndTime.Abs()).Add(1 * time.Second).Truncate(time.Second).UTC()
-
 			if startTime, err := util.ParseTime(startParam); err == nil {
 				if endTime, err := util.ParseTime(endParam); err == nil {
-					if isBetweenThresholds(util.TimeFromMillis(startTime), util.TimeFromMillis(endTime), startTimeThreshold, endTimeThreshold) {
+					if isWithinTimeAttributes(attribute, now, startTime, endTime) {
 						return priority.Priority
 					}
 				}
 			}
 
 			if instantTime, err := util.ParseTime(timeParam); err == nil {
-				if isBetweenThresholds(util.TimeFromMillis(instantTime), util.TimeFromMillis(instantTime), startTimeThreshold, endTimeThreshold) {
+				if isWithinTimeAttributes(attribute, now, instantTime, instantTime) {
 					return priority.Priority
 				}
 			}
 
 			if timeParam == "" {
-				if isBetweenThresholds(now, now, startTimeThreshold, endTimeThreshold) {
+				if isWithinTimeAttributes(attribute, now, util.TimeToMillis(now), util.TimeToMillis(now)) {
 					return priority.Priority
 				}
 			}
@@ -52,6 +49,26 @@ func GetPriority(requestParams url.Values, now time.Time, queryPriority validati
 	return queryPriority.DefaultPriority
 }
 
-func isBetweenThresholds(start, end, startThreshold, endThreshold time.Time) bool {
-	return (start.Equal(startThreshold) || start.After(startThreshold)) && (end.Equal(endThreshold) || end.Before(endThreshold))
+func isWithinTimeAttributes(attribute validation.QueryAttribute, now time.Time, startTime, endTime int64) bool {
+	if attribute.StartTime == 0 && attribute.EndTime == 0 {
+		return true
+	}
+
+	if attribute.StartTime != 0 {
+		startTimeThreshold := now.Add(-1 * time.Duration(attribute.StartTime).Abs()).Truncate(time.Second)
+
+		if util.TimeFromMillis(startTime).Before(startTimeThreshold) {
+			return false
+		}
+	}
+
+	if attribute.EndTime != 0 {
+		endTimeThreshold := now.Add(-1 * time.Duration(attribute.EndTime).Abs()).Add(1 * time.Second).Truncate(time.Second)
+
+		if util.TimeFromMillis(endTime).After(endTimeThreshold) {
+			return false
+		}
+	}
+
+	return true
 }
diff --git a/pkg/util/query/priority_test.go b/pkg/util/query/priority_test.go
index 646aed17c1..fea552cde4 100644
--- a/pkg/util/query/priority_test.go
+++ b/pkg/util/query/priority_test.go
@@ -7,6 +7,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/assert"
 
 	"github.com/cortexproject/cortex/pkg/util/validation"
@@ -21,8 +22,6 @@ func Test_GetPriorityShouldReturnDefaultPriorityIfNotEnabledOrEmptyQueryString(t
 				{
 					Regex:         ".*",
 					CompiledRegex: regexp.MustCompile(".*"),
-					StartTime:     2 * time.Hour,
-					EndTime:       0 * time.Hour,
 				},
 			},
 		},
@@ -52,8 +51,6 @@ func Test_GetPriorityShouldConsiderRegex(t *testing.T) {
 				{
 					Regex:         "sum",
 					CompiledRegex: regexp.MustCompile("sum"),
-					StartTime:     2 * time.Hour,
-					EndTime:       0 * time.Hour,
 				},
 			},
 		},
@@ -128,8 +125,8 @@ func Test_GetPriorityShouldConsiderStartAndEndTime(t *testing.T) {
 			Priority: 1,
 			QueryAttributes: []validation.QueryAttribute{
 				{
-					StartTime: 45 * time.Minute,
-					EndTime:   15 * time.Minute,
+					StartTime: model.Duration(45 * time.Minute),
+					EndTime:   model.Duration(15 * time.Minute),
 				},
 			},
 		},
@@ -186,3 +183,42 @@ func Test_GetPriorityShouldConsiderStartAndEndTime(t *testing.T) {
 		"end":   []string{strconv.FormatInt(now.Add(-1*time.Minute).Unix(), 10)},
 	}, now, queryPriority))
 }
+
+func Test_GetPriorityShouldSKipStartAndEndTimeIfEmpty(t *testing.T) {
+	now := time.Now()
+	priorities := []validation.PriorityDef{
+		{
+			Priority: 1,
+			QueryAttributes: []validation.QueryAttribute{
+				{
+					Regex: "^test$",
+				},
+			},
+		},
+	}
+	queryPriority := validation.QueryPriority{
+		Enabled:    true,
+		Priorities: priorities,
+	}
+
+	assert.Equal(t, int64(1), GetPriority(url.Values{
+		"query": []string{"test"},
+	}, now, queryPriority))
+	assert.Equal(t, int64(1), GetPriority(url.Values{
+		"query": []string{"test"},
+		"time":  []string{strconv.FormatInt(now.Add(8760*time.Hour).Unix(), 10)},
+	}, now, queryPriority))
+	assert.Equal(t, int64(1), GetPriority(url.Values{
+		"query": []string{"test"},
+		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
+	}, now, queryPriority))
+	assert.Equal(t, int64(1), GetPriority(url.Values{
+		"query": []string{"test"},
+		"time":  []string{strconv.FormatInt(now.Add(-8760*time.Hour).Unix(), 10)},
+	}, now, queryPriority))
+	assert.Equal(t, int64(1), GetPriority(url.Values{
+		"query": []string{"test"},
+		"start": []string{strconv.FormatInt(now.Add(-100000*time.Minute).Unix(), 10)},
+		"end":   []string{strconv.FormatInt(now.Add(100000*time.Minute).Unix(), 10)},
+	}, now, queryPriority))
+}
diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go
index 35751c44fd..b9b005d0df 100644
--- a/pkg/util/validation/limits.go
+++ b/pkg/util/validation/limits.go
@@ -62,9 +62,9 @@ type PriorityDef struct {
 }
 
 type QueryAttribute struct {
-	Regex         string        `yaml:"regex" json:"regex" doc:"nocli|description=Query string regex.|default=.*"`
-	StartTime     time.Duration `yaml:"start_time" json:"start_time" doc:"nocli|description=Query start time.|default=0s"`
-	EndTime       time.Duration `yaml:"end_time" json:"end_time" doc:"nocli|description=Query end time.|default=0s"`
+	Regex         string         `yaml:"regex" json:"regex" doc:"nocli|description=Query string regex. If set to empty string, it will not match anything.|default=\"\""`
+	StartTime     model.Duration `yaml:"start_time" json:"start_time" doc:"nocli|description=Query start time. If set to 0, the start time won't be checked.'.|default=0"`
+	EndTime       model.Duration `yaml:"end_time" json:"end_time" doc:"nocli|description=Query end time. If set to 0, the end time won't be checked.|default=0"`
 	CompiledRegex *regexp.Regexp
 }
 
diff --git a/pkg/util/validation/limits_test.go b/pkg/util/validation/limits_test.go
index 7d4542f491..32a0bf924c 100644
--- a/pkg/util/validation/limits_test.go
+++ b/pkg/util/validation/limits_test.go
@@ -620,7 +620,7 @@ func TestHasQueryPriorityRegexChanged(t *testing.T) {
 
 	require.True(t, l.hasQueryPriorityRegexChanged())
 
-	l.QueryPriority.Priorities[0].QueryAttributes[0].StartTime = 2 * time.Hour
+	l.QueryPriority.Priorities[0].QueryAttributes[0].StartTime = model.Duration(2 * time.Hour)
 
 	require.False(t, l.hasQueryPriorityRegexChanged())
 

From aa38a8c556756af3337a3deb8537758cceaad585 Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Thu, 23 Nov 2023 16:34:44 -0800
Subject: [PATCH 43/49] Assign priority before splitting the query

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 docs/configuration/config-file-reference.md   |  16 +-
 pkg/frontend/transport/handler.go             |   9 +-
 pkg/frontend/transport/roundtripper.go        |  20 +-
 pkg/frontend/v1/frontend.go                   |  27 +-
 pkg/frontend/v2/frontend.go                   |  13 +-
 pkg/frontend/v2/frontend_scheduler_worker.go  |   1 -
 pkg/frontend/v2/frontend_test.go              |  13 +-
 .../tripperware/instantquery/instant_query.go |  18 +-
 pkg/querier/tripperware/limits.go             |   9 +-
 pkg/querier/tripperware/priority.go           |  94 ++++++
 pkg/querier/tripperware/priority_test.go      | 277 ++++++++++++++++++
 .../tripperware/queryrange/limits_test.go     |   5 +
 pkg/querier/tripperware/roundtrip.go          |  23 +-
 .../tripperware/test_shard_by_query_utils.go  |   6 +
 pkg/scheduler/scheduler.go                    |  10 +-
 pkg/scheduler/schedulerpb/scheduler.pb.go     | 127 +++-----
 pkg/scheduler/schedulerpb/scheduler.proto     |   1 -
 pkg/util/http.go                              |   1 +
 pkg/util/httpgrpcutil/header.go               |  22 ++
 pkg/util/query/priority.go                    |  74 -----
 pkg/util/query/priority_test.go               | 224 --------------
 pkg/util/time.go                              |  14 +
 pkg/util/validation/limits.go                 |   6 +-
 tools/doc-generator/parser.go                 |   4 +
 24 files changed, 533 insertions(+), 481 deletions(-)
 create mode 100644 pkg/querier/tripperware/priority.go
 create mode 100644 pkg/querier/tripperware/priority_test.go
 create mode 100644 pkg/util/httpgrpcutil/header.go
 delete mode 100644 pkg/util/query/priority.go
 delete mode 100644 pkg/util/query/priority_test.go

diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md
index 1083b3fc6c..25286293dd 100644
--- a/docs/configuration/config-file-reference.md
+++ b/docs/configuration/config-file-reference.md
@@ -5050,8 +5050,8 @@ otel:
 # Priority level. Must be a unique value.
 [priority: <int> | default = 0]
 
-# Number of reserved queriers to handle priorities higher or equal to this value
-# only. Value between 0 and 1 will be used as a percentage.
+# Number of reserved queriers to handle priorities higher or equal to the
+# priority level. Value between 0 and 1 will be used as a percentage.
 [reserved_queriers: <float> | default = 0]
 
 # List of query attributes to assign the priority.
@@ -5061,14 +5061,14 @@ otel:
 ### `QueryAttribute`
 
 ```yaml
-# Query string regex.
-[regex: <string> | default = ".*"]
+# Query string regex. If set to empty string, it will not match anything.
+[regex: <string> | default = ""]
 
-# Query start time.
-[start_time: <duration> | default = 0s]
+# Query start time. If set to 0, the start time won't be checked.
+[start_time: <int> | default = 0]
 
-# Query end time.
-[end_time: <duration> | default = 0s]
+# Query end time. If set to 0, the end time won't be checked.
+[end_time: <int> | default = 0]
 ```
 
 ### `DisabledRuleGroup`
diff --git a/pkg/frontend/transport/handler.go b/pkg/frontend/transport/handler.go
index 6bb21b92ba..042dbd32b7 100644
--- a/pkg/frontend/transport/handler.go
+++ b/pkg/frontend/transport/handler.go
@@ -200,12 +200,8 @@ func (f *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		r.Body = io.NopCloser(&buf)
 	}
 
+	r.Header.Get("test")
 	startTime := time.Now()
-	// get config
-	// assign priority
-	// embed it to the http request, header?
-	// extract Decode to here, to make sure all requests pass here
-	// log the priority as well
 	resp, err := f.roundTripper.RoundTrip(r)
 	queryResponseTime := time.Since(startTime)
 
@@ -348,6 +344,9 @@ func (f *Handler) reportQueryStats(r *http.Request, userID string, queryString u
 	if ua := r.Header.Get("User-Agent"); len(ua) > 0 {
 		logMessage = append(logMessage, "user_agent", ua)
 	}
+	if queryPriority := r.Header.Get(util.QueryPriorityHeaderKey); len(queryPriority) > 0 {
+		logMessage = append(logMessage, "priority", queryPriority)
+	}
 
 	if error != nil {
 		s, ok := status.FromError(error)
diff --git a/pkg/frontend/transport/roundtripper.go b/pkg/frontend/transport/roundtripper.go
index a0529ba6a4..583fc22d04 100644
--- a/pkg/frontend/transport/roundtripper.go
+++ b/pkg/frontend/transport/roundtripper.go
@@ -5,9 +5,6 @@ import (
 	"context"
 	"io"
 	"net/http"
-	"net/url"
-	"strings"
-	"time"
 
 	"github.com/weaveworks/common/httpgrpc"
 	"github.com/weaveworks/common/httpgrpc/server"
@@ -15,7 +12,7 @@ import (
 
 // GrpcRoundTripper is similar to http.RoundTripper, but works with HTTP requests converted to protobuf messages.
 type GrpcRoundTripper interface {
-	RoundTripGRPC(context.Context, *httpgrpc.HTTPRequest, url.Values, time.Time) (*httpgrpc.HTTPResponse, error)
+	RoundTripGRPC(context.Context, *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error)
 }
 
 func AdaptGrpcRoundTripperToHTTPRoundTripper(r GrpcRoundTripper) http.RoundTripper {
@@ -42,20 +39,7 @@ func (a *grpcRoundTripperAdapter) RoundTrip(r *http.Request) (*http.Response, er
 		return nil, err
 	}
 
-	var (
-		resp      *httpgrpc.HTTPResponse
-		reqValues url.Values
-		ts        time.Time
-	)
-
-	if strings.HasSuffix(r.URL.Path, "/query") || strings.HasSuffix(r.URL.Path, "/query_range") {
-		if err = r.ParseForm(); err == nil {
-			reqValues = r.Form
-			ts = time.Now()
-		}
-	}
-
-	resp, err = a.roundTripper.RoundTripGRPC(r.Context(), req, reqValues, ts)
+	resp, err := a.roundTripper.RoundTripGRPC(r.Context(), req)
 	if err != nil {
 		return nil, err
 	}
diff --git a/pkg/frontend/v1/frontend.go b/pkg/frontend/v1/frontend.go
index ef80fd07b1..024c1f961a 100644
--- a/pkg/frontend/v1/frontend.go
+++ b/pkg/frontend/v1/frontend.go
@@ -5,7 +5,7 @@ import (
 	"flag"
 	"fmt"
 	"net/http"
-	"net/url"
+	"strconv"
 	"time"
 
 	"github.com/go-kit/log"
@@ -23,7 +23,6 @@ import (
 	"github.com/cortexproject/cortex/pkg/tenant"
 	"github.com/cortexproject/cortex/pkg/util"
 	"github.com/cortexproject/cortex/pkg/util/httpgrpcutil"
-	util_query "github.com/cortexproject/cortex/pkg/util/query"
 	"github.com/cortexproject/cortex/pkg/util/services"
 	"github.com/cortexproject/cortex/pkg/util/validation"
 )
@@ -98,11 +97,15 @@ type request struct {
 	request  *httpgrpc.HTTPRequest
 	err      chan error
 	response chan *httpgrpc.HTTPResponse
-	priority int64
 }
 
 func (r request) Priority() int64 {
-	return r.priority
+	priority, err := strconv.ParseInt(httpgrpcutil.GetHeader(*r.request, util.QueryPriorityHeaderKey), 10, 64)
+	if err != nil {
+		return 0
+	}
+
+	return priority
 }
 
 // New creates a new frontend. Frontend implements service, and must be started and stopped.
@@ -181,7 +184,7 @@ func (f *Frontend) cleanupInactiveUserMetrics(user string) {
 }
 
 // RoundTripGRPC round trips a proto (instead of a HTTP request).
-func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest, reqParams url.Values, ts time.Time) (*httpgrpc.HTTPResponse, error) {
+func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error) {
 	// Propagate trace context in gRPC too - this will be ignored if using HTTP.
 	tracer, span := opentracing.GlobalTracer(), opentracing.SpanFromContext(ctx)
 	if tracer != nil && span != nil {
@@ -192,12 +195,6 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest,
 		}
 	}
 
-	tenantIDs, err := tenant.TenantIDs(ctx)
-	if err != nil {
-		return nil, err
-	}
-	userID := tenant.JoinTenantIDs(tenantIDs)
-
 	return f.retry.Do(ctx, func() (*httpgrpc.HTTPResponse, error) {
 		request := request{
 			request:     req,
@@ -210,14 +207,6 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest,
 			response: make(chan *httpgrpc.HTTPResponse, 1),
 		}
 
-		if reqParams != nil {
-			queryPriority := f.limits.QueryPriority(userID)
-
-			if queryPriority.Enabled {
-				request.priority = util_query.GetPriority(reqParams, ts, queryPriority)
-			}
-		}
-
 		if err := f.queueRequest(ctx, &request); err != nil {
 			return nil, err
 		}
diff --git a/pkg/frontend/v2/frontend.go b/pkg/frontend/v2/frontend.go
index b11d8c04bc..2df0f8f344 100644
--- a/pkg/frontend/v2/frontend.go
+++ b/pkg/frontend/v2/frontend.go
@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"math/rand"
 	"net/http"
-	"net/url"
 	"sync"
 	"time"
 
@@ -28,7 +27,6 @@ import (
 	"github.com/cortexproject/cortex/pkg/util/grpcclient"
 	"github.com/cortexproject/cortex/pkg/util/httpgrpcutil"
 	util_log "github.com/cortexproject/cortex/pkg/util/log"
-	util_query "github.com/cortexproject/cortex/pkg/util/query"
 	"github.com/cortexproject/cortex/pkg/util/services"
 )
 
@@ -89,7 +87,6 @@ type frontendRequest struct {
 	request      *httpgrpc.HTTPRequest
 	userID       string
 	statsEnabled bool
-	priority     int64
 
 	cancel context.CancelFunc
 
@@ -170,7 +167,7 @@ func (f *Frontend) stopping(_ error) error {
 }
 
 // RoundTripGRPC round trips a proto (instead of a HTTP request).
-func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest, reqParams url.Values, ts time.Time) (*httpgrpc.HTTPResponse, error) {
+func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest) (*httpgrpc.HTTPResponse, error) {
 	if s := f.State(); s != services.Running {
 		return nil, fmt.Errorf("frontend not running: %v", s)
 	}
@@ -210,14 +207,6 @@ func (f *Frontend) RoundTripGRPC(ctx context.Context, req *httpgrpc.HTTPRequest,
 			retryOnTooManyOutstandingRequests: f.cfg.RetryOnTooManyOutstandingRequests && f.schedulerWorkers.getWorkersCount() > 1,
 		}
 
-		if reqParams != nil {
-			queryPriority := f.limits.QueryPriority(userID)
-
-			if queryPriority.Enabled {
-				freq.priority = util_query.GetPriority(reqParams, ts, queryPriority)
-			}
-		}
-
 		f.requests.put(freq)
 		defer f.requests.delete(freq.queryID)
 
diff --git a/pkg/frontend/v2/frontend_scheduler_worker.go b/pkg/frontend/v2/frontend_scheduler_worker.go
index 2abff24bf6..bbe1e2ed74 100644
--- a/pkg/frontend/v2/frontend_scheduler_worker.go
+++ b/pkg/frontend/v2/frontend_scheduler_worker.go
@@ -263,7 +263,6 @@ func (w *frontendSchedulerWorker) schedulerLoop(loop schedulerpb.SchedulerForFro
 				HttpRequest:     req.request,
 				FrontendAddress: w.frontendAddr,
 				StatsEnabled:    req.statsEnabled,
-				Priority:        req.priority,
 			})
 
 			if err != nil {
diff --git a/pkg/frontend/v2/frontend_test.go b/pkg/frontend/v2/frontend_test.go
index 4c40d88b66..6b20926e89 100644
--- a/pkg/frontend/v2/frontend_test.go
+++ b/pkg/frontend/v2/frontend_test.go
@@ -3,7 +3,6 @@ package v2
 import (
 	"context"
 	"net"
-	"net/url"
 	"strconv"
 	"strings"
 	"sync"
@@ -112,7 +111,7 @@ func TestFrontendBasicWorkflow(t *testing.T) {
 		return &schedulerpb.SchedulerToFrontend{Status: schedulerpb.OK}
 	}, 0)
 
-	resp, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), &httpgrpc.HTTPRequest{}, url.Values{}, time.Now())
+	resp, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), &httpgrpc.HTTPRequest{})
 	require.NoError(t, err)
 	require.Equal(t, int32(200), resp.Code)
 	require.Equal(t, []byte(body), resp.Body)
@@ -142,7 +141,7 @@ func TestFrontendRetryRequest(t *testing.T) {
 		return &schedulerpb.SchedulerToFrontend{Status: schedulerpb.OK}
 	}, 3)
 
-	res, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), &httpgrpc.HTTPRequest{}, url.Values{}, time.Now())
+	res, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), &httpgrpc.HTTPRequest{})
 	require.NoError(t, err)
 	require.Equal(t, int32(200), res.Code)
 }
@@ -169,7 +168,7 @@ func TestFrontendRetryEnqueue(t *testing.T) {
 		return &schedulerpb.SchedulerToFrontend{Status: schedulerpb.OK}
 	}, 0)
 
-	_, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), &httpgrpc.HTTPRequest{}, url.Values{}, time.Now())
+	_, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), userID), &httpgrpc.HTTPRequest{})
 	require.NoError(t, err)
 }
 
@@ -178,7 +177,7 @@ func TestFrontendEnqueueFailure(t *testing.T) {
 		return &schedulerpb.SchedulerToFrontend{Status: schedulerpb.SHUTTING_DOWN}
 	}, 0)
 
-	_, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), "test"), &httpgrpc.HTTPRequest{}, url.Values{}, time.Now())
+	_, err := f.RoundTripGRPC(user.InjectOrgID(context.Background(), "test"), &httpgrpc.HTTPRequest{})
 	require.Error(t, err)
 	require.True(t, strings.Contains(err.Error(), "failed to enqueue request"))
 }
@@ -189,7 +188,7 @@ func TestFrontendCancellation(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
 	defer cancel()
 
-	resp, err := f.RoundTripGRPC(user.InjectOrgID(ctx, "test"), &httpgrpc.HTTPRequest{}, url.Values{}, time.Now())
+	resp, err := f.RoundTripGRPC(user.InjectOrgID(ctx, "test"), &httpgrpc.HTTPRequest{})
 	require.EqualError(t, err, context.DeadlineExceeded.Error())
 	require.Nil(t, resp)
 
@@ -238,7 +237,7 @@ func TestFrontendFailedCancellation(t *testing.T) {
 	}()
 
 	// send request
-	resp, err := f.RoundTripGRPC(user.InjectOrgID(ctx, "test"), &httpgrpc.HTTPRequest{}, url.Values{}, time.Now())
+	resp, err := f.RoundTripGRPC(user.InjectOrgID(ctx, "test"), &httpgrpc.HTTPRequest{})
 	require.EqualError(t, err, context.Canceled.Error())
 	require.Nil(t, resp)
 
diff --git a/pkg/querier/tripperware/instantquery/instant_query.go b/pkg/querier/tripperware/instantquery/instant_query.go
index a7350e65d5..60bea7a8d1 100644
--- a/pkg/querier/tripperware/instantquery/instant_query.go
+++ b/pkg/querier/tripperware/instantquery/instant_query.go
@@ -4,18 +4,17 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"github.com/cortexproject/cortex/pkg/util"
 	"io"
 	"net/http"
 	"net/url"
 	"sort"
-	"strconv"
 	"strings"
 	"time"
 
 	jsoniter "github.com/json-iterator/go"
 	"github.com/opentracing/opentracing-go"
 	otlog "github.com/opentracing/opentracing-go/log"
-	"github.com/pkg/errors"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/model/timestamp"
@@ -26,7 +25,6 @@ import (
 	"github.com/cortexproject/cortex/pkg/cortexpb"
 	"github.com/cortexproject/cortex/pkg/querier/tripperware"
 	"github.com/cortexproject/cortex/pkg/querier/tripperware/queryrange"
-	"github.com/cortexproject/cortex/pkg/util"
 	"github.com/cortexproject/cortex/pkg/util/spanlogger"
 )
 
@@ -132,7 +130,7 @@ func (resp *PrometheusInstantQueryResponse) HTTPHeaders() map[string][]string {
 func (c instantQueryCodec) DecodeRequest(_ context.Context, r *http.Request, forwardHeaders []string) (tripperware.Request, error) {
 	result := PrometheusRequest{Headers: map[string][]string{}}
 	var err error
-	result.Time, err = parseTimeParam(r, "time", c.now().Unix())
+	result.Time, err = util.ParseTimeParam(r, "time", c.now().Unix())
 	if err != nil {
 		return nil, decorateWithParamName(err, "time")
 	}
@@ -630,15 +628,3 @@ func (s *PrometheusInstantQueryData) MarshalJSON() ([]byte, error) {
 		return s.Result.GetRawBytes(), nil
 	}
 }
-
-func parseTimeParam(r *http.Request, paramName string, defaultValue int64) (int64, error) {
-	val := r.FormValue(paramName)
-	if val == "" {
-		val = strconv.FormatInt(defaultValue, 10)
-	}
-	result, err := util.ParseTime(val)
-	if err != nil {
-		return 0, errors.Wrapf(err, "Invalid time value for '%s'", paramName)
-	}
-	return result, nil
-}
diff --git a/pkg/querier/tripperware/limits.go b/pkg/querier/tripperware/limits.go
index 15ce78592f..815693b3c1 100644
--- a/pkg/querier/tripperware/limits.go
+++ b/pkg/querier/tripperware/limits.go
@@ -1,6 +1,10 @@
 package tripperware
 
-import "time"
+import (
+	"time"
+
+	"github.com/cortexproject/cortex/pkg/util/validation"
+)
 
 // Limits allows us to specify per-tenant runtime limits on the behavior of
 // the query handling code.
@@ -21,4 +25,7 @@ type Limits interface {
 
 	// QueryVerticalShardSize returns the maximum number of queriers that can handle requests for this user.
 	QueryVerticalShardSize(userID string) int
+
+	// QueryPriority returns the query priority config for the tenant, including different priorities and their attributes.
+	QueryPriority(userID string) validation.QueryPriority
 }
diff --git a/pkg/querier/tripperware/priority.go b/pkg/querier/tripperware/priority.go
new file mode 100644
index 0000000000..5ce4e0d3fa
--- /dev/null
+++ b/pkg/querier/tripperware/priority.go
@@ -0,0 +1,94 @@
+package tripperware
+
+import (
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/prometheus/prometheus/promql/parser"
+
+	"github.com/cortexproject/cortex/pkg/util"
+	"github.com/cortexproject/cortex/pkg/util/validation"
+)
+
+func GetPriority(r *http.Request, userID string, limits Limits, now time.Time) (int64, error) {
+	isQuery := strings.HasSuffix(r.URL.Path, "/query")
+	isQueryRange := strings.HasSuffix(r.URL.Path, "/query_range")
+	queryPriority := limits.QueryPriority(userID)
+	query := r.FormValue("query")
+
+	if (!isQuery && !isQueryRange) || !queryPriority.Enabled || query == "" {
+		return 0, nil
+	}
+
+	expr, err := parser.ParseExpr(query)
+	if err != nil {
+		return 0, err
+	}
+
+	var startTime, endTime int64
+	if isQuery {
+		if t, err := util.ParseTimeParam(r, "time", now.Unix()); err == nil {
+			startTime = t
+			endTime = t
+		}
+	} else if isQueryRange {
+		if st, err := util.ParseTime(r.FormValue("start")); err == nil {
+			if et, err := util.ParseTime(r.FormValue("end")); err == nil {
+				startTime = st
+				endTime = et
+			}
+		}
+	}
+
+	es := &parser.EvalStmt{
+		Expr:          expr,
+		Start:         util.TimeFromMillis(startTime),
+		End:           util.TimeFromMillis(endTime),
+		LookbackDelta: limits.MaxQueryLookback(userID), // this is available from querier flag.
+	}
+
+	minTime, maxTime := FindMinMaxTime(es)
+
+	for _, priority := range queryPriority.Priorities {
+		for _, attribute := range priority.QueryAttributes {
+			if attribute.Regex == "" || (attribute.CompiledRegex != nil && !attribute.CompiledRegex.MatchString(query)) {
+				continue
+			}
+
+			if isWithinTimeAttributes(attribute, now, minTime, maxTime) {
+				return priority.Priority, nil
+			}
+		}
+	}
+
+	return queryPriority.DefaultPriority, nil
+}
+
+func isWithinTimeAttributes(attribute validation.QueryAttribute, now time.Time, startTime, endTime int64) bool {
+	if attribute.StartTime == 0 && attribute.EndTime == 0 {
+		return true
+	}
+
+	if attribute.StartTime != 0 {
+		startTimeThreshold := now.Add(-1 * time.Duration(attribute.StartTime).Abs()).Truncate(time.Second).Unix()
+		if startTime < startTimeThreshold {
+			return false
+		}
+	}
+
+	if attribute.EndTime != 0 {
+		endTimeThreshold := now.Add(-1 * time.Duration(attribute.EndTime).Abs()).Add(1 * time.Second).Truncate(time.Second).Unix()
+		if endTime > endTimeThreshold {
+			return false
+		}
+	}
+
+	return true
+}
+
+func FindMinMaxTime(s *parser.EvalStmt) (int64, int64) {
+	// Placeholder until Prometheus is updated to >=0.48.0
+	// which includes https://github.com/prometheus/prometheus/commit/9e3df532d8294d4fe3284bde7bc96db336a33552
+	return s.Start.Unix(), s.End.Unix()
+}
diff --git a/pkg/querier/tripperware/priority_test.go b/pkg/querier/tripperware/priority_test.go
new file mode 100644
index 0000000000..2918842346
--- /dev/null
+++ b/pkg/querier/tripperware/priority_test.go
@@ -0,0 +1,277 @@
+package tripperware
+
+import (
+	"bytes"
+	"net/http"
+	"regexp"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/prometheus/common/model"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/cortexproject/cortex/pkg/util/validation"
+)
+
+func Test_GetPriorityShouldReturnDefaultPriorityIfNotEnabledOrEmptyQueryString(t *testing.T) {
+	now := time.Now()
+	limits := mockLimits{queryPriority: validation.QueryPriority{
+		Priorities: []validation.PriorityDef{
+			{
+				Priority: 1,
+				QueryAttributes: []validation.QueryAttribute{
+					{
+						Regex:         ".*",
+						CompiledRegex: regexp.MustCompile(".*"),
+					},
+				},
+			},
+		},
+	}}
+
+	type testCase struct {
+		url                  string
+		queryPriorityEnabled bool
+	}
+
+	tests := map[string]testCase{
+		"should miss if query priority not enabled": {
+			url: "/query?query=up",
+		},
+		"should miss if query string empty": {
+			url:                  "/query?query=",
+			queryPriorityEnabled: true,
+		},
+		"should miss if query string empty - range query": {
+			url:                  "/query_range?query=",
+			queryPriorityEnabled: true,
+		},
+		"should miss if neither instant nor range query": {
+			url:                  "/series",
+			queryPriorityEnabled: true,
+		},
+	}
+
+	for testName, testData := range tests {
+		t.Run(testName, func(t *testing.T) {
+			limits.queryPriority.Enabled = testData.queryPriorityEnabled
+			req, _ := http.NewRequest(http.MethodPost, testData.url, bytes.NewReader([]byte{}))
+			priority, err := GetPriority(req, "", limits, now)
+			assert.NoError(t, err)
+			assert.Equal(t, int64(0), priority)
+		})
+	}
+}
+
+func Test_GetPriorityShouldConsiderRegex(t *testing.T) {
+	now := time.Now()
+	limits := mockLimits{queryPriority: validation.QueryPriority{
+		Enabled: true,
+		Priorities: []validation.PriorityDef{
+			{
+				Priority: 1,
+				QueryAttributes: []validation.QueryAttribute{
+					{},
+				},
+			},
+		},
+	}}
+
+	type testCase struct {
+		regex            string
+		query            string
+		expectedPriority int
+	}
+
+	tests := map[string]testCase{
+		"should hit if regex matches": {
+			regex:            "(^sum|c(.+)t)",
+			query:            "sum(up)",
+			expectedPriority: 1,
+		},
+		"should miss if regex doesn't match": {
+			regex:            "(^sum|c(.+)t)",
+			query:            "min(up)",
+			expectedPriority: 0,
+		},
+		"should hit if regex matches - .*": {
+			regex:            ".*",
+			query:            "count(sum(up))",
+			expectedPriority: 1,
+		},
+		"should hit if regex matches - .+": {
+			regex:            ".+",
+			query:            "count(sum(up))",
+			expectedPriority: 1,
+		},
+		"should miss if regex is an empty string": {
+			regex:            "",
+			query:            "sum(up)",
+			expectedPriority: 0,
+		},
+	}
+
+	for testName, testData := range tests {
+		t.Run(testName, func(t *testing.T) {
+			limits.queryPriority.Priorities[0].QueryAttributes[0].Regex = testData.regex
+			limits.queryPriority.Priorities[0].QueryAttributes[0].CompiledRegex = regexp.MustCompile(testData.regex)
+			req, _ := http.NewRequest(http.MethodPost, "/query?query="+testData.query, bytes.NewReader([]byte{}))
+			priority, err := GetPriority(req, "", limits, now)
+			assert.NoError(t, err)
+			assert.Equal(t, int64(testData.expectedPriority), priority)
+		})
+	}
+}
+
+func Test_GetPriorityShouldConsiderStartAndEndTime(t *testing.T) {
+	now := time.Now()
+	limits := mockLimits{queryPriority: validation.QueryPriority{
+		Enabled: true,
+		Priorities: []validation.PriorityDef{
+			{
+				Priority: 1,
+				QueryAttributes: []validation.QueryAttribute{
+					{
+						Regex:         ".*",
+						CompiledRegex: regexp.MustCompile(".*"),
+						StartTime:     model.Duration(45 * time.Minute),
+						EndTime:       model.Duration(15 * time.Minute),
+					},
+				},
+			},
+		},
+	}}
+
+	type testCase struct {
+		time             time.Time
+		start            time.Time
+		end              time.Time
+		expectedPriority int
+	}
+
+	tests := map[string]testCase{
+		"should hit instant query between start and end time": {
+			time:             now.Add(-30 * time.Minute),
+			expectedPriority: 1,
+		},
+		"should hit instant query equal to start time": {
+			time:             now.Add(-45 * time.Minute),
+			expectedPriority: 1,
+		},
+		"should hit instant query equal to end time": {
+			time:             now.Add(-15 * time.Minute),
+			expectedPriority: 1,
+		},
+		"should miss instant query outside of end time": {
+			expectedPriority: 0,
+		},
+		"should miss instant query outside of start time": {
+			time:             now.Add(-60 * time.Minute),
+			expectedPriority: 0,
+		},
+		"should hit range query between start and end time": {
+			start:            now.Add(-40 * time.Minute),
+			end:              now.Add(-20 * time.Minute),
+			expectedPriority: 1,
+		},
+		"should hit range query equal to start and end time": {
+			start:            now.Add(-45 * time.Minute),
+			end:              now.Add(-15 * time.Minute),
+			expectedPriority: 1,
+		},
+		"should miss range query outside of start time": {
+			start:            now.Add(-50 * time.Minute),
+			end:              now.Add(-15 * time.Minute),
+			expectedPriority: 0,
+		},
+		"should miss range query completely outside of start time": {
+			start:            now.Add(-50 * time.Minute),
+			end:              now.Add(-45 * time.Minute),
+			expectedPriority: 0,
+		},
+		"should miss range query outside of end time": {
+			start:            now.Add(-45 * time.Minute),
+			end:              now.Add(-10 * time.Minute),
+			expectedPriority: 0,
+		},
+		"should miss range query completely outside of end time": {
+			start:            now.Add(-15 * time.Minute),
+			end:              now.Add(-10 * time.Minute),
+			expectedPriority: 0,
+		},
+	}
+
+	for testName, testData := range tests {
+		t.Run(testName, func(t *testing.T) {
+			var url string
+			if !testData.time.IsZero() {
+				url = "/query?query=sum(up)&time=" + strconv.FormatInt(testData.time.Unix(), 10)
+			} else if !testData.start.IsZero() {
+				url = "/query_range?query=sum(up)&start=" + strconv.FormatInt(testData.start.Unix(), 10)
+				url += "&end=" + strconv.FormatInt(testData.end.Unix(), 10)
+			} else {
+				url = "/query?query=sum(up)"
+			}
+			req, _ := http.NewRequest(http.MethodPost, url, bytes.NewReader([]byte{}))
+			priority, err := GetPriority(req, "", limits, now)
+			assert.NoError(t, err)
+			assert.Equal(t, int64(testData.expectedPriority), priority)
+		})
+	}
+}
+
+func Test_GetPriorityShouldNotConsiderStartAndEndTimeIfEmpty(t *testing.T) {
+	now := time.Now()
+	limits := mockLimits{queryPriority: validation.QueryPriority{
+		Enabled: true,
+		Priorities: []validation.PriorityDef{
+			{
+				Priority: 1,
+				QueryAttributes: []validation.QueryAttribute{
+					{
+						Regex: "^sum\\(up\\)$",
+					},
+				},
+			},
+		},
+	}}
+
+	type testCase struct {
+		time  time.Time
+		start time.Time
+		end   time.Time
+	}
+
+	tests := map[string]testCase{
+		"should hit instant query with no time": {},
+		"should hit instant query with future time": {
+			time: now.Add(1000000 * time.Hour),
+		},
+		"should hit instant query with very old time": {
+			time: now.Add(-1000000 * time.Hour),
+		},
+		"should hit range query with very wide time window": {
+			start: now.Add(-1000000 * time.Hour),
+			end:   now.Add(1000000 * time.Hour),
+		},
+	}
+
+	for testName, testData := range tests {
+		t.Run(testName, func(t *testing.T) {
+			var url string
+			if !testData.time.IsZero() {
+				url = "/query?query=sum(up)&time=" + strconv.FormatInt(testData.time.Unix(), 10)
+			} else if !testData.start.IsZero() {
+				url = "/query_range?query=sum(up)&start=" + strconv.FormatInt(testData.start.Unix(), 10)
+				url += "&end=" + strconv.FormatInt(testData.end.Unix(), 10)
+			} else {
+				url = "/query?query=sum(up)"
+			}
+			req, _ := http.NewRequest(http.MethodPost, url, bytes.NewReader([]byte{}))
+			priority, err := GetPriority(req, "", limits, now)
+			assert.NoError(t, err)
+			assert.Equal(t, int64(1), priority)
+		})
+	}
+}
diff --git a/pkg/querier/tripperware/queryrange/limits_test.go b/pkg/querier/tripperware/queryrange/limits_test.go
index 1569ea2e3a..c1e5954421 100644
--- a/pkg/querier/tripperware/queryrange/limits_test.go
+++ b/pkg/querier/tripperware/queryrange/limits_test.go
@@ -2,6 +2,7 @@ package queryrange
 
 import (
 	"context"
+	"github.com/cortexproject/cortex/pkg/util/validation"
 	"testing"
 	"time"
 
@@ -219,6 +220,10 @@ func (m mockLimits) QueryVerticalShardSize(userID string) int {
 	return 0
 }
 
+func (m mockLimits) QueryPriority(userID string) validation.QueryPriority {
+	return validation.QueryPriority{}
+}
+
 type mockHandler struct {
 	mock.Mock
 }
diff --git a/pkg/querier/tripperware/roundtrip.go b/pkg/querier/tripperware/roundtrip.go
index 6aefe4ccec..6c7cd13508 100644
--- a/pkg/querier/tripperware/roundtrip.go
+++ b/pkg/querier/tripperware/roundtrip.go
@@ -19,6 +19,7 @@ import (
 	"context"
 	"io"
 	"net/http"
+	"strconv"
 	"strings"
 	"time"
 
@@ -142,15 +143,27 @@ func NewQueryTripperware(
 				if err != nil {
 					return nil, err
 				}
+				now := time.Now()
 				userStr := tenant.JoinTenantIDs(tenantIDs)
-				activeUsers.UpdateUserTimestamp(userStr, time.Now())
+				activeUsers.UpdateUserTimestamp(userStr, now)
 				queriesPerTenant.WithLabelValues(op, userStr).Inc()
 
-				if maxSubQuerySteps > 0 && (isQuery || isQueryRange) {
+				if isQuery || isQueryRange {
 					query := r.FormValue("query")
-					// Check subquery step size.
-					if err := SubQueryStepSizeCheck(query, defaultSubQueryInterval, maxSubQuerySteps); err != nil {
-						return nil, err
+
+					if maxSubQuerySteps > 0 {
+						// Check subquery step size.
+						if err := SubQueryStepSizeCheck(query, defaultSubQueryInterval, maxSubQuerySteps); err != nil {
+							return nil, err
+						}
+					}
+
+					if limits.QueryPriority(userStr).Enabled {
+						priority, err := GetPriority(r, userStr, limits, now)
+						if err != nil {
+							return nil, err
+						}
+						r.Header.Set(util.QueryPriorityHeaderKey, strconv.FormatInt(priority, 10))
 					}
 				}
 
diff --git a/pkg/querier/tripperware/test_shard_by_query_utils.go b/pkg/querier/tripperware/test_shard_by_query_utils.go
index 657d7daa3a..5cbad93ca8 100644
--- a/pkg/querier/tripperware/test_shard_by_query_utils.go
+++ b/pkg/querier/tripperware/test_shard_by_query_utils.go
@@ -21,6 +21,7 @@ import (
 	"github.com/weaveworks/common/user"
 
 	"github.com/cortexproject/cortex/pkg/querysharding"
+	"github.com/cortexproject/cortex/pkg/util/validation"
 )
 
 func TestQueryShardQuery(t *testing.T, instantQueryCodec Codec, shardedPrometheusCodec Codec) {
@@ -466,6 +467,7 @@ type mockLimits struct {
 	maxQueryLength    time.Duration
 	maxCacheFreshness time.Duration
 	shardSize         int
+	queryPriority     validation.QueryPriority
 }
 
 func (m mockLimits) MaxQueryLookback(string) time.Duration {
@@ -488,6 +490,10 @@ func (m mockLimits) QueryVerticalShardSize(userID string) int {
 	return m.shardSize
 }
 
+func (m mockLimits) QueryPriority(userID string) validation.QueryPriority {
+	return m.queryPriority
+}
+
 type singleHostRoundTripper struct {
 	host string
 	next http.RoundTripper
diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go
index 65235540a3..fa28485298 100644
--- a/pkg/scheduler/scheduler.go
+++ b/pkg/scheduler/scheduler.go
@@ -5,6 +5,7 @@ import (
 	"flag"
 	"io"
 	"net/http"
+	"strconv"
 	"sync"
 	"time"
 
@@ -154,7 +155,6 @@ type schedulerRequest struct {
 	queryID         uint64
 	request         *httpgrpc.HTTPRequest
 	statsEnabled    bool
-	priority        int64
 
 	enqueueTime time.Time
 
@@ -167,7 +167,12 @@ type schedulerRequest struct {
 }
 
 func (s schedulerRequest) Priority() int64 {
-	return s.priority
+	priority, err := strconv.ParseInt(httpgrpcutil.GetHeader(*s.request, util.QueryPriorityHeaderKey), 10, 64)
+	if err != nil {
+		return 0
+	}
+
+	return priority
 }
 
 // FrontendLoop handles connection from frontend.
@@ -298,7 +303,6 @@ func (s *Scheduler) enqueueRequest(frontendContext context.Context, frontendAddr
 		queryID:         msg.QueryID,
 		request:         msg.HttpRequest,
 		statsEnabled:    msg.StatsEnabled,
-		priority:        msg.Priority,
 	}
 
 	now := time.Now()
diff --git a/pkg/scheduler/schedulerpb/scheduler.pb.go b/pkg/scheduler/schedulerpb/scheduler.pb.go
index 2f352a8138..d3288f95b3 100644
--- a/pkg/scheduler/schedulerpb/scheduler.pb.go
+++ b/pkg/scheduler/schedulerpb/scheduler.pb.go
@@ -219,7 +219,6 @@ type FrontendToScheduler struct {
 	UserID       string                `protobuf:"bytes,4,opt,name=userID,proto3" json:"userID,omitempty"`
 	HttpRequest  *httpgrpc.HTTPRequest `protobuf:"bytes,5,opt,name=httpRequest,proto3" json:"httpRequest,omitempty"`
 	StatsEnabled bool                  `protobuf:"varint,6,opt,name=statsEnabled,proto3" json:"statsEnabled,omitempty"`
-	Priority     int64                 `protobuf:"varint,7,opt,name=priority,proto3" json:"priority,omitempty"`
 }
 
 func (m *FrontendToScheduler) Reset()      { *m = FrontendToScheduler{} }
@@ -296,13 +295,6 @@ func (m *FrontendToScheduler) GetStatsEnabled() bool {
 	return false
 }
 
-func (m *FrontendToScheduler) GetPriority() int64 {
-	if m != nil {
-		return m.Priority
-	}
-	return 0
-}
-
 type SchedulerToFrontend struct {
 	Status SchedulerToFrontendStatus `protobuf:"varint,1,opt,name=status,proto3,enum=schedulerpb.SchedulerToFrontendStatus" json:"status,omitempty"`
 	Error  string                    `protobuf:"bytes,2,opt,name=error,proto3" json:"error,omitempty"`
@@ -446,49 +438,48 @@ func init() {
 func init() { proto.RegisterFile("scheduler.proto", fileDescriptor_2b3fc28395a6d9c5) }
 
 var fileDescriptor_2b3fc28395a6d9c5 = []byte{
-	// 661 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0xcd, 0x4e, 0xdb, 0x40,
-	0x10, 0xf6, 0xe6, 0x0f, 0x98, 0xd0, 0xe2, 0x2e, 0xd0, 0xa6, 0x11, 0x5d, 0x2c, 0xab, 0xaa, 0x52,
-	0x0e, 0x49, 0x95, 0x56, 0x6a, 0x0f, 0xa8, 0x52, 0x0a, 0xa6, 0x44, 0xa5, 0x0e, 0x6c, 0x1c, 0xf5,
-	0xe7, 0x12, 0x91, 0x64, 0x49, 0x22, 0xc0, 0x6b, 0xd6, 0x76, 0x51, 0x6e, 0x7d, 0x84, 0x3e, 0x44,
-	0x0f, 0x7d, 0x94, 0x5e, 0x2a, 0x71, 0xe4, 0xd0, 0x43, 0x31, 0x97, 0x1e, 0x79, 0x84, 0x2a, 0x8e,
-	0xe3, 0x3a, 0x90, 0x00, 0xb7, 0x99, 0xf1, 0xf7, 0x79, 0xe7, 0xfb, 0x66, 0x76, 0x61, 0xce, 0x6e,
-	0x76, 0x58, 0xcb, 0x3d, 0x60, 0x22, 0x6f, 0x09, 0xee, 0x70, 0x9c, 0x0e, 0x0b, 0x56, 0x23, 0xbb,
-	0xd0, 0xe6, 0x6d, 0xee, 0xd7, 0x0b, 0xfd, 0x68, 0x00, 0xc9, 0xbe, 0x68, 0x77, 0x9d, 0x8e, 0xdb,
-	0xc8, 0x37, 0xf9, 0x61, 0xe1, 0x98, 0xed, 0x7e, 0x61, 0xc7, 0x5c, 0xec, 0xdb, 0x85, 0x26, 0x3f,
-	0x3c, 0xe4, 0x66, 0xa1, 0xe3, 0x38, 0x56, 0x5b, 0x58, 0xcd, 0x30, 0x18, 0xb0, 0xd4, 0x22, 0xe0,
-	0x1d, 0x97, 0x89, 0x2e, 0x13, 0x06, 0xaf, 0x0e, 0xcf, 0xc0, 0x4b, 0x30, 0x73, 0x34, 0xa8, 0x96,
-	0xd7, 0x33, 0x48, 0x41, 0xb9, 0x19, 0xfa, 0xbf, 0xa0, 0xfe, 0x42, 0x80, 0x43, 0xac, 0xc1, 0x03,
-	0x3e, 0xce, 0xc0, 0x54, 0x1f, 0xd3, 0x0b, 0x28, 0x09, 0x3a, 0x4c, 0xf1, 0x4b, 0x48, 0xf7, 0x8f,
-	0xa5, 0xec, 0xc8, 0x65, 0xb6, 0x93, 0x89, 0x29, 0x28, 0x97, 0x2e, 0x2e, 0xe6, 0xc3, 0x56, 0x36,
-	0x0d, 0x63, 0x3b, 0xf8, 0x48, 0xa3, 0x48, 0x9c, 0x83, 0xb9, 0x3d, 0xc1, 0x4d, 0x87, 0x99, 0xad,
-	0x52, 0xab, 0x25, 0x98, 0x6d, 0x67, 0xe2, 0x7e, 0x37, 0x97, 0xcb, 0xf8, 0x3e, 0xa4, 0x5c, 0xdb,
-	0x6f, 0x37, 0xe1, 0x03, 0x82, 0x0c, 0xab, 0x30, 0x6b, 0x3b, 0xbb, 0x8e, 0xad, 0x99, 0xbb, 0x8d,
-	0x03, 0xd6, 0xca, 0x24, 0x15, 0x94, 0x9b, 0xa6, 0x23, 0x35, 0xf5, 0x7b, 0x0c, 0xe6, 0x37, 0x82,
-	0xff, 0x45, 0x5d, 0x78, 0x05, 0x09, 0xa7, 0x67, 0x31, 0x5f, 0xcd, 0xdd, 0xe2, 0xe3, 0x7c, 0x64,
-	0x06, 0xf9, 0x31, 0x78, 0xa3, 0x67, 0x31, 0xea, 0x33, 0xc6, 0xf5, 0x1d, 0x1b, 0xdf, 0x77, 0xc4,
-	0xb4, 0xf8, 0xa8, 0x69, 0x93, 0x14, 0x5d, 0x32, 0x33, 0x79, 0x6b, 0x33, 0x2f, 0x5b, 0x91, 0xba,
-	0x6a, 0x05, 0xce, 0xc2, 0xb4, 0x25, 0xba, 0x5c, 0x74, 0x9d, 0x5e, 0x66, 0x4a, 0x41, 0xb9, 0x38,
-	0x0d, 0x73, 0x75, 0x1f, 0xe6, 0x23, 0x53, 0x1f, 0x1a, 0x80, 0x5f, 0x43, 0xaa, 0xff, 0x0b, 0xd7,
-	0x0e, 0x7c, 0x7a, 0x32, 0xe2, 0xd3, 0x18, 0x46, 0xd5, 0x47, 0xd3, 0x80, 0x85, 0x17, 0x20, 0xc9,
-	0x84, 0xe0, 0x22, 0x70, 0x68, 0x90, 0xa8, 0xab, 0xb0, 0xa4, 0x73, 0xa7, 0xbb, 0xd7, 0x0b, 0xb6,
-	0xab, 0xda, 0x71, 0x9d, 0x16, 0x3f, 0x36, 0x87, 0x62, 0xae, 0xdf, 0xd0, 0x65, 0x78, 0x34, 0x81,
-	0x6d, 0x5b, 0xdc, 0xb4, 0xd9, 0xca, 0x2a, 0x3c, 0x98, 0x30, 0x41, 0x3c, 0x0d, 0x89, 0xb2, 0x5e,
-	0x36, 0x64, 0x09, 0xa7, 0x61, 0x4a, 0xd3, 0x77, 0x6a, 0x5a, 0x4d, 0x93, 0x11, 0x06, 0x48, 0xad,
-	0x95, 0xf4, 0x35, 0x6d, 0x4b, 0x8e, 0xad, 0x34, 0xe1, 0xe1, 0x44, 0x5d, 0x38, 0x05, 0xb1, 0xca,
-	0x3b, 0x59, 0xc2, 0x0a, 0x2c, 0x19, 0x95, 0x4a, 0xfd, 0x7d, 0x49, 0xff, 0x54, 0xa7, 0xda, 0x4e,
-	0x4d, 0xab, 0x1a, 0xd5, 0xfa, 0xb6, 0x46, 0xeb, 0x86, 0xa6, 0x97, 0x74, 0x43, 0x46, 0x78, 0x06,
-	0x92, 0x1a, 0xa5, 0x15, 0x2a, 0xc7, 0xf0, 0x3d, 0xb8, 0x53, 0xdd, 0xac, 0x19, 0x46, 0x59, 0x7f,
-	0x5b, 0x5f, 0xaf, 0x7c, 0xd0, 0xe5, 0x78, 0xf1, 0x37, 0x8a, 0xf8, 0xbd, 0xc1, 0xc5, 0xf0, 0x9a,
-	0xd5, 0x20, 0x1d, 0x84, 0x5b, 0x9c, 0x5b, 0x78, 0x79, 0xc4, 0xee, 0xab, 0x77, 0x39, 0xbb, 0x3c,
-	0x69, 0x1e, 0x01, 0x56, 0x95, 0x72, 0xe8, 0x19, 0xc2, 0x26, 0x2c, 0x8e, 0xb5, 0x0c, 0x3f, 0x1d,
-	0xe1, 0x5f, 0x37, 0x94, 0xec, 0xca, 0x6d, 0xa0, 0x83, 0x09, 0x14, 0x2d, 0x58, 0x88, 0xaa, 0x0b,
-	0xd7, 0xe9, 0x23, 0xcc, 0x0e, 0x63, 0x5f, 0x9f, 0x72, 0xd3, 0xb5, 0xcb, 0x2a, 0x37, 0x2d, 0xdc,
-	0x40, 0xe1, 0x9b, 0xd2, 0xc9, 0x19, 0x91, 0x4e, 0xcf, 0x88, 0x74, 0x71, 0x46, 0xd0, 0x57, 0x8f,
-	0xa0, 0x1f, 0x1e, 0x41, 0x3f, 0x3d, 0x82, 0x4e, 0x3c, 0x82, 0xfe, 0x78, 0x04, 0xfd, 0xf5, 0x88,
-	0x74, 0xe1, 0x11, 0xf4, 0xed, 0x9c, 0x48, 0x27, 0xe7, 0x44, 0x3a, 0x3d, 0x27, 0xd2, 0xe7, 0xe8,
-	0xcb, 0xdb, 0x48, 0xf9, 0x8f, 0xe6, 0xf3, 0x7f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x83, 0xb8, 0xa1,
-	0x26, 0xa0, 0x05, 0x00, 0x00,
+	// 644 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0x4f, 0x4f, 0xdb, 0x4e,
+	0x10, 0xf5, 0x86, 0x24, 0xc0, 0x84, 0xdf, 0x0f, 0x77, 0x81, 0x36, 0x8d, 0xe8, 0x12, 0x45, 0x55,
+	0x95, 0x72, 0x48, 0xaa, 0xb4, 0x52, 0x7b, 0x40, 0x95, 0x52, 0x30, 0x25, 0x2a, 0x75, 0x60, 0xb3,
+	0x51, 0xff, 0x5c, 0x22, 0x92, 0x2c, 0x09, 0x02, 0xbc, 0x66, 0x6d, 0x17, 0xe5, 0xd6, 0x63, 0x8f,
+	0xfd, 0x18, 0xfd, 0x28, 0xbd, 0x54, 0xe2, 0xc8, 0xa1, 0x87, 0x62, 0x2e, 0x3d, 0xf2, 0x11, 0xaa,
+	0x38, 0x76, 0xea, 0xa4, 0x0e, 0x70, 0x9b, 0x1d, 0xbf, 0xe7, 0x9d, 0xf7, 0x66, 0x66, 0x61, 0xde,
+	0x6a, 0x75, 0x79, 0xdb, 0x39, 0xe2, 0xb2, 0x60, 0x4a, 0x61, 0x0b, 0x9c, 0x1a, 0x26, 0xcc, 0x66,
+	0x66, 0xb1, 0x23, 0x3a, 0xc2, 0xcb, 0x17, 0xfb, 0xd1, 0x00, 0x92, 0x79, 0xd6, 0x39, 0xb0, 0xbb,
+	0x4e, 0xb3, 0xd0, 0x12, 0xc7, 0xc5, 0x53, 0xbe, 0xf7, 0x89, 0x9f, 0x0a, 0x79, 0x68, 0x15, 0x5b,
+	0xe2, 0xf8, 0x58, 0x18, 0xc5, 0xae, 0x6d, 0x9b, 0x1d, 0x69, 0xb6, 0x86, 0xc1, 0x80, 0x95, 0x2b,
+	0x01, 0xde, 0x75, 0xb8, 0x3c, 0xe0, 0x92, 0x89, 0x5a, 0x70, 0x07, 0x5e, 0x86, 0xd9, 0x93, 0x41,
+	0xb6, 0xb2, 0x91, 0x46, 0x59, 0x94, 0x9f, 0xa5, 0x7f, 0x13, 0xb9, 0x1f, 0x08, 0xf0, 0x10, 0xcb,
+	0x84, 0xcf, 0xc7, 0x69, 0x98, 0xee, 0x63, 0x7a, 0x3e, 0x25, 0x4e, 0x83, 0x23, 0x7e, 0x0e, 0xa9,
+	0xfe, 0xb5, 0x94, 0x9f, 0x38, 0xdc, 0xb2, 0xd3, 0xb1, 0x2c, 0xca, 0xa7, 0x4a, 0x4b, 0x85, 0x61,
+	0x29, 0x5b, 0x8c, 0xed, 0xf8, 0x1f, 0x69, 0x18, 0x89, 0xf3, 0x30, 0xbf, 0x2f, 0x85, 0x61, 0x73,
+	0xa3, 0x5d, 0x6e, 0xb7, 0x25, 0xb7, 0xac, 0xf4, 0x94, 0x57, 0xcd, 0x78, 0x1a, 0xdf, 0x85, 0xa4,
+	0x63, 0x79, 0xe5, 0xc6, 0x3d, 0x80, 0x7f, 0xc2, 0x39, 0x98, 0xb3, 0xec, 0x3d, 0xdb, 0xd2, 0x8c,
+	0xbd, 0xe6, 0x11, 0x6f, 0xa7, 0x13, 0x59, 0x94, 0x9f, 0xa1, 0x23, 0xb9, 0xdc, 0x97, 0x18, 0x2c,
+	0x6c, 0xfa, 0xff, 0x0b, 0xbb, 0xf0, 0x02, 0xe2, 0x76, 0xcf, 0xe4, 0x9e, 0x9a, 0xff, 0x4b, 0x0f,
+	0x0b, 0xa1, 0x1e, 0x14, 0x22, 0xf0, 0xac, 0x67, 0x72, 0xea, 0x31, 0xa2, 0xea, 0x8e, 0x45, 0xd7,
+	0x1d, 0x32, 0x6d, 0x6a, 0xd4, 0xb4, 0x49, 0x8a, 0xc6, 0xcc, 0x4c, 0xdc, 0xda, 0xcc, 0x71, 0x2b,
+	0x92, 0x11, 0x56, 0x1c, 0xc2, 0x42, 0xa8, 0xb3, 0x81, 0x48, 0xfc, 0x12, 0x92, 0x7d, 0x98, 0x63,
+	0xf9, 0x5e, 0x3c, 0x1a, 0xf1, 0x22, 0x82, 0x51, 0xf3, 0xd0, 0xd4, 0x67, 0xe1, 0x45, 0x48, 0x70,
+	0x29, 0x85, 0xf4, 0x5d, 0x18, 0x1c, 0x72, 0x6b, 0xb0, 0xac, 0x0b, 0xfb, 0x60, 0xbf, 0xe7, 0x4f,
+	0x50, 0xad, 0xeb, 0xd8, 0x6d, 0x71, 0x6a, 0x04, 0x05, 0x5f, 0x3f, 0x85, 0x2b, 0xf0, 0x60, 0x02,
+	0xdb, 0x32, 0x85, 0x61, 0xf1, 0xd5, 0x35, 0xb8, 0x37, 0xa1, 0x4b, 0x78, 0x06, 0xe2, 0x15, 0xbd,
+	0xc2, 0x54, 0x05, 0xa7, 0x60, 0x5a, 0xd3, 0x77, 0xeb, 0x5a, 0x5d, 0x53, 0x11, 0x06, 0x48, 0xae,
+	0x97, 0xf5, 0x75, 0x6d, 0x5b, 0x8d, 0xad, 0xb6, 0xe0, 0xfe, 0x44, 0x5d, 0x38, 0x09, 0xb1, 0xea,
+	0x1b, 0x55, 0xc1, 0x59, 0x58, 0x66, 0xd5, 0x6a, 0xe3, 0x6d, 0x59, 0xff, 0xd0, 0xa0, 0xda, 0x6e,
+	0x5d, 0xab, 0xb1, 0x5a, 0x63, 0x47, 0xa3, 0x0d, 0xa6, 0xe9, 0x65, 0x9d, 0xa9, 0x08, 0xcf, 0x42,
+	0x42, 0xa3, 0xb4, 0x4a, 0xd5, 0x18, 0xbe, 0x03, 0xff, 0xd5, 0xb6, 0xea, 0x8c, 0x55, 0xf4, 0xd7,
+	0x8d, 0x8d, 0xea, 0x3b, 0x5d, 0x9d, 0x2a, 0xfd, 0x44, 0x21, 0xbf, 0x37, 0x85, 0x0c, 0x56, 0xa9,
+	0x0e, 0x29, 0x3f, 0xdc, 0x16, 0xc2, 0xc4, 0x2b, 0x23, 0x76, 0xff, 0xbb, 0xaf, 0x99, 0x95, 0x49,
+	0xfd, 0xf0, 0xb1, 0x39, 0x25, 0x8f, 0x9e, 0x20, 0x6c, 0xc0, 0x52, 0xa4, 0x65, 0xf8, 0xf1, 0x08,
+	0xff, 0xba, 0xa6, 0x64, 0x56, 0x6f, 0x03, 0x1d, 0x74, 0xa0, 0x64, 0xc2, 0x62, 0x58, 0xdd, 0x70,
+	0x9c, 0xde, 0xc3, 0x5c, 0x10, 0x7b, 0xfa, 0xb2, 0x37, 0xad, 0x56, 0x26, 0x7b, 0xd3, 0xc0, 0x0d,
+	0x14, 0xbe, 0x2a, 0x9f, 0x5d, 0x10, 0xe5, 0xfc, 0x82, 0x28, 0x57, 0x17, 0x04, 0x7d, 0x76, 0x09,
+	0xfa, 0xe6, 0x12, 0xf4, 0xdd, 0x25, 0xe8, 0xcc, 0x25, 0xe8, 0x97, 0x4b, 0xd0, 0x6f, 0x97, 0x28,
+	0x57, 0x2e, 0x41, 0x5f, 0x2f, 0x89, 0x72, 0x76, 0x49, 0x94, 0xf3, 0x4b, 0xa2, 0x7c, 0x0c, 0xbf,
+	0xae, 0xcd, 0xa4, 0xf7, 0x30, 0x3e, 0xfd, 0x13, 0x00, 0x00, 0xff, 0xff, 0x88, 0x0c, 0xfe, 0x56,
+	0x84, 0x05, 0x00, 0x00,
 }
 
 func (x FrontendToSchedulerType) String() string {
@@ -602,9 +593,6 @@ func (this *FrontendToScheduler) Equal(that interface{}) bool {
 	if this.StatsEnabled != that1.StatsEnabled {
 		return false
 	}
-	if this.Priority != that1.Priority {
-		return false
-	}
 	return true
 }
 func (this *SchedulerToFrontend) Equal(that interface{}) bool {
@@ -709,7 +697,7 @@ func (this *FrontendToScheduler) GoString() string {
 	if this == nil {
 		return "nil"
 	}
-	s := make([]string, 0, 11)
+	s := make([]string, 0, 10)
 	s = append(s, "&schedulerpb.FrontendToScheduler{")
 	s = append(s, "Type: "+fmt.Sprintf("%#v", this.Type)+",\n")
 	s = append(s, "FrontendAddress: "+fmt.Sprintf("%#v", this.FrontendAddress)+",\n")
@@ -719,7 +707,6 @@ func (this *FrontendToScheduler) GoString() string {
 		s = append(s, "HttpRequest: "+fmt.Sprintf("%#v", this.HttpRequest)+",\n")
 	}
 	s = append(s, "StatsEnabled: "+fmt.Sprintf("%#v", this.StatsEnabled)+",\n")
-	s = append(s, "Priority: "+fmt.Sprintf("%#v", this.Priority)+",\n")
 	s = append(s, "}")
 	return strings.Join(s, "")
 }
@@ -1155,11 +1142,6 @@ func (m *FrontendToScheduler) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 	_ = i
 	var l int
 	_ = l
-	if m.Priority != 0 {
-		i = encodeVarintScheduler(dAtA, i, uint64(m.Priority))
-		i--
-		dAtA[i] = 0x38
-	}
 	if m.StatsEnabled {
 		i--
 		if m.StatsEnabled {
@@ -1375,9 +1357,6 @@ func (m *FrontendToScheduler) Size() (n int) {
 	if m.StatsEnabled {
 		n += 2
 	}
-	if m.Priority != 0 {
-		n += 1 + sovScheduler(uint64(m.Priority))
-	}
 	return n
 }
 
@@ -1460,7 +1439,6 @@ func (this *FrontendToScheduler) String() string {
 		`UserID:` + fmt.Sprintf("%v", this.UserID) + `,`,
 		`HttpRequest:` + strings.Replace(fmt.Sprintf("%v", this.HttpRequest), "HTTPRequest", "httpgrpc.HTTPRequest", 1) + `,`,
 		`StatsEnabled:` + fmt.Sprintf("%v", this.StatsEnabled) + `,`,
-		`Priority:` + fmt.Sprintf("%v", this.Priority) + `,`,
 		`}`,
 	}, "")
 	return s
@@ -1967,25 +1945,6 @@ func (m *FrontendToScheduler) Unmarshal(dAtA []byte) error {
 				}
 			}
 			m.StatsEnabled = bool(v != 0)
-		case 7:
-			if wireType != 0 {
-				return fmt.Errorf("proto: wrong wireType = %d for field Priority", wireType)
-			}
-			m.Priority = 0
-			for shift := uint(0); ; shift += 7 {
-				if shift >= 64 {
-					return ErrIntOverflowScheduler
-				}
-				if iNdEx >= l {
-					return io.ErrUnexpectedEOF
-				}
-				b := dAtA[iNdEx]
-				iNdEx++
-				m.Priority |= int64(b&0x7F) << shift
-				if b < 0x80 {
-					break
-				}
-			}
 		default:
 			iNdEx = preIndex
 			skippy, err := skipScheduler(dAtA[iNdEx:])
diff --git a/pkg/scheduler/schedulerpb/scheduler.proto b/pkg/scheduler/schedulerpb/scheduler.proto
index 706c34de4f..eea28717b8 100644
--- a/pkg/scheduler/schedulerpb/scheduler.proto
+++ b/pkg/scheduler/schedulerpb/scheduler.proto
@@ -78,7 +78,6 @@ message FrontendToScheduler {
   string userID = 4;
   httpgrpc.HTTPRequest httpRequest = 5;
   bool statsEnabled = 6;
-  int64 priority = 7;
 }
 
 enum SchedulerToFrontendStatus {
diff --git a/pkg/util/http.go b/pkg/util/http.go
index 09fb3df38c..41daae0fc6 100644
--- a/pkg/util/http.go
+++ b/pkg/util/http.go
@@ -21,6 +21,7 @@ import (
 	yamlv3 "gopkg.in/yaml.v3"
 )
 
+const QueryPriorityHeaderKey = "X-Cortex-Query-Priority"
 const messageSizeLargerErrFmt = "received message larger than max (%d vs %d)"
 
 // IsRequestBodyTooLarge returns true if the error is "http: request body too large".
diff --git a/pkg/util/httpgrpcutil/header.go b/pkg/util/httpgrpcutil/header.go
new file mode 100644
index 0000000000..b844e0c65f
--- /dev/null
+++ b/pkg/util/httpgrpcutil/header.go
@@ -0,0 +1,22 @@
+package httpgrpcutil
+
+import (
+	"github.com/weaveworks/common/httpgrpc"
+)
+
+// GetHeader is similar to http.Header.Get, which gets the first value associated with the given key.
+// If there are no values associated with the key, it returns "".
+func GetHeader(r httpgrpc.HTTPRequest, key string) string {
+	return GetHeaderValues(r, key)[0]
+}
+
+// GetHeaderValues is similar to http.Header.Values, which returns all values associated with the given key.
+func GetHeaderValues(r httpgrpc.HTTPRequest, key string) []string {
+	for _, header := range r.Headers {
+		if header.GetKey() == key {
+			return header.GetValues()
+		}
+	}
+
+	return []string{}
+}
diff --git a/pkg/util/query/priority.go b/pkg/util/query/priority.go
deleted file mode 100644
index 2a7f61ba10..0000000000
--- a/pkg/util/query/priority.go
+++ /dev/null
@@ -1,74 +0,0 @@
-package query
-
-import (
-	"net/url"
-	"time"
-
-	"github.com/cortexproject/cortex/pkg/util"
-	"github.com/cortexproject/cortex/pkg/util/validation"
-)
-
-func GetPriority(requestParams url.Values, now time.Time, queryPriority validation.QueryPriority) int64 {
-	queryParam := requestParams.Get("query")
-	timeParam := requestParams.Get("time")
-	startParam := requestParams.Get("start")
-	endParam := requestParams.Get("end")
-
-	if queryParam == "" || !queryPriority.Enabled {
-		return queryPriority.DefaultPriority
-	}
-
-	for _, priority := range queryPriority.Priorities {
-		for _, attribute := range priority.QueryAttributes {
-			if attribute.CompiledRegex != nil && !attribute.CompiledRegex.MatchString(queryParam) {
-				continue
-			}
-
-			if startTime, err := util.ParseTime(startParam); err == nil {
-				if endTime, err := util.ParseTime(endParam); err == nil {
-					if isWithinTimeAttributes(attribute, now, startTime, endTime) {
-						return priority.Priority
-					}
-				}
-			}
-
-			if instantTime, err := util.ParseTime(timeParam); err == nil {
-				if isWithinTimeAttributes(attribute, now, instantTime, instantTime) {
-					return priority.Priority
-				}
-			}
-
-			if timeParam == "" {
-				if isWithinTimeAttributes(attribute, now, util.TimeToMillis(now), util.TimeToMillis(now)) {
-					return priority.Priority
-				}
-			}
-		}
-	}
-
-	return queryPriority.DefaultPriority
-}
-
-func isWithinTimeAttributes(attribute validation.QueryAttribute, now time.Time, startTime, endTime int64) bool {
-	if attribute.StartTime == 0 && attribute.EndTime == 0 {
-		return true
-	}
-
-	if attribute.StartTime != 0 {
-		startTimeThreshold := now.Add(-1 * time.Duration(attribute.StartTime).Abs()).Truncate(time.Second)
-
-		if util.TimeFromMillis(startTime).Before(startTimeThreshold) {
-			return false
-		}
-	}
-
-	if attribute.EndTime != 0 {
-		endTimeThreshold := now.Add(-1 * time.Duration(attribute.EndTime).Abs()).Add(1 * time.Second).Truncate(time.Second)
-
-		if util.TimeFromMillis(endTime).After(endTimeThreshold) {
-			return false
-		}
-	}
-
-	return true
-}
diff --git a/pkg/util/query/priority_test.go b/pkg/util/query/priority_test.go
deleted file mode 100644
index fea552cde4..0000000000
--- a/pkg/util/query/priority_test.go
+++ /dev/null
@@ -1,224 +0,0 @@
-package query
-
-import (
-	"net/url"
-	"regexp"
-	"strconv"
-	"testing"
-	"time"
-
-	"github.com/prometheus/common/model"
-	"github.com/stretchr/testify/assert"
-
-	"github.com/cortexproject/cortex/pkg/util/validation"
-)
-
-func Test_GetPriorityShouldReturnDefaultPriorityIfNotEnabledOrEmptyQueryString(t *testing.T) {
-	now := time.Now()
-	priorities := []validation.PriorityDef{
-		{
-			Priority: 1,
-			QueryAttributes: []validation.QueryAttribute{
-				{
-					Regex:         ".*",
-					CompiledRegex: regexp.MustCompile(".*"),
-				},
-			},
-		},
-	}
-	queryPriority := validation.QueryPriority{
-		Priorities: priorities,
-	}
-
-	assert.Equal(t, int64(0), GetPriority(url.Values{
-		"query": []string{"sum(up)"},
-		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
-	}, now, queryPriority))
-
-	queryPriority.Enabled = true
-	assert.Equal(t, int64(0), GetPriority(url.Values{
-		"query": []string{""},
-		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
-	}, now, queryPriority))
-}
-
-func Test_GetPriorityShouldConsiderRegex(t *testing.T) {
-	now := time.Now()
-	priorities := []validation.PriorityDef{
-		{
-			Priority: 1,
-			QueryAttributes: []validation.QueryAttribute{
-				{
-					Regex:         "sum",
-					CompiledRegex: regexp.MustCompile("sum"),
-				},
-			},
-		},
-	}
-	queryPriority := validation.QueryPriority{
-		Enabled:    true,
-		Priorities: priorities,
-	}
-
-	assert.Equal(t, int64(1), GetPriority(url.Values{
-		"query": []string{"sum(up)"},
-		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
-	}, now, queryPriority))
-	assert.Equal(t, int64(0), GetPriority(url.Values{
-		"query": []string{"count(up)"},
-		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
-	}, now, queryPriority))
-
-	queryPriority.Priorities[0].QueryAttributes[0].Regex = "(^sum$|c(.+)t)"
-	queryPriority.Priorities[0].QueryAttributes[0].CompiledRegex = regexp.MustCompile("(^sum$|c(.+)t)")
-
-	assert.Equal(t, int64(0), GetPriority(url.Values{
-		"query": []string{"sum(up)"},
-		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
-	}, now, queryPriority))
-	assert.Equal(t, int64(1), GetPriority(url.Values{
-		"query": []string{"count(up)"},
-		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
-	}, now, queryPriority))
-
-	queryPriority.Priorities[0].QueryAttributes[0].Regex = ".*"
-	queryPriority.Priorities[0].QueryAttributes[0].CompiledRegex = regexp.MustCompile(".*")
-
-	assert.Equal(t, int64(1), GetPriority(url.Values{
-		"query": []string{"sum(up)"},
-		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
-	}, now, queryPriority))
-	assert.Equal(t, int64(1), GetPriority(url.Values{
-		"query": []string{"count(up)"},
-		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
-	}, now, queryPriority))
-
-	queryPriority.Priorities[0].QueryAttributes[0].Regex = ".+"
-	queryPriority.Priorities[0].QueryAttributes[0].CompiledRegex = regexp.MustCompile(".+")
-
-	assert.Equal(t, int64(1), GetPriority(url.Values{
-		"query": []string{"sum(up)"},
-		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
-	}, now, queryPriority))
-	assert.Equal(t, int64(1), GetPriority(url.Values{
-		"query": []string{"count(up)"},
-		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
-	}, now, queryPriority))
-
-	queryPriority.Priorities[0].QueryAttributes[0].Regex = ""
-	queryPriority.Priorities[0].QueryAttributes[0].CompiledRegex = regexp.MustCompile("")
-
-	assert.Equal(t, int64(1), GetPriority(url.Values{
-		"query": []string{"sum(up)"},
-		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
-	}, now, queryPriority))
-	assert.Equal(t, int64(1), GetPriority(url.Values{
-		"query": []string{"count(up)"},
-		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
-	}, now, queryPriority))
-}
-
-func Test_GetPriorityShouldConsiderStartAndEndTime(t *testing.T) {
-	now := time.Now()
-	priorities := []validation.PriorityDef{
-		{
-			Priority: 1,
-			QueryAttributes: []validation.QueryAttribute{
-				{
-					StartTime: model.Duration(45 * time.Minute),
-					EndTime:   model.Duration(15 * time.Minute),
-				},
-			},
-		},
-	}
-	queryPriority := validation.QueryPriority{
-		Enabled:    true,
-		Priorities: priorities,
-	}
-
-	assert.Equal(t, int64(0), GetPriority(url.Values{
-		"query": []string{"sum(up)"},
-		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
-	}, now, queryPriority))
-	assert.Equal(t, int64(1), GetPriority(url.Values{
-		"query": []string{"sum(up)"},
-		"time":  []string{strconv.FormatInt(now.Add(-30*time.Minute).Unix(), 10)},
-	}, now, queryPriority))
-	assert.Equal(t, int64(0), GetPriority(url.Values{
-		"query": []string{"sum(up)"},
-		"time":  []string{strconv.FormatInt(now.Add(-60*time.Minute).Unix(), 10)},
-	}, now, queryPriority))
-	assert.Equal(t, int64(1), GetPriority(url.Values{
-		"query": []string{"sum(up)"},
-		"time":  []string{strconv.FormatInt(now.Add(-45*time.Minute).Unix(), 10)},
-	}, now, queryPriority))
-	assert.Equal(t, int64(1), GetPriority(url.Values{
-		"query": []string{"sum(up)"},
-		"time":  []string{strconv.FormatInt(now.Add(-15*time.Minute).Unix(), 10)},
-	}, now, queryPriority))
-
-	assert.Equal(t, int64(1), GetPriority(url.Values{
-		"query": []string{"sum(up)"},
-		"start": []string{strconv.FormatInt(now.Add(-45*time.Minute).Unix(), 10)},
-		"end":   []string{strconv.FormatInt(now.Add(-15*time.Minute).Unix(), 10)},
-	}, now, queryPriority))
-	assert.Equal(t, int64(0), GetPriority(url.Values{
-		"query": []string{"sum(up)"},
-		"start": []string{strconv.FormatInt(now.Add(-50*time.Minute).Unix(), 10)},
-		"end":   []string{strconv.FormatInt(now.Add(-15*time.Minute).Unix(), 10)},
-	}, now, queryPriority))
-	assert.Equal(t, int64(0), GetPriority(url.Values{
-		"query": []string{"sum(up)"},
-		"start": []string{strconv.FormatInt(now.Add(-45*time.Minute).Unix(), 10)},
-		"end":   []string{strconv.FormatInt(now.Add(-10*time.Minute).Unix(), 10)},
-	}, now, queryPriority))
-	assert.Equal(t, int64(0), GetPriority(url.Values{
-		"query": []string{"sum(up)"},
-		"start": []string{strconv.FormatInt(now.Add(-60*time.Minute).Unix(), 10)},
-		"end":   []string{strconv.FormatInt(now.Add(-45*time.Minute).Unix(), 10)},
-	}, now, queryPriority))
-	assert.Equal(t, int64(0), GetPriority(url.Values{
-		"query": []string{"sum(up)"},
-		"start": []string{strconv.FormatInt(now.Add(-15*time.Minute).Unix(), 10)},
-		"end":   []string{strconv.FormatInt(now.Add(-1*time.Minute).Unix(), 10)},
-	}, now, queryPriority))
-}
-
-func Test_GetPriorityShouldSKipStartAndEndTimeIfEmpty(t *testing.T) {
-	now := time.Now()
-	priorities := []validation.PriorityDef{
-		{
-			Priority: 1,
-			QueryAttributes: []validation.QueryAttribute{
-				{
-					Regex: "^test$",
-				},
-			},
-		},
-	}
-	queryPriority := validation.QueryPriority{
-		Enabled:    true,
-		Priorities: priorities,
-	}
-
-	assert.Equal(t, int64(1), GetPriority(url.Values{
-		"query": []string{"test"},
-	}, now, queryPriority))
-	assert.Equal(t, int64(1), GetPriority(url.Values{
-		"query": []string{"test"},
-		"time":  []string{strconv.FormatInt(now.Add(8760*time.Hour).Unix(), 10)},
-	}, now, queryPriority))
-	assert.Equal(t, int64(1), GetPriority(url.Values{
-		"query": []string{"test"},
-		"time":  []string{strconv.FormatInt(now.Unix(), 10)},
-	}, now, queryPriority))
-	assert.Equal(t, int64(1), GetPriority(url.Values{
-		"query": []string{"test"},
-		"time":  []string{strconv.FormatInt(now.Add(-8760*time.Hour).Unix(), 10)},
-	}, now, queryPriority))
-	assert.Equal(t, int64(1), GetPriority(url.Values{
-		"query": []string{"test"},
-		"start": []string{strconv.FormatInt(now.Add(-100000*time.Minute).Unix(), 10)},
-		"end":   []string{strconv.FormatInt(now.Add(100000*time.Minute).Unix(), 10)},
-	}, now, queryPriority))
-}
diff --git a/pkg/util/time.go b/pkg/util/time.go
index 8816b1d7d2..a28c84b046 100644
--- a/pkg/util/time.go
+++ b/pkg/util/time.go
@@ -7,6 +7,7 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/pkg/errors"
 	"github.com/prometheus/common/model"
 	"github.com/weaveworks/common/httpgrpc"
 )
@@ -48,6 +49,19 @@ func ParseTime(s string) (int64, error) {
 	return 0, httpgrpc.Errorf(http.StatusBadRequest, "cannot parse %q to a valid timestamp", s)
 }
 
+// ParseTimeParam parses the time request parameter into an int64, milliseconds since epoch.
+func ParseTimeParam(r *http.Request, paramName string, defaultValue int64) (int64, error) {
+	val := r.FormValue(paramName)
+	if val == "" {
+		val = strconv.FormatInt(defaultValue, 10)
+	}
+	result, err := ParseTime(val)
+	if err != nil {
+		return 0, errors.Wrapf(err, "Invalid time value for '%s'", paramName)
+	}
+	return result, nil
+}
+
 // DurationWithJitter returns random duration from "input - input*variance" to "input + input*variance" interval.
 func DurationWithJitter(input time.Duration, variancePerc float64) time.Duration {
 	// No duration? No jitter.
diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go
index b9b005d0df..cd65e875da 100644
--- a/pkg/util/validation/limits.go
+++ b/pkg/util/validation/limits.go
@@ -57,13 +57,13 @@ type QueryPriority struct {
 
 type PriorityDef struct {
 	Priority         int64            `yaml:"priority" json:"priority" doc:"nocli|description=Priority level. Must be a unique value.|default=0"`
-	ReservedQueriers float64          `yaml:"reserved_queriers" json:"reserved_queriers" doc:"nocli|description=Number of reserved queriers to handle priorities higher or equal to this value only. Value between 0 and 1 will be used as a percentage.|default=0"`
+	ReservedQueriers float64          `yaml:"reserved_queriers" json:"reserved_queriers" doc:"nocli|description=Number of reserved queriers to handle priorities higher or equal to the priority level. Value between 0 and 1 will be used as a percentage.|default=0"`
 	QueryAttributes  []QueryAttribute `yaml:"query_attributes" json:"query_attributes" doc:"nocli|description=List of query attributes to assign the priority."`
 }
 
 type QueryAttribute struct {
-	Regex         string         `yaml:"regex" json:"regex" doc:"nocli|description=Query string regex. If set to empty string, it will not match anything.|default=\"\""`
-	StartTime     model.Duration `yaml:"start_time" json:"start_time" doc:"nocli|description=Query start time. If set to 0, the start time won't be checked.'.|default=0"`
+	Regex         string         `yaml:"regex" json:"regex" doc:"nocli|description=Query string regex. If set to empty string, it will not match anything."`
+	StartTime     model.Duration `yaml:"start_time" json:"start_time" doc:"nocli|description=Query start time. If set to 0, the start time won't be checked.|default=0"`
 	EndTime       model.Duration `yaml:"end_time" json:"end_time" doc:"nocli|description=Query end time. If set to 0, the end time won't be checked.|default=0"`
 	CompiledRegex *regexp.Regexp
 }
diff --git a/tools/doc-generator/parser.go b/tools/doc-generator/parser.go
index 7a6dbfebfb..28ee31ba4f 100644
--- a/tools/doc-generator/parser.go
+++ b/tools/doc-generator/parser.go
@@ -409,6 +409,10 @@ func getCustomFieldEntry(parent reflect.Type, field reflect.StructField, fieldVa
 			return nil, err
 		}
 
+		if fieldFlag == nil {
+			return nil, nil
+		}
+
 		return &configEntry{
 			kind:         "field",
 			name:         getFieldName(field),

From 6c1813b75ecf8d7d6e65cd556550c9bdc709ffe5 Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Tue, 28 Nov 2023 10:13:51 -0800
Subject: [PATCH 44/49] Attempt to fix tests

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 pkg/frontend/transport/handler.go   | 1 -
 pkg/querier/tripperware/priority.go | 4 ++--
 pkg/util/httpgrpcutil/header.go     | 7 ++++++-
 3 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/pkg/frontend/transport/handler.go b/pkg/frontend/transport/handler.go
index 042dbd32b7..700145312f 100644
--- a/pkg/frontend/transport/handler.go
+++ b/pkg/frontend/transport/handler.go
@@ -200,7 +200,6 @@ func (f *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		r.Body = io.NopCloser(&buf)
 	}
 
-	r.Header.Get("test")
 	startTime := time.Now()
 	resp, err := f.roundTripper.RoundTrip(r)
 	queryResponseTime := time.Since(startTime)
diff --git a/pkg/querier/tripperware/priority.go b/pkg/querier/tripperware/priority.go
index 5ce4e0d3fa..b96a3b0e9a 100644
--- a/pkg/querier/tripperware/priority.go
+++ b/pkg/querier/tripperware/priority.go
@@ -88,7 +88,7 @@ func isWithinTimeAttributes(attribute validation.QueryAttribute, now time.Time,
 }
 
 func FindMinMaxTime(s *parser.EvalStmt) (int64, int64) {
-	// Placeholder until Prometheus is updated to >=0.48.0
-	// which includes https://github.com/prometheus/prometheus/commit/9e3df532d8294d4fe3284bde7bc96db336a33552
+	// Placeholder until Prometheus is updated to include
+	// https://github.com/prometheus/prometheus/commit/9e3df532d8294d4fe3284bde7bc96db336a33552
 	return s.Start.Unix(), s.End.Unix()
 }
diff --git a/pkg/util/httpgrpcutil/header.go b/pkg/util/httpgrpcutil/header.go
index b844e0c65f..ed453f01fa 100644
--- a/pkg/util/httpgrpcutil/header.go
+++ b/pkg/util/httpgrpcutil/header.go
@@ -7,7 +7,12 @@ import (
 // GetHeader is similar to http.Header.Get, which gets the first value associated with the given key.
 // If there are no values associated with the key, it returns "".
 func GetHeader(r httpgrpc.HTTPRequest, key string) string {
-	return GetHeaderValues(r, key)[0]
+	values := GetHeaderValues(r, key)
+	if len(values) == 0 {
+		return ""
+	}
+
+	return values[0]
 }
 
 // GetHeaderValues is similar to http.Header.Values, which returns all values associated with the given key.

From cdd2a0c785aacb62ce44535a41a619db55670558 Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Wed, 29 Nov 2023 10:12:44 -0800
Subject: [PATCH 45/49] Address comments

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 pkg/cortex/modules.go                         |  1 +
 pkg/frontend/transport/handler_test.go        | 80 +++++++++++++------
 .../tripperware/instantquery/instant_query.go |  2 +-
 pkg/querier/tripperware/priority.go           |  4 +-
 pkg/querier/tripperware/priority_test.go      |  8 +-
 .../query_range_middlewares_test.go           |  1 +
 pkg/querier/tripperware/roundtrip.go          |  5 +-
 pkg/querier/tripperware/roundtrip_test.go     |  1 +
 pkg/util/validation/limits.go                 | 13 ++-
 9 files changed, 78 insertions(+), 37 deletions(-)

diff --git a/pkg/cortex/modules.go b/pkg/cortex/modules.go
index 7858483c68..4fc0969ee7 100644
--- a/pkg/cortex/modules.go
+++ b/pkg/cortex/modules.go
@@ -487,6 +487,7 @@ func (t *Cortex) initQueryFrontendTripperware() (serv services.Service, err erro
 		queryAnalyzer,
 		t.Cfg.Querier.DefaultEvaluationInterval,
 		t.Cfg.Querier.MaxSubQuerySteps,
+		t.Cfg.Querier.LookbackDelta,
 	)
 
 	return services.NewIdleService(nil, func(_ error) error {
diff --git a/pkg/frontend/transport/handler_test.go b/pkg/frontend/transport/handler_test.go
index 955323fa99..801bda6d81 100644
--- a/pkg/frontend/transport/handler_test.go
+++ b/pkg/frontend/transport/handler_test.go
@@ -21,6 +21,7 @@ import (
 	"github.com/weaveworks/common/user"
 
 	querier_stats "github.com/cortexproject/cortex/pkg/querier/stats"
+	"github.com/cortexproject/cortex/pkg/util"
 )
 
 type roundTripperFunc func(*http.Request) (*http.Response, error)
@@ -301,34 +302,63 @@ func TestReportQueryStatsFormat(t *testing.T) {
 	outputBuf := bytes.NewBuffer(nil)
 	logger := log.NewSyncLogger(log.NewLogfmtLogger(outputBuf))
 	handler := NewHandler(HandlerConfig{QueryStatsEnabled: true}, http.DefaultTransport, logger, nil)
-
 	userID := "fake"
-	queryString := url.Values(map[string][]string{"query": {"up"}})
-	req, err := http.NewRequest(http.MethodGet, "http://localhost:8080/prometheus/api/v1/query", nil)
-	require.NoError(t, err)
-	req.Header = http.Header{
-		"User-Agent": []string{"Grafana"},
-	}
-	resp := &http.Response{
-		ContentLength: 1000,
+	req, _ := http.NewRequest(http.MethodGet, "http://localhost:8080/prometheus/api/v1/query", nil)
+	resp := &http.Response{ContentLength: 1000}
+	responseTime := time.Second
+	statusCode := http.StatusOK
+
+	type testCase struct {
+		queryString url.Values
+		queryStats  *querier_stats.QueryStats
+		header      http.Header
+		responseErr error
+		expectedLog string
 	}
-	stats := &querier_stats.QueryStats{
-		Stats: querier_stats.Stats{
-			WallTime:            3 * time.Second,
-			FetchedSeriesCount:  100,
-			FetchedChunksCount:  200,
-			FetchedSamplesCount: 300,
-			FetchedChunkBytes:   1024,
-			FetchedDataBytes:    2048,
+
+	tests := map[string]testCase{
+		"should not include query and header details if empty": {
+			expectedLog: `level=info msg="query stats" component=query-frontend method=GET path=/prometheus/api/v1/query response_time=1s query_wall_time_seconds=0 fetched_series_count=0 fetched_chunks_count=0 fetched_samples_count=0 fetched_chunks_bytes=0 fetched_data_bytes=0 status_code=200 response_size=1000`,
+		},
+		"should include query length and string at the end": {
+			queryString: url.Values(map[string][]string{"query": {"up"}}),
+			expectedLog: `level=info msg="query stats" component=query-frontend method=GET path=/prometheus/api/v1/query response_time=1s query_wall_time_seconds=0 fetched_series_count=0 fetched_chunks_count=0 fetched_samples_count=0 fetched_chunks_bytes=0 fetched_data_bytes=0 status_code=200 response_size=1000 query_length=2 param_query=up`,
+		},
+		"should include query stats": {
+			queryStats: &querier_stats.QueryStats{
+				Stats: querier_stats.Stats{
+					WallTime:            3 * time.Second,
+					FetchedSeriesCount:  100,
+					FetchedChunksCount:  200,
+					FetchedSamplesCount: 300,
+					FetchedChunkBytes:   1024,
+					FetchedDataBytes:    2048,
+				},
+			},
+			expectedLog: `level=info msg="query stats" component=query-frontend method=GET path=/prometheus/api/v1/query response_time=1s query_wall_time_seconds=3 fetched_series_count=100 fetched_chunks_count=200 fetched_samples_count=300 fetched_chunks_bytes=1024 fetched_data_bytes=2048 status_code=200 response_size=1000`,
+		},
+		"should include user agent": {
+			header:      http.Header{"User-Agent": []string{"Grafana"}},
+			expectedLog: `level=info msg="query stats" component=query-frontend method=GET path=/prometheus/api/v1/query response_time=1s query_wall_time_seconds=0 fetched_series_count=0 fetched_chunks_count=0 fetched_samples_count=0 fetched_chunks_bytes=0 fetched_data_bytes=0 status_code=200 response_size=1000 user_agent=Grafana`,
+		},
+		"should include response error": {
+			responseErr: errors.New("foo_err"),
+			expectedLog: `level=error msg="query stats" component=query-frontend method=GET path=/prometheus/api/v1/query response_time=1s query_wall_time_seconds=0 fetched_series_count=0 fetched_chunks_count=0 fetched_samples_count=0 fetched_chunks_bytes=0 fetched_data_bytes=0 status_code=200 response_size=1000 error=foo_err`,
+		},
+		"should include query priority": {
+			queryString: url.Values(map[string][]string{"query": {"up"}}),
+			header:      http.Header{util.QueryPriorityHeaderKey: []string{"99"}},
+			expectedLog: `level=info msg="query stats" component=query-frontend method=GET path=/prometheus/api/v1/query response_time=1s query_wall_time_seconds=0 fetched_series_count=0 fetched_chunks_count=0 fetched_samples_count=0 fetched_chunks_bytes=0 fetched_data_bytes=0 status_code=200 response_size=1000 query_length=2 priority=99 param_query=up`,
 		},
 	}
-	responseErr := errors.New("foo_err")
-	handler.reportQueryStats(req, userID, queryString, time.Second, stats, responseErr, http.StatusOK, resp)
 
-	data, err := io.ReadAll(outputBuf)
-	require.NoError(t, err)
-
-	expectedLog := `level=error msg="query stats" component=query-frontend method=GET path=/prometheus/api/v1/query response_time=1s query_wall_time_seconds=3 fetched_series_count=100 fetched_chunks_count=200 fetched_samples_count=300 fetched_chunks_bytes=1024 fetched_data_bytes=2048 status_code=200 response_size=1000 query_length=2 user_agent=Grafana error=foo_err param_query=up
-`
-	require.Equal(t, expectedLog, string(data))
+	for testName, testData := range tests {
+		t.Run(testName, func(t *testing.T) {
+			req.Header = testData.header
+			handler.reportQueryStats(req, userID, testData.queryString, responseTime, testData.queryStats, testData.responseErr, statusCode, resp)
+			data, err := io.ReadAll(outputBuf)
+			require.NoError(t, err)
+			require.Equal(t, testData.expectedLog+"\n", string(data))
+		})
+	}
 }
diff --git a/pkg/querier/tripperware/instantquery/instant_query.go b/pkg/querier/tripperware/instantquery/instant_query.go
index 60bea7a8d1..252443f5a2 100644
--- a/pkg/querier/tripperware/instantquery/instant_query.go
+++ b/pkg/querier/tripperware/instantquery/instant_query.go
@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"github.com/cortexproject/cortex/pkg/util"
 	"io"
 	"net/http"
 	"net/url"
@@ -25,6 +24,7 @@ import (
 	"github.com/cortexproject/cortex/pkg/cortexpb"
 	"github.com/cortexproject/cortex/pkg/querier/tripperware"
 	"github.com/cortexproject/cortex/pkg/querier/tripperware/queryrange"
+	"github.com/cortexproject/cortex/pkg/util"
 	"github.com/cortexproject/cortex/pkg/util/spanlogger"
 )
 
diff --git a/pkg/querier/tripperware/priority.go b/pkg/querier/tripperware/priority.go
index b96a3b0e9a..0bdd4c1be5 100644
--- a/pkg/querier/tripperware/priority.go
+++ b/pkg/querier/tripperware/priority.go
@@ -11,7 +11,7 @@ import (
 	"github.com/cortexproject/cortex/pkg/util/validation"
 )
 
-func GetPriority(r *http.Request, userID string, limits Limits, now time.Time) (int64, error) {
+func GetPriority(r *http.Request, userID string, limits Limits, now time.Time, lookbackDelta time.Duration) (int64, error) {
 	isQuery := strings.HasSuffix(r.URL.Path, "/query")
 	isQueryRange := strings.HasSuffix(r.URL.Path, "/query_range")
 	queryPriority := limits.QueryPriority(userID)
@@ -45,7 +45,7 @@ func GetPriority(r *http.Request, userID string, limits Limits, now time.Time) (
 		Expr:          expr,
 		Start:         util.TimeFromMillis(startTime),
 		End:           util.TimeFromMillis(endTime),
-		LookbackDelta: limits.MaxQueryLookback(userID), // this is available from querier flag.
+		LookbackDelta: lookbackDelta,
 	}
 
 	minTime, maxTime := FindMinMaxTime(es)
diff --git a/pkg/querier/tripperware/priority_test.go b/pkg/querier/tripperware/priority_test.go
index 2918842346..6cfaa45b49 100644
--- a/pkg/querier/tripperware/priority_test.go
+++ b/pkg/querier/tripperware/priority_test.go
@@ -57,7 +57,7 @@ func Test_GetPriorityShouldReturnDefaultPriorityIfNotEnabledOrEmptyQueryString(t
 		t.Run(testName, func(t *testing.T) {
 			limits.queryPriority.Enabled = testData.queryPriorityEnabled
 			req, _ := http.NewRequest(http.MethodPost, testData.url, bytes.NewReader([]byte{}))
-			priority, err := GetPriority(req, "", limits, now)
+			priority, err := GetPriority(req, "", limits, now, 0)
 			assert.NoError(t, err)
 			assert.Equal(t, int64(0), priority)
 		})
@@ -117,7 +117,7 @@ func Test_GetPriorityShouldConsiderRegex(t *testing.T) {
 			limits.queryPriority.Priorities[0].QueryAttributes[0].Regex = testData.regex
 			limits.queryPriority.Priorities[0].QueryAttributes[0].CompiledRegex = regexp.MustCompile(testData.regex)
 			req, _ := http.NewRequest(http.MethodPost, "/query?query="+testData.query, bytes.NewReader([]byte{}))
-			priority, err := GetPriority(req, "", limits, now)
+			priority, err := GetPriority(req, "", limits, now, 0)
 			assert.NoError(t, err)
 			assert.Equal(t, int64(testData.expectedPriority), priority)
 		})
@@ -214,7 +214,7 @@ func Test_GetPriorityShouldConsiderStartAndEndTime(t *testing.T) {
 				url = "/query?query=sum(up)"
 			}
 			req, _ := http.NewRequest(http.MethodPost, url, bytes.NewReader([]byte{}))
-			priority, err := GetPriority(req, "", limits, now)
+			priority, err := GetPriority(req, "", limits, now, 0)
 			assert.NoError(t, err)
 			assert.Equal(t, int64(testData.expectedPriority), priority)
 		})
@@ -269,7 +269,7 @@ func Test_GetPriorityShouldNotConsiderStartAndEndTimeIfEmpty(t *testing.T) {
 				url = "/query?query=sum(up)"
 			}
 			req, _ := http.NewRequest(http.MethodPost, url, bytes.NewReader([]byte{}))
-			priority, err := GetPriority(req, "", limits, now)
+			priority, err := GetPriority(req, "", limits, now, 0)
 			assert.NoError(t, err)
 			assert.Equal(t, int64(1), priority)
 		})
diff --git a/pkg/querier/tripperware/queryrange/query_range_middlewares_test.go b/pkg/querier/tripperware/queryrange/query_range_middlewares_test.go
index 6276be72bd..518cdce885 100644
--- a/pkg/querier/tripperware/queryrange/query_range_middlewares_test.go
+++ b/pkg/querier/tripperware/queryrange/query_range_middlewares_test.go
@@ -75,6 +75,7 @@ func TestRoundTrip(t *testing.T) {
 		qa,
 		time.Minute,
 		0,
+		0,
 	)
 
 	for i, tc := range []struct {
diff --git a/pkg/querier/tripperware/roundtrip.go b/pkg/querier/tripperware/roundtrip.go
index 6c7cd13508..8ad3dd7fe0 100644
--- a/pkg/querier/tripperware/roundtrip.go
+++ b/pkg/querier/tripperware/roundtrip.go
@@ -105,6 +105,7 @@ func NewQueryTripperware(
 	queryAnalyzer querysharding.Analyzer,
 	defaultSubQueryInterval time.Duration,
 	maxSubQuerySteps int64,
+	lookbackDelta time.Duration,
 ) Tripperware {
 	// Per tenant query metrics.
 	queriesPerTenant := promauto.With(registerer).NewCounterVec(prometheus.CounterOpts{
@@ -158,8 +159,8 @@ func NewQueryTripperware(
 						}
 					}
 
-					if limits.QueryPriority(userStr).Enabled {
-						priority, err := GetPriority(r, userStr, limits, now)
+					if limits != nil && limits.QueryPriority(userStr).Enabled {
+						priority, err := GetPriority(r, userStr, limits, now, lookbackDelta)
 						if err != nil {
 							return nil, err
 						}
diff --git a/pkg/querier/tripperware/roundtrip_test.go b/pkg/querier/tripperware/roundtrip_test.go
index e52514ee8d..2754c17baa 100644
--- a/pkg/querier/tripperware/roundtrip_test.go
+++ b/pkg/querier/tripperware/roundtrip_test.go
@@ -201,6 +201,7 @@ func TestRoundTrip(t *testing.T) {
 				querysharding.NewQueryAnalyzer(),
 				time.Minute,
 				tc.maxSubQuerySteps,
+				0,
 			)
 			resp, err := tw(downstream).RoundTrip(req)
 			if tc.expectedErr == nil {
diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go
index cd65e875da..7253bf3108 100644
--- a/pkg/util/validation/limits.go
+++ b/pkg/util/validation/limits.go
@@ -10,6 +10,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/cespare/xxhash/v2"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/prometheus/model/relabel"
 	"golang.org/x/time/rate"
@@ -125,7 +126,7 @@ type Limits struct {
 	// Query Frontend / Scheduler enforced limits.
 	MaxOutstandingPerTenant    int           `yaml:"max_outstanding_requests_per_tenant" json:"max_outstanding_requests_per_tenant"`
 	QueryPriority              QueryPriority `yaml:"query_priority" json:"query_priority" doc:"nocli|description=Configuration for query priority."`
-	queryPriorityRegexHash     string
+	queryPriorityRegexHash     uint64
 	queryPriorityCompiledRegex map[string]*regexp.Regexp
 
 	// Ruler defaults and limits.
@@ -314,12 +315,18 @@ func (l *Limits) copyNotificationIntegrationLimits(defaults NotificationRateLimi
 }
 
 func (l *Limits) hasQueryPriorityRegexChanged() bool {
-	var newHash string
+	var newHash uint64
+
+	var seps = []byte{'\xff'}
+	h := xxhash.New()
 	for _, priority := range l.QueryPriority.Priorities {
 		for _, attribute := range priority.QueryAttributes {
-			newHash += attribute.Regex
+			_, _ = h.WriteString(attribute.Regex)
+			_, _ = h.Write(seps)
 		}
 	}
+	newHash = h.Sum64()
+
 	if newHash != l.queryPriorityRegexHash {
 		l.queryPriorityRegexHash = newHash
 		return true

From 3f347e4d154aa0da6af9d04c6430cd4714069969 Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Wed, 29 Nov 2023 17:00:07 -0800
Subject: [PATCH 46/49] Minor improvements

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 pkg/querier/tripperware/priority.go      | 10 ++++++++--
 pkg/querier/tripperware/priority_test.go |  4 ++--
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/pkg/querier/tripperware/priority.go b/pkg/querier/tripperware/priority.go
index 0bdd4c1be5..361008a581 100644
--- a/pkg/querier/tripperware/priority.go
+++ b/pkg/querier/tripperware/priority.go
@@ -26,6 +26,10 @@ func GetPriority(r *http.Request, userID string, limits Limits, now time.Time, l
 		return 0, err
 	}
 
+	if len(queryPriority.Priorities) == 0 {
+		return queryPriority.DefaultPriority, nil
+	}
+
 	var startTime, endTime int64
 	if isQuery {
 		if t, err := util.ParseTimeParam(r, "time", now.Unix()); err == nil {
@@ -52,8 +56,10 @@ func GetPriority(r *http.Request, userID string, limits Limits, now time.Time, l
 
 	for _, priority := range queryPriority.Priorities {
 		for _, attribute := range priority.QueryAttributes {
-			if attribute.Regex == "" || (attribute.CompiledRegex != nil && !attribute.CompiledRegex.MatchString(query)) {
-				continue
+			if attribute.Regex != "" && attribute.Regex != ".*" && attribute.Regex != ".+" {
+				if attribute.CompiledRegex != nil && !attribute.CompiledRegex.MatchString(query) {
+					continue
+				}
 			}
 
 			if isWithinTimeAttributes(attribute, now, minTime, maxTime) {
diff --git a/pkg/querier/tripperware/priority_test.go b/pkg/querier/tripperware/priority_test.go
index 6cfaa45b49..9aac92ffe3 100644
--- a/pkg/querier/tripperware/priority_test.go
+++ b/pkg/querier/tripperware/priority_test.go
@@ -105,10 +105,10 @@ func Test_GetPriorityShouldConsiderRegex(t *testing.T) {
 			query:            "count(sum(up))",
 			expectedPriority: 1,
 		},
-		"should miss if regex is an empty string": {
+		"should hit if regex is an empty string": {
 			regex:            "",
 			query:            "sum(up)",
-			expectedPriority: 0,
+			expectedPriority: 1,
 		},
 	}
 

From 3ee0b0da92096ab018104900d9b8c20cb1471a7c Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Wed, 29 Nov 2023 21:13:41 -0800
Subject: [PATCH 47/49] Rename query start end time

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 docs/configuration/config-file-reference.md | 14 +++++++++-----
 pkg/querier/tripperware/priority.go         | 14 +++++++-------
 pkg/querier/tripperware/priority_test.go    |  6 ++++--
 pkg/util/validation/limits.go               | 10 +++++++---
 pkg/util/validation/limits_test.go          |  2 +-
 5 files changed, 28 insertions(+), 18 deletions(-)

diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md
index 25286293dd..49c244f87a 100644
--- a/docs/configuration/config-file-reference.md
+++ b/docs/configuration/config-file-reference.md
@@ -5061,14 +5061,18 @@ otel:
 ### `QueryAttribute`
 
 ```yaml
-# Query string regex. If set to empty string, it will not match anything.
+# Regex that the query string should match. If not set, it won't be checked.'
 [regex: <string> | default = ""]
 
-# Query start time. If set to 0, the start time won't be checked.
-[start_time: <int> | default = 0]
+# Time window that the query should be within. If not set, it won't be checked.'
+time_window:
+  # Start of the time window that the query should be within. If set to 0, it
+  # won't be checked.
+  [start: <int> | default = 0]
 
-# Query end time. If set to 0, the end time won't be checked.
-[end_time: <int> | default = 0]
+  # End of the time window that the query should be within. If set to 0, it
+  # won't be checked.
+  [end: <int> | default = 0]
 ```
 
 ### `DisabledRuleGroup`
diff --git a/pkg/querier/tripperware/priority.go b/pkg/querier/tripperware/priority.go
index 361008a581..4a8d3ae07f 100644
--- a/pkg/querier/tripperware/priority.go
+++ b/pkg/querier/tripperware/priority.go
@@ -62,7 +62,7 @@ func GetPriority(r *http.Request, userID string, limits Limits, now time.Time, l
 				}
 			}
 
-			if isWithinTimeAttributes(attribute, now, minTime, maxTime) {
+			if isWithinTimeAttributes(attribute.TimeWindow, now, minTime, maxTime) {
 				return priority.Priority, nil
 			}
 		}
@@ -71,20 +71,20 @@ func GetPriority(r *http.Request, userID string, limits Limits, now time.Time, l
 	return queryPriority.DefaultPriority, nil
 }
 
-func isWithinTimeAttributes(attribute validation.QueryAttribute, now time.Time, startTime, endTime int64) bool {
-	if attribute.StartTime == 0 && attribute.EndTime == 0 {
+func isWithinTimeAttributes(timeWindow validation.TimeWindow, now time.Time, startTime, endTime int64) bool {
+	if timeWindow.Start == 0 && timeWindow.End == 0 {
 		return true
 	}
 
-	if attribute.StartTime != 0 {
-		startTimeThreshold := now.Add(-1 * time.Duration(attribute.StartTime).Abs()).Truncate(time.Second).Unix()
+	if timeWindow.Start != 0 {
+		startTimeThreshold := now.Add(-1 * time.Duration(timeWindow.Start).Abs()).Truncate(time.Second).Unix()
 		if startTime < startTimeThreshold {
 			return false
 		}
 	}
 
-	if attribute.EndTime != 0 {
-		endTimeThreshold := now.Add(-1 * time.Duration(attribute.EndTime).Abs()).Add(1 * time.Second).Truncate(time.Second).Unix()
+	if timeWindow.End != 0 {
+		endTimeThreshold := now.Add(-1 * time.Duration(timeWindow.End).Abs()).Add(1 * time.Second).Truncate(time.Second).Unix()
 		if endTime > endTimeThreshold {
 			return false
 		}
diff --git a/pkg/querier/tripperware/priority_test.go b/pkg/querier/tripperware/priority_test.go
index 9aac92ffe3..e95d9170d9 100644
--- a/pkg/querier/tripperware/priority_test.go
+++ b/pkg/querier/tripperware/priority_test.go
@@ -135,8 +135,10 @@ func Test_GetPriorityShouldConsiderStartAndEndTime(t *testing.T) {
 					{
 						Regex:         ".*",
 						CompiledRegex: regexp.MustCompile(".*"),
-						StartTime:     model.Duration(45 * time.Minute),
-						EndTime:       model.Duration(15 * time.Minute),
+						TimeWindow: validation.TimeWindow{
+							Start: model.Duration(45 * time.Minute),
+							End:   model.Duration(15 * time.Minute),
+						},
 					},
 				},
 			},
diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go
index 7253bf3108..9531ee8555 100644
--- a/pkg/util/validation/limits.go
+++ b/pkg/util/validation/limits.go
@@ -63,12 +63,16 @@ type PriorityDef struct {
 }
 
 type QueryAttribute struct {
-	Regex         string         `yaml:"regex" json:"regex" doc:"nocli|description=Query string regex. If set to empty string, it will not match anything."`
-	StartTime     model.Duration `yaml:"start_time" json:"start_time" doc:"nocli|description=Query start time. If set to 0, the start time won't be checked.|default=0"`
-	EndTime       model.Duration `yaml:"end_time" json:"end_time" doc:"nocli|description=Query end time. If set to 0, the end time won't be checked.|default=0"`
+	Regex         string     `yaml:"regex" json:"regex" doc:"nocli|description=Regex that the query string should match. If not set, it won't be checked.'"`
+	TimeWindow    TimeWindow `yaml:"time_window" json:"time_window" doc:"nocli|description=Time window that the query should be within. If not set, it won't be checked.'"`
 	CompiledRegex *regexp.Regexp
 }
 
+type TimeWindow struct {
+	Start model.Duration `yaml:"start" json:"start" doc:"nocli|description=Start of the time window that the query should be within. If set to 0, it won't be checked.|default=0"`
+	End   model.Duration `yaml:"end" json:"end" doc:"nocli|description=End of the time window that the query should be within. If set to 0, it won't be checked.|default=0"`
+}
+
 // Limits describe all the limits for users; can be used to describe global default
 // limits via flags, or per-user limits via yaml config.
 type Limits struct {
diff --git a/pkg/util/validation/limits_test.go b/pkg/util/validation/limits_test.go
index 32a0bf924c..e3b8c6d3ff 100644
--- a/pkg/util/validation/limits_test.go
+++ b/pkg/util/validation/limits_test.go
@@ -620,7 +620,7 @@ func TestHasQueryPriorityRegexChanged(t *testing.T) {
 
 	require.True(t, l.hasQueryPriorityRegexChanged())
 
-	l.QueryPriority.Priorities[0].QueryAttributes[0].StartTime = model.Duration(2 * time.Hour)
+	l.QueryPriority.Priorities[0].QueryAttributes[0].TimeWindow.Start = model.Duration(2 * time.Hour)
 
 	require.False(t, l.hasQueryPriorityRegexChanged())
 

From c0252f15067b4097ebfb461d31144086e5170cf8 Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Wed, 29 Nov 2023 22:08:46 -0800
Subject: [PATCH 48/49] Improve tests

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 .../tripperware/queryrange/limits_test.go     |   2 +-
 pkg/scheduler/queue/user_queues.go            |  17 ++-
 pkg/scheduler/queue/user_queues_test.go       | 101 +++++++-----------
 3 files changed, 53 insertions(+), 67 deletions(-)

diff --git a/pkg/querier/tripperware/queryrange/limits_test.go b/pkg/querier/tripperware/queryrange/limits_test.go
index c1e5954421..d5d4a6b230 100644
--- a/pkg/querier/tripperware/queryrange/limits_test.go
+++ b/pkg/querier/tripperware/queryrange/limits_test.go
@@ -2,7 +2,6 @@ package queryrange
 
 import (
 	"context"
-	"github.com/cortexproject/cortex/pkg/util/validation"
 	"testing"
 	"time"
 
@@ -13,6 +12,7 @@ import (
 
 	"github.com/cortexproject/cortex/pkg/querier/tripperware"
 	"github.com/cortexproject/cortex/pkg/util"
+	"github.com/cortexproject/cortex/pkg/util/validation"
 )
 
 func TestLimitsMiddleware_MaxQueryLookback(t *testing.T) {
diff --git a/pkg/scheduler/queue/user_queues.go b/pkg/scheduler/queue/user_queues.go
index b0ae8069ac..25f562ee02 100644
--- a/pkg/scheduler/queue/user_queues.go
+++ b/pkg/scheduler/queue/user_queues.go
@@ -2,7 +2,6 @@ package queue
 
 import (
 	"math/rand"
-	"reflect"
 	"sort"
 	"time"
 
@@ -136,7 +135,7 @@ func (q *queues) getOrAddQueue(userID string, maxQueriers int) userRequestQueue
 	uq := q.userQueues[userID]
 	priorityEnabled := q.limits.QueryPriority(userID).Enabled
 	maxOutstanding := q.limits.MaxOutstandingPerTenant(userID)
-	priorityList := getPriorityList(q.limits.QueryPriority(userID), len(q.queriers))
+	priorityList := getPriorityList(q.limits.QueryPriority(userID), maxQueriers)
 
 	if uq == nil {
 		uq = &userQueue{
@@ -180,7 +179,7 @@ func (q *queues) getOrAddQueue(userID string, maxQueriers int) userRequestQueue
 		uq.queriers = shuffleQueriersForUser(uq.seed, maxQueriers, q.sortedQueriers, nil)
 	}
 
-	if priorityEnabled && !reflect.DeepEqual(uq.priorityList, priorityList) {
+	if priorityEnabled && hasPriorityListChanged(uq.priorityList, priorityList) {
 		reservedQueriers := make(map[string]int64)
 
 		i := 0
@@ -406,6 +405,18 @@ func getPriorityList(queryPriority validation.QueryPriority, totalQuerierCount i
 	return priorityList
 }
 
+func hasPriorityListChanged(old, new []int64) bool {
+	if len(old) != len(new) {
+		return true
+	}
+	for i := range old {
+		if old[i] != new[i] {
+			return true
+		}
+	}
+	return false
+}
+
 // MockLimits implements the Limits interface. Used in tests only.
 type MockLimits struct {
 	MaxOutstanding        int
diff --git a/pkg/scheduler/queue/user_queues_test.go b/pkg/scheduler/queue/user_queues_test.go
index ae812aac6a..ded597baa0 100644
--- a/pkg/scheduler/queue/user_queues_test.go
+++ b/pkg/scheduler/queue/user_queues_test.go
@@ -399,6 +399,7 @@ func TestGetOrAddQueueShouldUpdateProperties(t *testing.T) {
 	assert.Equal(t, 3, q.userQueues["userID"].maxQueriers)
 	assert.Equal(t, 5, len(q.queriers))
 	assert.Equal(t, 3, len(q.userQueues["userID"].queriers))
+	assert.Equal(t, 2, len(q.userQueues["userID"].reservedQueriers))
 	assert.IsType(t, &PriorityRequestQueue{}, queue)
 	assert.Equal(t, 1, queue.length())
 	assert.ElementsMatch(t, []int64{1, 1}, q.userQueues["userID"].priorityList)
@@ -406,6 +407,38 @@ func TestGetOrAddQueueShouldUpdateProperties(t *testing.T) {
 	assert.Subset(t, getKeys(q.queriers), getKeys(q.userQueues["userID"].queriers))
 	assert.Subset(t, getKeys(q.userQueues["userID"].queriers), getKeys(q.userQueues["userID"].reservedQueriers))
 
+	limits.QueryPriorityVal = validation.QueryPriority{Enabled: true, Priorities: []validation.PriorityDef{
+		{
+			Priority:         1,
+			ReservedQueriers: 0.5,
+		},
+	}}
+	q.limits = limits
+	_ = q.getOrAddQueue("userID", 3)
+
+	assert.Equal(t, 10, q.userQueues["userID"].maxOutstanding)
+	assert.Equal(t, 3, q.userQueues["userID"].maxQueriers)
+	assert.Equal(t, 5, len(q.queriers))
+	assert.Equal(t, 3, len(q.userQueues["userID"].queriers))
+	assert.Equal(t, 2, len(q.userQueues["userID"].reservedQueriers))
+	assert.ElementsMatch(t, []int64{1, 1}, q.userQueues["userID"].priorityList)
+
+	limits.QueryPriorityVal = validation.QueryPriority{Enabled: true, Priorities: []validation.PriorityDef{
+		{
+			Priority:         1,
+			ReservedQueriers: 10,
+		},
+	}}
+	q.limits = limits
+	_ = q.getOrAddQueue("userID", 3)
+
+	assert.Equal(t, 10, q.userQueues["userID"].maxOutstanding)
+	assert.Equal(t, 3, q.userQueues["userID"].maxQueriers)
+	assert.Equal(t, 5, len(q.queriers))
+	assert.Equal(t, 3, len(q.userQueues["userID"].queriers))
+	assert.Equal(t, 0, len(q.userQueues["userID"].reservedQueriers))
+	assert.ElementsMatch(t, []int64{}, q.userQueues["userID"].priorityList)
+
 	limits.QueryPriorityVal.Enabled = false
 	q.limits = limits
 	queue = q.getOrAddQueue("userID", 3)
@@ -582,69 +615,11 @@ func TestShuffleQueriersCorrectness(t *testing.T) {
 	}
 }
 
-func TestShuffleQueriers_WithReservedQueriers(t *testing.T) {
-	//allQueriers := []string{"a", "b", "c", "d", "e"}
-	//
-	//queriers, reservedQueriers := shuffleQueriersForUser(12345, 0, allQueriers, 0, nil)
-	//require.Nil(t, queriers)
-	//require.Equal(t, 0, len(reservedQueriers))
-	//
-	//queriers, reservedQueriers = shuffleQueriersForUser(12345, 0, allQueriers, 0.5, nil)
-	//require.Nil(t, queriers)
-	//require.Equal(t, 3, len(reservedQueriers))
-	//
-	//queriers, reservedQueriers = shuffleQueriersForUser(12345, 0, allQueriers, 1, nil)
-	//require.Nil(t, queriers)
-	//require.Equal(t, 1, len(reservedQueriers))
-	//
-	//queriers, reservedQueriers = shuffleQueriersForUser(12345, 0, allQueriers, 100, nil)
-	//require.Nil(t, queriers)
-	//require.Equal(t, 5, len(reservedQueriers))
-	//
-	//queriers, reservedQueriers = shuffleQueriersForUser(12345, 3, allQueriers, 0, nil)
-	//require.Equal(t, 3, len(queriers))
-	//require.Equal(t, 0, len(reservedQueriers))
-	//
-	//queriers, reservedQueriers = shuffleQueriersForUser(12345, 3, allQueriers, 0.5, nil)
-	//require.Equal(t, 3, len(queriers))
-	//require.Equal(t, 2, len(reservedQueriers))
-	//
-	//queriers, reservedQueriers = shuffleQueriersForUser(12345, 3, allQueriers, 1, nil)
-	//require.Equal(t, 3, len(queriers))
-	//require.Equal(t, 1, len(reservedQueriers))
-	//
-	//queriers, reservedQueriers = shuffleQueriersForUser(12345, 3, allQueriers, 100, nil)
-	//require.Equal(t, 3, len(queriers))
-	//require.Equal(t, 3, len(reservedQueriers))
-	//
-	//queriers, reservedQueriers = shuffleQueriersForUser(12345, 100, allQueriers, 0, nil)
-	//require.Nil(t, queriers)
-	//require.Equal(t, 0, len(reservedQueriers))
-	//
-	//queriers, reservedQueriers = shuffleQueriersForUser(12345, 100, allQueriers, 0.5, nil)
-	//require.Nil(t, queriers)
-	//require.Equal(t, 3, len(reservedQueriers))
-	//
-	//queriers, reservedQueriers = shuffleQueriersForUser(12345, 100, allQueriers, 1, nil)
-	//require.Nil(t, queriers)
-	//require.Equal(t, 1, len(reservedQueriers))
-	//
-	//queriers, reservedQueriers = shuffleQueriersForUser(12345, 100, allQueriers, 100, nil)
-	//require.Nil(t, queriers)
-	//require.Equal(t, 5, len(reservedQueriers))
-}
-
-func TestShuffleQueriers_WithReservedQueriers_Correctness(t *testing.T) {
-	//allQueriers := []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n"}
-	//
-	//prevQueriers, prevReservedQueriers := shuffleQueriersForUser(12345, 10, allQueriers, 5, nil)
-	//for i := 0; i < 100; i++ {
-	//	queriers, reservedQueriers := shuffleQueriersForUser(12345, 10, allQueriers, 5, nil)
-	//	require.Equal(t, prevQueriers, queriers)
-	//	require.Equal(t, prevReservedQueriers, reservedQueriers)
-	//	prevQueriers = queriers
-	//	prevReservedQueriers = reservedQueriers
-	//}
+func TestHasPriorityListChanged(t *testing.T) {
+	require.True(t, hasPriorityListChanged([]int64{1, 2}, []int64{1, 3}))
+	require.False(t, hasPriorityListChanged([]int64{1, 2}, []int64{1, 2}))
+	require.True(t, hasPriorityListChanged([]int64{1, 2}, []int64{1}))
+	require.False(t, hasPriorityListChanged([]int64{}, []int64{}))
 }
 
 func TestGetPriorityList(t *testing.T) {

From b6eb74d4cfb927c5de553ef3ead0ab036c429a94 Mon Sep 17 00:00:00 2001
From: Justin Jung <jungjust@amazon.com>
Date: Wed, 29 Nov 2023 22:17:50 -0800
Subject: [PATCH 49/49] Nit

Signed-off-by: Justin Jung <jungjust@amazon.com>
---
 docs/configuration/config-file-reference.md | 4 ++--
 pkg/util/validation/limits.go               | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md
index 49c244f87a..b5ab659830 100644
--- a/docs/configuration/config-file-reference.md
+++ b/docs/configuration/config-file-reference.md
@@ -5061,10 +5061,10 @@ otel:
 ### `QueryAttribute`
 
 ```yaml
-# Regex that the query string should match. If not set, it won't be checked.'
+# Regex that the query string should match. If not set, it won't be checked.
 [regex: <string> | default = ""]
 
-# Time window that the query should be within. If not set, it won't be checked.'
+# Time window that the query should be within. If not set, it won't be checked.
 time_window:
   # Start of the time window that the query should be within. If set to 0, it
   # won't be checked.
diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go
index 9531ee8555..a10e3f1f78 100644
--- a/pkg/util/validation/limits.go
+++ b/pkg/util/validation/limits.go
@@ -63,8 +63,8 @@ type PriorityDef struct {
 }
 
 type QueryAttribute struct {
-	Regex         string     `yaml:"regex" json:"regex" doc:"nocli|description=Regex that the query string should match. If not set, it won't be checked.'"`
-	TimeWindow    TimeWindow `yaml:"time_window" json:"time_window" doc:"nocli|description=Time window that the query should be within. If not set, it won't be checked.'"`
+	Regex         string     `yaml:"regex" json:"regex" doc:"nocli|description=Regex that the query string should match. If not set, it won't be checked."`
+	TimeWindow    TimeWindow `yaml:"time_window" json:"time_window" doc:"nocli|description=Time window that the query should be within. If not set, it won't be checked."`
 	CompiledRegex *regexp.Regexp
 }