forked from cortexproject/cortex
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Ruler: optimised <prefix>/api/v1/rules and <prefix>/api/v1/alerts (co…
…rtexproject#3916) * Use a grpc clients pool in the ruler Signed-off-by: Marco Pracucci <marco@pracucci.com> * Concurrently fetch rules from all rulers Signed-off-by: Marco Pracucci <marco@pracucci.com> * Added subservices manager Signed-off-by: Marco Pracucci <marco@pracucci.com> * Fixed Rules() grpc call Signed-off-by: Marco Pracucci <marco@pracucci.com> * Added integration test Signed-off-by: Marco Pracucci <marco@pracucci.com> * Added CHANGELOG entry Signed-off-by: Marco Pracucci <marco@pracucci.com> * Addressed review comments Signed-off-by: Marco Pracucci <marco@pracucci.com> * Fixed CHANGELOG Signed-off-by: Marco Pracucci <marco@pracucci.com>
- Loading branch information
1 parent
d27b2d0
commit 2c28f24
Showing
8 changed files
with
345 additions
and
42 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -4,7 +4,7 @@ import ( | |
"context" | ||
"encoding/json" | ||
"errors" | ||
io "io" | ||
"io" | ||
"io/ioutil" | ||
"net/http" | ||
"net/http/httptest" | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,79 @@ | ||
package ruler | ||
|
||
import ( | ||
"time" | ||
|
||
"github.com/go-kit/kit/log" | ||
"github.com/pkg/errors" | ||
"github.com/prometheus/client_golang/prometheus" | ||
"github.com/prometheus/client_golang/prometheus/promauto" | ||
"google.golang.org/grpc" | ||
"google.golang.org/grpc/health/grpc_health_v1" | ||
|
||
"github.com/cortexproject/cortex/pkg/ring/client" | ||
"github.com/cortexproject/cortex/pkg/util/grpcclient" | ||
) | ||
|
||
func newRulerClientPool(clientCfg grpcclient.Config, logger log.Logger, reg prometheus.Registerer) *client.Pool { | ||
// We prefer sane defaults instead of exposing further config options. | ||
poolCfg := client.PoolConfig{ | ||
CheckInterval: time.Minute, | ||
HealthCheckEnabled: true, | ||
HealthCheckTimeout: 10 * time.Second, | ||
} | ||
|
||
clientsCount := promauto.With(reg).NewGauge(prometheus.GaugeOpts{ | ||
Name: "cortex_ruler_clients", | ||
Help: "The current number of ruler clients in the pool.", | ||
}) | ||
|
||
return client.NewPool("ruler", poolCfg, nil, newRulerClientFactory(clientCfg, reg), clientsCount, logger) | ||
} | ||
|
||
func newRulerClientFactory(clientCfg grpcclient.Config, reg prometheus.Registerer) client.PoolFactory { | ||
requestDuration := promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{ | ||
Name: "cortex_ruler_client_request_duration_seconds", | ||
Help: "Time spent executing requests to the ruler.", | ||
Buckets: prometheus.ExponentialBuckets(0.008, 4, 7), | ||
}, []string{"operation", "status_code"}) | ||
|
||
return func(addr string) (client.PoolClient, error) { | ||
return dialRulerClient(clientCfg, addr, requestDuration) | ||
} | ||
} | ||
|
||
func dialRulerClient(clientCfg grpcclient.Config, addr string, requestDuration *prometheus.HistogramVec) (*rulerExtendedClient, error) { | ||
opts, err := clientCfg.DialOption(grpcclient.Instrument(requestDuration)) | ||
if err != nil { | ||
return nil, err | ||
} | ||
|
||
conn, err := grpc.Dial(addr, opts...) | ||
if err != nil { | ||
return nil, errors.Wrapf(err, "failed to dial ruler %s", addr) | ||
} | ||
|
||
return &rulerExtendedClient{ | ||
RulerClient: NewRulerClient(conn), | ||
HealthClient: grpc_health_v1.NewHealthClient(conn), | ||
conn: conn, | ||
}, nil | ||
} | ||
|
||
type rulerExtendedClient struct { | ||
RulerClient | ||
grpc_health_v1.HealthClient | ||
conn *grpc.ClientConn | ||
} | ||
|
||
func (c *rulerExtendedClient) Close() error { | ||
return c.conn.Close() | ||
} | ||
|
||
func (c *rulerExtendedClient) String() string { | ||
return c.RemoteAddress() | ||
} | ||
|
||
func (c *rulerExtendedClient) RemoteAddress() string { | ||
return c.conn.Target() | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,68 @@ | ||
package ruler | ||
|
||
import ( | ||
"context" | ||
"net" | ||
"testing" | ||
|
||
"github.com/prometheus/client_golang/prometheus" | ||
dto "github.com/prometheus/client_model/go" | ||
"github.com/stretchr/testify/assert" | ||
"github.com/stretchr/testify/require" | ||
"github.com/weaveworks/common/user" | ||
"google.golang.org/grpc" | ||
|
||
"github.com/cortexproject/cortex/pkg/util/flagext" | ||
"github.com/cortexproject/cortex/pkg/util/grpcclient" | ||
) | ||
|
||
func Test_newRulerClientFactory(t *testing.T) { | ||
// Create a GRPC server used to query the mocked service. | ||
grpcServer := grpc.NewServer() | ||
defer grpcServer.GracefulStop() | ||
|
||
srv := &mockRulerServer{} | ||
RegisterRulerServer(grpcServer, srv) | ||
|
||
listener, err := net.Listen("tcp", "localhost:0") | ||
require.NoError(t, err) | ||
|
||
go func() { | ||
require.NoError(t, grpcServer.Serve(listener)) | ||
}() | ||
|
||
// Create a client factory and query back the mocked service | ||
// with different clients. | ||
cfg := grpcclient.Config{} | ||
flagext.DefaultValues(&cfg) | ||
|
||
reg := prometheus.NewPedanticRegistry() | ||
factory := newRulerClientFactory(cfg, reg) | ||
|
||
for i := 0; i < 2; i++ { | ||
client, err := factory(listener.Addr().String()) | ||
require.NoError(t, err) | ||
defer client.Close() //nolint:errcheck | ||
|
||
ctx := user.InjectOrgID(context.Background(), "test") | ||
_, err = client.(*rulerExtendedClient).Rules(ctx, &RulesRequest{}) | ||
assert.NoError(t, err) | ||
} | ||
|
||
// Assert on the request duration metric, but since it's a duration histogram and | ||
// we can't predict the exact time it took, we need to workaround it. | ||
metrics, err := reg.Gather() | ||
require.NoError(t, err) | ||
|
||
assert.Len(t, metrics, 1) | ||
assert.Equal(t, "cortex_ruler_client_request_duration_seconds", metrics[0].GetName()) | ||
assert.Equal(t, dto.MetricType_HISTOGRAM, metrics[0].GetType()) | ||
assert.Len(t, metrics[0].GetMetric(), 1) | ||
assert.Equal(t, uint64(2), metrics[0].GetMetric()[0].GetHistogram().GetSampleCount()) | ||
} | ||
|
||
type mockRulerServer struct{} | ||
|
||
func (m *mockRulerServer) Rules(context.Context, *RulesRequest) (*RulesResponse, error) { | ||
return &RulesResponse{}, nil | ||
} |
Oops, something went wrong.