From 310f750d689e85993f395729dff1f719dd300a5e Mon Sep 17 00:00:00 2001 From: Jon Johnson Date: Fri, 4 Aug 2023 09:52:31 -0700 Subject: [PATCH] Push rate limiting down into rekor client This requires us to fork a small package from sigstore/rekor, but it gives us a few benefits: We rate limit per HTTP request, which is more aligned with the actual rate limit (rather than per rekor client method call). We rate limit the innermost transport, which means retries won't bypass the limit. We can modify the default transport to use a pooled transport, which should reuse connections, which should be faster with concurrent requests. Signed-off-by: Jon Johnson --- go.mod | 6 +- internal/provider/provider.go | 9 +- internal/provider/resource_attest.go | 3 - internal/provider/resource_sign.go | 3 - internal/secant/rekor/client/options.go | 108 +++++++++++++++++++ internal/secant/rekor/client/rekor_client.go | 63 +++++++++++ 6 files changed, 175 insertions(+), 17 deletions(-) create mode 100644 internal/secant/rekor/client/options.go create mode 100644 internal/secant/rekor/client/rekor_client.go diff --git a/go.mod b/go.mod index fd4837ef..93ff205a 100644 --- a/go.mod +++ b/go.mod @@ -8,11 +8,14 @@ replace github.com/theupdateframework/go-tuf => github.com/theupdateframework/go require ( github.com/chainguard-dev/terraform-provider-oci v0.0.4 github.com/cyberphone/json-canonicalization v0.0.0-20230710064741-aa7fe85c7dbd + github.com/go-openapi/runtime v0.26.0 github.com/go-openapi/strfmt v0.21.7 github.com/go-openapi/swag v0.22.4 github.com/google/certificate-transparency-go v1.1.6 github.com/google/go-containerregistry v0.15.3-0.20230607134719-145eebe7465d github.com/google/uuid v1.3.0 + github.com/hashicorp/go-cleanhttp v0.5.2 + github.com/hashicorp/go-retryablehttp v0.7.4 github.com/hashicorp/terraform-plugin-docs v0.14.1 github.com/hashicorp/terraform-plugin-framework v1.3.3 github.com/hashicorp/terraform-plugin-framework-validators v0.10.0 @@ -133,7 +136,6 @@ require ( github.com/go-openapi/jsonpointer v0.20.0 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/loads v0.21.2 // indirect - github.com/go-openapi/runtime v0.26.0 // indirect github.com/go-openapi/spec v0.20.9 // indirect github.com/go-openapi/validate v0.22.1 // indirect github.com/go-playground/locales v0.14.1 // indirect @@ -158,12 +160,10 @@ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-checkpoint v0.5.0 // indirect - github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-cty v1.4.1-0.20200414143053-d3edf31b6320 // indirect github.com/hashicorp/go-hclog v1.5.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-plugin v1.4.10 // indirect - github.com/hashicorp/go-retryablehttp v0.7.4 // indirect github.com/hashicorp/go-rootcerts v1.0.2 // indirect github.com/hashicorp/go-secure-stdlib/parseutil v0.1.7 // indirect github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect diff --git a/internal/provider/provider.go b/internal/provider/provider.go index c341ffb5..82a14b21 100644 --- a/internal/provider/provider.go +++ b/internal/provider/provider.go @@ -6,6 +6,7 @@ import ( "sync" "github.com/chainguard-dev/terraform-provider-cosign/internal/secant/fulcio" + rclient "github.com/chainguard-dev/terraform-provider-cosign/internal/secant/rekor/client" "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/v1/google" "github.com/google/go-containerregistry/pkg/v1/remote" @@ -14,9 +15,7 @@ import ( "github.com/hashicorp/terraform-plugin-framework/provider/schema" "github.com/hashicorp/terraform-plugin-framework/resource" "github.com/sigstore/fulcio/pkg/api" - rclient "github.com/sigstore/rekor/pkg/client" "github.com/sigstore/rekor/pkg/generated/client" - "go.uber.org/ratelimit" ) // Ensure Provider satisfies various provider interfaces. @@ -43,9 +42,6 @@ type ProviderOpts struct { // Keyed off rekor URL. rekorClients map[string]*client.Rekor - - // Client-side rate limiting to avoid rekor 429s. - limiter ratelimit.Limiter } func (p *ProviderOpts) rekorClient(rekorUrl string) (*client.Rekor, error) { @@ -133,9 +129,6 @@ func (p *Provider) Configure(ctx context.Context, req provider.ConfigureRequest, oidc: &oidcProvider{}, signers: map[string]*fulcio.SignerVerifier{}, rekorClients: map[string]*client.Rekor{}, - // A little bird told me that rekor allows 500 requests per minute. - // We want to stay well under that, so we'll round down to 5 QPS. - limiter: ratelimit.New(5, ratelimit.WithoutSlack), } // Make provider opts available to resources and data sources. diff --git a/internal/provider/resource_attest.go b/internal/provider/resource_attest.go index d75362d0..5156b1fa 100644 --- a/internal/provider/resource_attest.go +++ b/internal/provider/resource_attest.go @@ -236,9 +236,6 @@ func (r *AttestResource) doAttest(ctx context.Context, data *AttestResourceModel return "", nil, fmt.Errorf("creating rekor client: %w", err) } - // Avoid hitting rekor rate limits. - r.popts.limiter.Take() - ctx, cancel := context.WithTimeout(ctx, options.DefaultTimeout) defer cancel() diff --git a/internal/provider/resource_sign.go b/internal/provider/resource_sign.go index b97ba626..d0108f60 100644 --- a/internal/provider/resource_sign.go +++ b/internal/provider/resource_sign.go @@ -121,9 +121,6 @@ func (r *SignResource) doSign(ctx context.Context, data *SignResourceModel) (str return "", nil, fmt.Errorf("creating rekor client: %w", err) } - // Avoid hitting rekor rate limits. - r.popts.limiter.Take() - ctx, cancel := context.WithTimeout(ctx, options.DefaultTimeout) defer cancel() diff --git a/internal/secant/rekor/client/options.go b/internal/secant/rekor/client/options.go new file mode 100644 index 00000000..cb13310d --- /dev/null +++ b/internal/secant/rekor/client/options.go @@ -0,0 +1,108 @@ +// Copyright 2021 The Sigstore Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +import ( + "net/http" + + "github.com/hashicorp/go-retryablehttp" + "go.uber.org/ratelimit" +) + +// Option is a functional option for customizing static signatures. +type Option func(*options) + +type options struct { + UserAgent string + RetryCount uint + Logger interface{} + + // Client-side rate limiting to avoid rekor 429s. + // This is the only real difference from upstream. + // I'd rather just make the transport pluggable if we upstream this. + limiter ratelimit.Limiter +} + +const ( + // DefaultRetryCount is the default number of retries. + DefaultRetryCount = 3 +) + +func makeOptions(opts ...Option) *options { + o := &options{ + UserAgent: "", + RetryCount: DefaultRetryCount, + // A little bird told me that rekor allows 500 requests per minute. + // We want to stay well under that, so we'll round down to 5 QPS. + limiter: ratelimit.New(5, ratelimit.WithoutSlack), + } + + for _, opt := range opts { + opt(o) + } + + return o +} + +// WithUserAgent sets the media type of the signature. +func WithUserAgent(userAgent string) Option { + return func(o *options) { + o.UserAgent = userAgent + } +} + +// WithRetryCount sets the number of retries. +func WithRetryCount(retryCount uint) Option { + return func(o *options) { + o.RetryCount = retryCount + } +} + +// WithLogger sets the logger; it must implement either retryablehttp.Logger or retryablehttp.LeveledLogger; if not, this will not take effect. +func WithLogger(logger interface{}) Option { + return func(o *options) { + switch logger.(type) { + case retryablehttp.Logger, retryablehttp.LeveledLogger: + o.Logger = logger + } + } +} + +type roundTripper struct { + http.RoundTripper + UserAgent string + limiter ratelimit.Limiter +} + +// RoundTrip implements `http.RoundTripper` +func (rt *roundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + req.Header.Set("User-Agent", rt.UserAgent) + + // Blocks to avoid hitting rate limits. + rt.limiter.Take() + + return rt.RoundTripper.RoundTrip(req) +} + +func createRoundTripper(inner http.RoundTripper, o *options) http.RoundTripper { + if inner == nil { + inner = http.DefaultTransport + } + return &roundTripper{ + RoundTripper: inner, + UserAgent: o.UserAgent, + limiter: o.limiter, + } +} diff --git a/internal/secant/rekor/client/rekor_client.go b/internal/secant/rekor/client/rekor_client.go new file mode 100644 index 00000000..92b0bf49 --- /dev/null +++ b/internal/secant/rekor/client/rekor_client.go @@ -0,0 +1,63 @@ +// Copyright 2021 The Sigstore Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +import ( + "net/http" + "net/url" + + "github.com/go-openapi/runtime" + httptransport "github.com/go-openapi/runtime/client" + "github.com/go-openapi/strfmt" + "github.com/hashicorp/go-cleanhttp" + retryablehttp "github.com/hashicorp/go-retryablehttp" + "github.com/sigstore/rekor/pkg/generated/client" + "github.com/sigstore/rekor/pkg/util" +) + +func GetRekorClient(rekorServerURL string, opts ...Option) (*client.Rekor, error) { + url, err := url.Parse(rekorServerURL) + if err != nil { + return nil, err + } + o := makeOptions(opts...) + + retryableClient := retryablehttp.NewClient() + + // Another difference from upstream is that we want a DefaultPooledTransport + // because we have a single client per host. + defaultTransport := createRoundTripper(cleanhttp.DefaultPooledTransport(), o) + retryableClient.HTTPClient = &http.Client{ + Transport: defaultTransport, + } + retryableClient.RetryMax = int(o.RetryCount) + retryableClient.Logger = o.Logger + + httpClient := retryableClient.StandardClient() + + // sanitize path + if url.Path == "" { + url.Path = client.DefaultBasePath + } + + rt := httptransport.NewWithClient(url.Host, url.Path, []string{url.Scheme}, httpClient) + rt.Consumers["application/json"] = runtime.JSONConsumer() + rt.Consumers["application/x-pem-file"] = runtime.TextConsumer() + rt.Producers["application/json"] = runtime.JSONProducer() + + registry := strfmt.Default + registry.Add("signedCheckpoint", &util.SignedNote{}, util.SignedCheckpointValidator) + return client.New(rt, registry), nil +}