From 1dc7704736f66caa5b25430f3e3484e28a14e04f Mon Sep 17 00:00:00 2001 From: Seth Vargo Date: Thu, 14 Jan 2021 18:05:18 -0500 Subject: [PATCH 1/4] Add rotation harness and rotate token signing keys This adds a new service, rotation, which handles key and rotation events. It's designed to be invoked via Cloud Scheduler and has its own locking and time intervals. It's conceptually similar to the cleanup job. This also adds the database schema and example rotation for token signing keys. Note that this code is NOT in use in main code paths yet. --- cmd/cleanup/main.go | 16 +- cmd/rotation/main.go | 133 +++++++++ pkg/config/cleanup_server_config.go | 14 +- pkg/config/rotation_config.go | 84 ++++++ pkg/controller/cleanup/cleanup.go | 32 +- pkg/controller/cleanup/cleanup_test.go | 29 ++ pkg/controller/rotation/handle_rotate.go | 123 ++++++++ pkg/controller/rotation/handle_rotate_test.go | 157 ++++++++++ pkg/controller/rotation/metrics.go | 60 ++++ pkg/controller/rotation/rotation.go | 54 ++++ pkg/controller/rotation/rotation_test.go | 29 ++ pkg/controller/verifyapi/verify.go | 3 + pkg/database/migrations.go | 31 +- pkg/database/token_signing_keys.go | 218 ++++++++++++++ pkg/database/token_signing_keys_test.go | 281 ++++++++++++++++++ terraform/service_rotation.tf | 225 ++++++++++++++ 16 files changed, 1469 insertions(+), 20 deletions(-) create mode 100644 cmd/rotation/main.go create mode 100644 pkg/config/rotation_config.go create mode 100644 pkg/controller/cleanup/cleanup_test.go create mode 100644 pkg/controller/rotation/handle_rotate.go create mode 100644 pkg/controller/rotation/handle_rotate_test.go create mode 100644 pkg/controller/rotation/metrics.go create mode 100644 pkg/controller/rotation/rotation.go create mode 100644 pkg/controller/rotation/rotation_test.go create mode 100644 pkg/database/token_signing_keys.go create mode 100644 pkg/database/token_signing_keys_test.go create mode 100644 terraform/service_rotation.tf diff --git a/cmd/cleanup/main.go b/cmd/cleanup/main.go index a3629bdc5..124212248 100644 --- a/cmd/cleanup/main.go +++ b/cmd/cleanup/main.go @@ -26,6 +26,7 @@ import ( "github.com/google/exposure-notifications-verification-server/pkg/controller/middleware" "github.com/google/exposure-notifications-verification-server/pkg/render" + "github.com/google/exposure-notifications-server/pkg/keys" "github.com/google/exposure-notifications-server/pkg/logging" "github.com/google/exposure-notifications-server/pkg/observability" "github.com/google/exposure-notifications-server/pkg/server" @@ -96,6 +97,16 @@ func realMain(ctx context.Context) error { return fmt.Errorf("failed to create renderer: %w", err) } + // Get token key manager. + tokenSigner, err := keys.KeyManagerFor(ctx, &cfg.TokenSigning.Keys) + if err != nil { + return fmt.Errorf("failed to token signing key manager: %w", err) + } + tokenSignerTyp, ok := tokenSigner.(keys.SigningKeyManager) + if !ok { + return fmt.Errorf("token signing key manage is not a signing key manager (is %T)", tokenSigner) + } + // Create the router r := mux.NewRouter() @@ -110,10 +121,7 @@ func realMain(ctx context.Context) error { populateLogger := middleware.PopulateLogger(logger) r.Use(populateLogger) - cleanupController, err := cleanup.New(ctx, cfg, db, h) - if err != nil { - return fmt.Errorf("failed to create cleanup controller: %w", err) - } + cleanupController := cleanup.New(cfg, db, tokenSignerTyp, h) r.Handle("/", cleanupController.HandleCleanup()).Methods("GET") srv, err := server.New(cfg.Port) diff --git a/cmd/rotation/main.go b/cmd/rotation/main.go new file mode 100644 index 000000000..c647852db --- /dev/null +++ b/cmd/rotation/main.go @@ -0,0 +1,133 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This server implements the database cleanup. The server itself is unauthenticated +// and should not be deployed as a public service. +package main + +import ( + "context" + "fmt" + + "github.com/google/exposure-notifications-verification-server/pkg/buildinfo" + "github.com/google/exposure-notifications-verification-server/pkg/config" + "github.com/google/exposure-notifications-verification-server/pkg/controller/middleware" + "github.com/google/exposure-notifications-verification-server/pkg/controller/rotation" + "github.com/google/exposure-notifications-verification-server/pkg/render" + + "github.com/google/exposure-notifications-server/pkg/keys" + "github.com/google/exposure-notifications-server/pkg/logging" + "github.com/google/exposure-notifications-server/pkg/observability" + "github.com/google/exposure-notifications-server/pkg/server" + + "github.com/gorilla/mux" + "github.com/sethvargo/go-signalcontext" +) + +func main() { + ctx, done := signalcontext.OnInterrupt() + + logger := logging.NewLoggerFromEnv(). + With("build_id", buildinfo.BuildID). + With("build_tag", buildinfo.BuildTag) + ctx = logging.WithLogger(ctx, logger) + + defer func() { + done() + if r := recover(); r != nil { + logger.Fatalw("application panic", "panic", r) + } + }() + + err := realMain(ctx) + done() + + if err != nil { + logger.Fatal(err) + } + logger.Info("successful shutdown") +} + +func realMain(ctx context.Context) error { + logger := logging.FromContext(ctx) + + cfg, err := config.NewRotationConfig(ctx) + if err != nil { + return fmt.Errorf("failed to process config: %w", err) + } + + // Setup monitoring + logger.Info("configuring observability exporter") + oeConfig := cfg.ObservabilityExporterConfig() + oe, err := observability.NewFromEnv(oeConfig) + if err != nil { + return fmt.Errorf("unable to create ObservabilityExporter provider: %w", err) + } + if err := oe.StartExporter(ctx); err != nil { + return fmt.Errorf("error initializing observability exporter: %w", err) + } + defer oe.Close() + ctx, obs := middleware.WithObservability(ctx) + logger.Infow("observability exporter", "config", oeConfig) + + // Setup database + db, err := cfg.Database.Load(ctx) + if err != nil { + return fmt.Errorf("failed to load database config: %w", err) + } + if err := db.Open(ctx); err != nil { + return fmt.Errorf("failed to connect to database: %w", err) + } + defer db.Close() + + // Create the renderer + h, err := render.New(ctx, "", cfg.DevMode) + if err != nil { + return fmt.Errorf("failed to create renderer: %w", err) + } + + // Get token key manager. + tokenSigner, err := keys.KeyManagerFor(ctx, &cfg.TokenSigning.Keys) + if err != nil { + return fmt.Errorf("failed to token signing key manager: %w", err) + } + tokenSignerTyp, ok := tokenSigner.(keys.SigningKeyManager) + if !ok { + return fmt.Errorf("token signing key manage is not a signing key manager (is %T)", tokenSigner) + } + + // Create the router + r := mux.NewRouter() + + // Common observability context + r.Use(obs) + + // Request ID injection + populateRequestID := middleware.PopulateRequestID(h) + r.Use(populateRequestID) + + // Logger injection + populateLogger := middleware.PopulateLogger(logger) + r.Use(populateLogger) + + rotationController := rotation.New(cfg, db, tokenSignerTyp, h) + r.Handle("/", rotationController.HandleRotate()).Methods("GET") + + srv, err := server.New(cfg.Port) + if err != nil { + return fmt.Errorf("failed to create server: %w", err) + } + logger.Infow("server listening", "port", cfg.Port) + return srv.ServeHTTPHandler(ctx, r) +} diff --git a/pkg/config/cleanup_server_config.go b/pkg/config/cleanup_server_config.go index b8696a781..1e4fd0ad6 100644 --- a/pkg/config/cleanup_server_config.go +++ b/pkg/config/cleanup_server_config.go @@ -31,20 +31,28 @@ type CleanupConfig struct { Database database.Config Observability observability.Config + // TokenSigning is the token signing configuration to purge old keys in the + // key manager when they are cleaned. + TokenSigning TokenSigningConfig + // DevMode produces additional debugging information. Do not enable in // production environments. DevMode bool `env:"DEV_MODE"` + // Port is the port on which to bind. Port string `env:"PORT,default=8080"` - RateLimit uint64 `env:"RATE_LIMIT,default=60"` - // Cleanup config AuditEntryMaxAge time.Duration `env:"AUDIT_ENTRY_MAX_AGE, default=720h"` AuthorizedAppMaxAge time.Duration `env:"AUTHORIZED_APP_MAX_AGE, default=336h"` CleanupPeriod time.Duration `env:"CLEANUP_PERIOD, default=15m"` MobileAppMaxAge time.Duration `env:"MOBILE_APP_MAX_AGE, default=168h"` - UserPurgeMaxAge time.Duration `env:"USER_PURGE_MAX_AGE, default=720h"` + + // SigningTokenKeyMaxAge is the maximum amount of time that a rotated signing + // token key should remain unpurged. + SigningTokenKeyMaxAge time.Duration `env:"SIGNING_TOKEN_KEY_MAX_AGE, default=36h"` + + UserPurgeMaxAge time.Duration `env:"USER_PURGE_MAX_AGE, default=720h"` // VerificationCodeMaxAge is the period in which the full code should be available. // After this time it will be recycled. The code will be zeroed out, but its status persist. VerificationCodeMaxAge time.Duration `env:"VERIFICATION_CODE_MAX_AGE, default=48h"` diff --git a/pkg/config/rotation_config.go b/pkg/config/rotation_config.go new file mode 100644 index 000000000..541c43c61 --- /dev/null +++ b/pkg/config/rotation_config.go @@ -0,0 +1,84 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "context" + "time" + + "github.com/google/exposure-notifications-verification-server/pkg/database" + + "github.com/google/exposure-notifications-server/pkg/observability" + + "github.com/sethvargo/go-envconfig" +) + +// RotationConfig represents the environment-based configuration for the +// rotation service. +type RotationConfig struct { + Database database.Config + Observability observability.Config + + // Port is the port upon which to bind. + Port string `env:"PORT, default=8080"` + + // DevMode produces additional debugging information. Do not enable in + // production environments. + DevMode bool `env:"DEV_MODE"` + + // MinTTL is the minimum amount of time that must elapse between attempting + // rotation events. This is used to control whether rotation is actually + // attempted at the controller layer, independent of the data layer. In + // effect, it rate limits the number of rotation requests. + MinTTL time.Duration `env:"MIN_TTL, default=15m"` + + // TokenSigning is the token signing configuration. This defines the parent + // key and common data like issuer, but the individual versions are controlled + // by the database table. + TokenSigning TokenSigningConfig + + // TokenSigningKeyMaxAge is the maximum age for a token signing key. + TokenSigningKeyMaxAge time.Duration `env:"TOKEN_SIGNING_KEY_MAX_AGE, default=720h"` // 30 days +} + +// NewRotationConfig returns the config for the rotation service. +func NewRotationConfig(ctx context.Context) (*RotationConfig, error) { + var config RotationConfig + if err := ProcessWith(ctx, &config, envconfig.OsLookuper()); err != nil { + return nil, err + } + return &config, nil +} + +func (c *RotationConfig) Validate() error { + fields := []struct { + Var time.Duration + Name string + }{ + {c.TokenSigningKeyMaxAge, "TOKEN_SIGNING_KEY_MAX_AGE"}, + } + + for _, f := range fields { + if err := checkPositiveDuration(f.Var, f.Name); err != nil { + return err + } + } + + return nil +} + +func (c *RotationConfig) ObservabilityExporterConfig() *observability.Config { + return &c.Observability +} diff --git a/pkg/controller/cleanup/cleanup.go b/pkg/controller/cleanup/cleanup.go index ee80ddd2b..e77815c0e 100644 --- a/pkg/controller/cleanup/cleanup.go +++ b/pkg/controller/cleanup/cleanup.go @@ -21,6 +21,7 @@ import ( "net/http" "time" + "github.com/google/exposure-notifications-server/pkg/keys" "github.com/google/exposure-notifications-server/pkg/logging" "github.com/google/exposure-notifications-verification-server/pkg/config" "github.com/google/exposure-notifications-verification-server/pkg/database" @@ -33,18 +34,20 @@ import ( // Controller is a controller for the cleanup service. type Controller struct { - config *config.CleanupConfig - db *database.Database - h render.Renderer + config *config.CleanupConfig + db *database.Database + signingTokenKeyManager keys.SigningKeyManager + h render.Renderer } // New creates a new cleanup controller. -func New(ctx context.Context, config *config.CleanupConfig, db *database.Database, h render.Renderer) (*Controller, error) { +func New(config *config.CleanupConfig, db *database.Database, signingTokenKeyManager keys.SigningKeyManager, h render.Renderer) *Controller { return &Controller{ - config: config, - db: db, - h: h, - }, nil + config: config, + db: db, + signingTokenKeyManager: signingTokenKeyManager, + h: h, + } } func (c *Controller) shouldCleanup(ctx context.Context) (bool, error) { @@ -192,6 +195,19 @@ func (c *Controller) HandleCleanup() http.Handler { } }() + // Token signing keys + func() { + defer observability.RecordLatency(ctx, time.Now(), mLatencyMs, &result, &item) + item = tag.Upsert(itemTagKey, "TOKEN_SIGNING_KEY") + if count, err := c.db.PurgeTokenSigningKeys(ctx, c.signingTokenKeyManager, c.config.SigningTokenKeyMaxAge); err != nil { + merr = multierror.Append(merr, fmt.Errorf("failed to purge token signing keys: %w", err)) + result = observability.ResultError("FAILED") + } else { + logger.Infow("purged token signing keys", "count", count) + result = observability.ResultOK() + } + }() + // If there are any errors, return them if merr != nil { if errs := merr.WrappedErrors(); len(errs) > 0 { diff --git a/pkg/controller/cleanup/cleanup_test.go b/pkg/controller/cleanup/cleanup_test.go new file mode 100644 index 000000000..a68f9d5c8 --- /dev/null +++ b/pkg/controller/cleanup/cleanup_test.go @@ -0,0 +1,29 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cleanup + +import ( + "testing" + + "github.com/google/exposure-notifications-verification-server/pkg/database" +) + +var testDatabaseInstance *database.TestInstance + +func TestMain(m *testing.M) { + testDatabaseInstance = database.MustTestInstance() + defer testDatabaseInstance.MustClose() + m.Run() +} diff --git a/pkg/controller/rotation/handle_rotate.go b/pkg/controller/rotation/handle_rotate.go new file mode 100644 index 000000000..aeb286af8 --- /dev/null +++ b/pkg/controller/rotation/handle_rotate.go @@ -0,0 +1,123 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rotation + +import ( + "context" + "fmt" + "net/http" + "time" + + "github.com/google/exposure-notifications-server/pkg/logging" + "github.com/google/exposure-notifications-verification-server/pkg/observability" + "github.com/hashicorp/go-multierror" + "go.opencensus.io/stats" + "go.opencensus.io/tag" +) + +// HandleRotate handles key rotation. +func (c *Controller) HandleRotate() http.Handler { + type Result struct { + OK bool `json:"ok"` + Errors []error `json:"errors,omitempty"` + } + + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + logger := logging.FromContext(ctx).Named("rotation.HandleRotate") + + var merr *multierror.Error + + ok, err := c.shouldRotate(ctx) + if err != nil { + logger.Errorw("failed to run shouldRotate", "error", err) + c.h.RenderJSON(w, http.StatusInternalServerError, &Result{ + OK: false, + Errors: []error{err}, + }) + return + } + if !ok { + c.h.RenderJSON(w, http.StatusTooManyRequests, &Result{ + OK: false, + Errors: []error{fmt.Errorf("too early")}, + }) + } + + // Token signing keys + func() { + item := tag.Upsert(itemTagKey, "TOKEN_SIGNING_KEYS") + result := observability.ResultOK() + defer observability.RecordLatency(ctx, time.Now(), mLatencyMs, &result, &item) + + existing, err := c.db.ActiveTokenSigningKey() + if err != nil { + merr = multierror.Append(merr, fmt.Errorf("failed to lookup existing signing key: %w", err)) + result = observability.ResultError("FAILED") + return + } + if age, max := time.Now().UTC().Sub(existing.CreatedAt), c.config.TokenSigningKeyMaxAge; age < max { + logger.Debugw("token signing key does not require rotation", "age", age, "max", max) + return + } + + // TODO(sethvargo): figure out what to do with .TokenSigningKeys since it + // can be an array. + key, err := c.db.RotateTokenSigningKey(ctx, c.keyManager, c.config.TokenSigning.ActiveKey(), RotationActor) + if err != nil { + merr = multierror.Append(merr, fmt.Errorf("failed to rotate token signing key: %w", err)) + result = observability.ResultError("FAILED") + return + } + + logger.Infow("rotated token signing key", "new", key) + }() + + // If there are any errors, return them + if merr != nil { + if errs := merr.WrappedErrors(); len(errs) > 0 { + logger.Errorw("failed to rotate", "errors", errs) + c.h.RenderJSON(w, http.StatusInternalServerError, &Result{ + OK: false, + Errors: errs, + }) + return + } + } + + c.h.RenderJSON(w, http.StatusOK, &Result{ + OK: true, + }) + }) +} + +func (c *Controller) shouldRotate(ctx context.Context) (bool, error) { + cStat, err := c.db.CreateCleanup(rotationLock) + if err != nil { + return false, fmt.Errorf("failed to create rotation claim: %w", err) + } + + if cStat.NotBefore.After(time.Now().UTC()) { + return false, nil + } + + if _, err = c.db.ClaimCleanup(cStat, c.config.MinTTL); err != nil { + stats.RecordWithTags(ctx, []tag.Mutator{observability.ResultNotOK()}, mClaimRequests.M(1)) + return false, fmt.Errorf("failed to claim rotation: %w", err) + } + stats.RecordWithTags(ctx, []tag.Mutator{observability.ResultOK()}, mClaimRequests.M(1)) + return true, nil +} diff --git a/pkg/controller/rotation/handle_rotate_test.go b/pkg/controller/rotation/handle_rotate_test.go new file mode 100644 index 000000000..2fe2c66ef --- /dev/null +++ b/pkg/controller/rotation/handle_rotate_test.go @@ -0,0 +1,157 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rotation + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/google/exposure-notifications-server/pkg/keys" + "github.com/google/exposure-notifications-verification-server/internal/project" + "github.com/google/exposure-notifications-verification-server/pkg/config" + "github.com/google/exposure-notifications-verification-server/pkg/database" + "github.com/google/exposure-notifications-verification-server/pkg/render" +) + +func Test_shouldRotate(t *testing.T) { + t.Parallel() + + ttl := 1 * time.Second + + ctx := project.TestContext(t) + db, _ := testDatabaseInstance.NewDatabase(t, nil) + config := &config.RotationConfig{ + MinTTL: ttl, + } + c := New(config, db, nil, nil) + + if ok, err := c.shouldRotate(ctx); err != nil { + t.Fatal(err) + } else if !ok { + t.Fatalf("failed to claim lock when available") + } + + if ok, err := c.shouldRotate(ctx); err != nil { + t.Fatal(err) + } else if ok { + t.Fatalf("allowed to claim lock when it should not be available") + } + + time.Sleep(ttl) + + if ok, err := c.shouldRotate(ctx); err != nil { + t.Fatal(err) + } else if !ok { + t.Fatalf("failed to claim lock when available") + } +} + +func TestHandleRotate(t *testing.T) { + t.Parallel() + + ctx := project.TestContext(t) + + db, _ := testDatabaseInstance.NewDatabase(t, nil) + + keyManager := keys.TestKeyManager(t) + keyManagerSigner, ok := keyManager.(keys.SigningKeyManager) + if !ok { + t.Fatal("kms cannot manage signing keys") + } + tokenSigningKey := keys.TestSigningKey(t, keyManager) + + h, err := render.New(ctx, "", true) + if err != nil { + t.Fatal(err) + } + + config := &config.RotationConfig{ + TokenSigning: config.TokenSigningConfig{ + TokenSigningKeys: []string{tokenSigningKey}, + }, + TokenSigningKeyMaxAge: 30 * time.Second, + } + c := New(config, db, keyManagerSigner, h) + + t.Run("rotates", func(t *testing.T) { + t.Parallel() + + tokenSigningKeyVersion, err := keyManagerSigner.CreateKeyVersion(ctx, tokenSigningKey) + if err != nil { + t.Fatal(err) + } + + key := &database.TokenSigningKey{ + KeyVersionID: tokenSigningKeyVersion, + CreatedAt: time.Now().UTC().Add(-24 * time.Hour), + } + if err := db.SaveTokenSigningKey(key, database.SystemTest); err != nil { + t.Fatal(err) + } + if err := db.SaveTokenSigningKey(key, database.SystemTest); err != nil { + t.Fatal(err) + } + if err := db.ActivateTokenSigningKey(key.ID, database.SystemTest); err != nil { + t.Fatal(err) + } + + // Rotating should create a new key. + { + r, err := http.NewRequest("GET", "/", nil) + if err != nil { + t.Fatal(err) + } + r = r.Clone(ctx) + + w := httptest.NewRecorder() + + c.HandleRotate().ServeHTTP(w, r) + + keys, err := db.ListTokenSigningKeys() + if err != nil { + t.Fatal(err) + } + + if got, want := len(keys), 2; got != want { + t.Errorf("got %d keys, expected %d", got, want) + } + } + + // Rotating again should not create a new key (not enough time has elapsed + // since TokenSigningKeyMaxAge). + { + r, err := http.NewRequest("GET", "/", nil) + if err != nil { + t.Fatal(err) + } + r = r.Clone(ctx) + + w := httptest.NewRecorder() + + c.HandleRotate().ServeHTTP(w, r) + + keys, err := db.ListTokenSigningKeys() + if err != nil { + t.Fatal(err) + } + + if got, want := len(keys), 2; got != want { + t.Errorf("got %d keys, expected %d", got, want) + } + } + }) +} diff --git a/pkg/controller/rotation/metrics.go b/pkg/controller/rotation/metrics.go new file mode 100644 index 000000000..95ccbbbd7 --- /dev/null +++ b/pkg/controller/rotation/metrics.go @@ -0,0 +1,60 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rotation + +import ( + enobservability "github.com/google/exposure-notifications-server/pkg/observability" + "github.com/google/exposure-notifications-verification-server/pkg/observability" + + "go.opencensus.io/plugin/ochttp" + "go.opencensus.io/stats" + "go.opencensus.io/stats/view" + "go.opencensus.io/tag" +) + +const metricPrefix = observability.MetricRoot + "/rotation" + +var ( + mLatencyMs = stats.Float64(metricPrefix+"/requests", "The number of rotation requests.", stats.UnitMilliseconds) + mClaimRequests = stats.Int64(metricPrefix+"/claim_requests", "The number of rotation claim requests.", stats.UnitDimensionless) + + itemTagKey = tag.MustNewKey("item") +) + +func init() { + enobservability.CollectViews([]*view.View{ + { + Name: metricPrefix + "/requests_count", + Measure: mLatencyMs, + Description: "The count of the rotation requests", + TagKeys: append(observability.CommonTagKeys(), observability.ResultTagKey, itemTagKey), + Aggregation: view.Count(), + }, + { + Name: metricPrefix + "/requests_latency", + Measure: mLatencyMs, + Description: "The latency distribution of the rotation requests", + TagKeys: append(observability.CommonTagKeys(), observability.ResultTagKey, itemTagKey), + Aggregation: ochttp.DefaultLatencyDistribution, + }, + { + Name: metricPrefix + "/claim_requests_count", + Measure: mClaimRequests, + Description: "The count of the rotation claim requests", + TagKeys: append(observability.CommonTagKeys(), observability.ResultTagKey), + Aggregation: view.Count(), + }, + }...) +} diff --git a/pkg/controller/rotation/rotation.go b/pkg/controller/rotation/rotation.go new file mode 100644 index 000000000..ef0e41e2e --- /dev/null +++ b/pkg/controller/rotation/rotation.go @@ -0,0 +1,54 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package rotation implements periodic secret rotation. +package rotation + +import ( + "github.com/google/exposure-notifications-server/pkg/keys" + "github.com/google/exposure-notifications-verification-server/pkg/config" + "github.com/google/exposure-notifications-verification-server/pkg/database" + "github.com/google/exposure-notifications-verification-server/pkg/render" +) + +const rotationLock = "rotation" + +type Controller struct { + config *config.RotationConfig + db *database.Database + keyManager keys.SigningKeyManager + h render.Renderer +} + +func New(cfg *config.RotationConfig, db *database.Database, keyManager keys.SigningKeyManager, h render.Renderer) *Controller { + return &Controller{ + config: cfg, + db: db, + keyManager: keyManager, + h: h, + } +} + +// RotationActor is the actor in the database for rotation events. +var RotationActor database.Auditable = new(rotationActor) + +type rotationActor struct{} + +func (s *rotationActor) AuditID() string { + return "rotation:1" +} + +func (s *rotationActor) AuditDisplay() string { + return "Rotation" +} diff --git a/pkg/controller/rotation/rotation_test.go b/pkg/controller/rotation/rotation_test.go new file mode 100644 index 000000000..71fc5d051 --- /dev/null +++ b/pkg/controller/rotation/rotation_test.go @@ -0,0 +1,29 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rotation + +import ( + "testing" + + "github.com/google/exposure-notifications-verification-server/pkg/database" +) + +var testDatabaseInstance *database.TestInstance + +func TestMain(m *testing.M) { + testDatabaseInstance = database.MustTestInstance() + defer testDatabaseInstance.MustClose() + m.Run() +} diff --git a/pkg/controller/verifyapi/verify.go b/pkg/controller/verifyapi/verify.go index 55d3e206a..f16222668 100644 --- a/pkg/controller/verifyapi/verify.go +++ b/pkg/controller/verifyapi/verify.go @@ -134,6 +134,9 @@ func (c *Controller) HandleVerify() http.Handler { Subject: subject.String(), } token := jwt.NewWithClaims(jwt.SigningMethodES256, claims) + + // TODO(sethvargo): begin pulling this key from the token_signing_keys + // database instead. token.Header[verifyapi.KeyIDHeader] = c.config.TokenSigning.ActiveKeyID() signedJWT, err := jwthelper.SignJWT(token, signer) if err != nil { diff --git a/pkg/database/migrations.go b/pkg/database/migrations.go index 784656d2a..997ff8182 100644 --- a/pkg/database/migrations.go +++ b/pkg/database/migrations.go @@ -1980,6 +1980,25 @@ func (db *Database) Migrations(ctx context.Context) []*gormigrate.Migration { `ALTER TABLE realms DROP COLUMN IF EXISTS auto_rotate_certificate_key`) }, }, + { + ID: "00087-AddTokenSigningKeys", + Migrate: func(tx *gorm.DB) error { + return multiExec(tx, + `CREATE TABLE token_signing_keys ( + id BIGSERIAL, + key_version_id TEXT NOT NULL, + is_active BOOL NOT NULL DEFAULT FALSE, + created_at TIMESTAMP WITH TIME ZONE, + updated_at TIMESTAMP WITH TIME ZONE, + PRIMARY KEY (id))`, + `CREATE UNIQUE INDEX uix_token_signing_keys_is_active ON token_signing_keys (is_active) WHERE (is_active IS TRUE)`, + ) + }, + Rollback: func(tx *gorm.DB) error { + return multiExec(tx, + `DROP TABLE IF EXISTS token_signing_keys`) + }, + }, } } @@ -2004,10 +2023,12 @@ func (db *Database) MigrateTo(ctx context.Context, target string, rollback bool) // multiExec is a helper that executes the given sql clauses against the tx. func multiExec(tx *gorm.DB, sqls ...string) error { - for _, sql := range sqls { - if err := tx.Exec(sql).Error; err != nil { - return fmt.Errorf("failed to execute %q: %w", sql, err) + return tx.Transaction(func(tx *gorm.DB) error { + for _, sql := range sqls { + if err := tx.Exec(sql).Error; err != nil { + return fmt.Errorf("failed to execute %q: %w", sql, err) + } } - } - return nil + return nil + }) } diff --git a/pkg/database/token_signing_keys.go b/pkg/database/token_signing_keys.go new file mode 100644 index 000000000..911587217 --- /dev/null +++ b/pkg/database/token_signing_keys.go @@ -0,0 +1,218 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package database + +import ( + "context" + "fmt" + "time" + + "github.com/google/exposure-notifications-server/pkg/keys" + "github.com/jinzhu/gorm" +) + +// TokenSigningKey represents a collection of references to a KMS-backed signing +// key version for verification token signing. It is also used to track rotation +// schedules. +type TokenSigningKey struct { + Errorable + + // ID is the database auto-incrementing integer of the key. + ID uint64 + + // KeyVersionID is the full name of the signing key version. + KeyVersionID string + + // IsActive returns true if this signing key is the active one, false + // otherwise. There's a database-level constraint that only one row can have + // this value as true, so there is guaranteed to be exactly one active key at + // a time. + IsActive bool + + // CreatedAt is when the key was created and added to the system. UpdatedAt is + // when the key was last updated, which includes marking it as inactive. + CreatedAt time.Time + UpdatedAt time.Time +} + +// Ensure signing key can be an audited. +var _ Auditable = (*TokenSigningKey)(nil) + +// AuditID is how the user is stored in the audit entry. +func (k *TokenSigningKey) AuditID() string { + return fmt.Sprintf("token_signing_key:%d", k.ID) +} + +// AuditDisplay is how the user will be displayed in audit entries. +func (k *TokenSigningKey) AuditDisplay() string { + return fmt.Sprintf("%d (%s)", k.ID, k.KeyVersionID) +} + +// FindTokenSigningKey finds the given key by database ID. It returns an error +// if the record is not found. +func (db *Database) FindTokenSigningKey(id interface{}) (*TokenSigningKey, error) { + var key TokenSigningKey + if err := db.db. + Model(&TokenSigningKey{}). + Where("id = ?", id). + First(&key). + Error; err != nil { + return nil, err + } + return &key, nil +} + +// ActiveTokenSigningKey returns the currently-active token signing key. If no +// key is currently marked as active, it returns NotFound. +func (db *Database) ActiveTokenSigningKey() (*TokenSigningKey, error) { + var key TokenSigningKey + if err := db.db. + Model(&TokenSigningKey{}). + Where("is_active IS TRUE"). + First(&key). + Error; err != nil { + return nil, err + } + return &key, nil +} + +// ListTokenSigningKeys lists all keys sorted by their active state, then +// creation state descending. If there are no keys, it returns an empty list. To +// get the current active signing key, use ActiveTokenSigningKey. +func (db *Database) ListTokenSigningKeys() ([]*TokenSigningKey, error) { + var keys []*TokenSigningKey + if err := db.db. + Model(&TokenSigningKey{}). + Find(&keys). + Error; err != nil { + if IsNotFound(err) { + return keys, nil + } + return nil, err + } + return keys, nil +} + +// SaveTokenSigningKey saves the token signing key. +func (db *Database) SaveTokenSigningKey(key *TokenSigningKey, actor Auditable) error { + // TODO(sethvargo): auditing + return db.db.Save(key).Error +} + +// ActivateTokenSigningKey activates the signing key with the provided database +// ID. If no record corresponds to the given ID, an error is returned. If the +// given ID is already active, no action is taken. Otherwise, all existing key +// versions are marked as inactive and this key is marked as active. +func (db *Database) ActivateTokenSigningKey(id interface{}, actor Auditable) error { + return db.db.Transaction(func(tx *gorm.DB) error { + // Lookup the existing key. + var existing TokenSigningKey + if err := tx. + Set("gorm:query_option", "FOR UPDATE"). + Model(&TokenSigningKey{}). + Where("id = ?", id). + First(&existing). + Error; err != nil { + return fmt.Errorf("failed to find existing key version %s: %w", id, err) + } + + // If the provided key is already active, do not attempt to re-activate it. + if existing.IsActive { + return nil + } + + // Disable old actives. + if err := tx. + Model(&TokenSigningKey{}). + Where("is_active = ?", true). + Update("is_active", false). + Error; err != nil { + return fmt.Errorf("failed to deactivate old key versions: %w", err) + } + + // Enable new active version. + existing.IsActive = true + if err := tx.Save(existing).Error; err != nil { + return fmt.Errorf("failed to activate key version: %w", err) + } + + // Audit. + audit := BuildAuditEntry(actor, "activated token signing key version", &existing, 0) + if err := tx.Save(audit).Error; err != nil { + return fmt.Errorf("failed to save audits: %w", err) + } + + return nil + }) +} + +// RotateTokenSigningKey creates a new key in the upstream kms provider. If +// creating the upstream key fails, an error is returned. If the upstream key is +// successfully created, a new TokenSigningKey record is created in the database +// (not yet active). Finally, the new key is set as the active key. +func (db *Database) RotateTokenSigningKey(ctx context.Context, kms keys.SigningKeyManager, parent string, actor Auditable) (*TokenSigningKey, error) { + result, err := kms.CreateKeyVersion(ctx, parent) + if err != nil { + return nil, fmt.Errorf("failed to create key version in upstream kms: %w", err) + } + + key := &TokenSigningKey{KeyVersionID: result} + if err := db.SaveTokenSigningKey(key, actor); err != nil { + return nil, fmt.Errorf("failed to save token signing key: %w", err) + } + + if err := db.ActivateTokenSigningKey(key.ID, actor); err != nil { + return nil, fmt.Errorf("failed to activate token signing key: %w", err) + } + + // Go lookup the key. Note that we don't just return the key here, because it + // might have mutated state from other operations. This ensures the result is + // fresh from the database upon return. + return db.FindTokenSigningKey(key.ID) +} + +// PurgeTokenSigningKeys will delete token signing keys that have been rotated +// more than the provided max age. +func (db *Database) PurgeTokenSigningKeys(ctx context.Context, kms keys.KeyVersionDestroyer, maxAge time.Duration) (int64, error) { + if maxAge > 0 { + maxAge = -1 * maxAge + } + rotatedBefore := time.Now().UTC().Add(maxAge) + + // Select all keys currently targeted for deletion. + var keys []*TokenSigningKey + if err := db.db. + Unscoped(). + Where("is_active IS FALSE AND updated_at IS NOT NULL AND updated_at < ?", rotatedBefore). + Find(&keys). + Error; err != nil { + return 0, fmt.Errorf("failed to find existing keys: %w", err) + } + + // Iterate over each key and attempt to delete. + for _, key := range keys { + // Destroy upstream. + if err := kms.DestroyKeyVersion(ctx, key.KeyVersionID); err != nil { + return 0, fmt.Errorf("failed to destroy key version %q: %w", key.KeyVersionID, err) + } + + // Delete from database. + if err := db.db.Unscoped().Delete(key).Error; err != nil { + return 0, fmt.Errorf("failed to delete key version %d: %w", key.ID, err) + } + } + + return int64(len(keys)), nil +} diff --git a/pkg/database/token_signing_keys_test.go b/pkg/database/token_signing_keys_test.go new file mode 100644 index 000000000..2bc8c1674 --- /dev/null +++ b/pkg/database/token_signing_keys_test.go @@ -0,0 +1,281 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package database + +import ( + "testing" + "time" + + "github.com/google/exposure-notifications-server/pkg/keys" + "github.com/google/exposure-notifications-verification-server/internal/project" +) + +func TestDatabase_FindTokenSigningKey(t *testing.T) { + t.Parallel() + + db, _ := testDatabaseInstance.NewDatabase(t, nil) + + t.Run("not_exist", func(t *testing.T) { + t.Parallel() + + if _, err := db.FindTokenSigningKey(60221023); !IsNotFound(err) { + t.Errorf("expected err to be NotFound, got %v", err) + } + }) + + t.Run("finds", func(t *testing.T) { + t.Parallel() + + key := &TokenSigningKey{ + KeyVersionID: "foo/bar/baz", + } + if err := db.SaveTokenSigningKey(key, SystemTest); err != nil { + t.Fatal(err) + } + + result, err := db.FindTokenSigningKey(key.ID) + if err != nil { + t.Fatal(err) + } + + if got, want := result.ID, result.ID; got != want { + t.Errorf("expected %d to be %d", got, want) + } + }) +} + +func TestDatabase_ActiveTokenSigningKey(t *testing.T) { + t.Parallel() + + db, _ := testDatabaseInstance.NewDatabase(t, nil) + + key := &TokenSigningKey{ + KeyVersionID: "foo/bar/baz", + } + + // Note: intentionally NOT parallel + t.Run("none", func(t *testing.T) { + if _, err := db.ActiveTokenSigningKey(); !IsNotFound(err) { + t.Errorf("expected err to be NotFound, got %v", err) + } + }) + + t.Run("exists_not_active", func(t *testing.T) { + if err := db.SaveTokenSigningKey(key, SystemTest); err != nil { + t.Fatal(err) + } + + if _, err := db.ActiveTokenSigningKey(); !IsNotFound(err) { + t.Errorf("expected err to be NotFound, got %v", err) + } + }) + + t.Run("active", func(t *testing.T) { + if err := db.ActivateTokenSigningKey(key.ID, SystemTest); err != nil { + t.Fatal(err) + } + + result, err := db.ActiveTokenSigningKey() + if err != nil { + t.Fatal(err) + } + + if got, want := result.ID, result.ID; got != want { + t.Errorf("expected %d to be %d", got, want) + } + }) +} + +func TestDatabase_ListTokenSigningKeys(t *testing.T) { + t.Parallel() + + db, _ := testDatabaseInstance.NewDatabase(t, nil) + + // Note: intentionally NOT parallel + t.Run("none", func(t *testing.T) { + result, err := db.ListTokenSigningKeys() + if err != nil { + t.Fatal(err) + } + + if result == nil { + t.Fatal("result should not be nil") + } + }) + + t.Run("lists", func(t *testing.T) { + key := &TokenSigningKey{ + KeyVersionID: "foo/bar/baz", + } + if err := db.SaveTokenSigningKey(key, SystemTest); err != nil { + t.Fatal(err) + } + + list, err := db.ListTokenSigningKeys() + if err != nil { + t.Fatal(err) + } + if got, want := len(list), 1; got != want { + t.Fatalf("expected %d to be %d", got, want) + } + + if got, want := list[0].ID, key.ID; got != want { + t.Errorf("expected %d to be %d", got, want) + } + }) +} + +func TestDatabase_RotateTokenSigningKey(t *testing.T) { + t.Parallel() + + ctx := project.TestContext(t) + db, _ := testDatabaseInstance.NewDatabase(t, nil) + keyManager := keys.TestKeyManager(t) + keyManagerSigner, ok := keyManager.(keys.SigningKeyManager) + if !ok { + t.Fatal("kms cannot manage signing keys") + } + tokenSigningKey := keys.TestSigningKey(t, keyManager) + + key, err := db.RotateTokenSigningKey(ctx, keyManagerSigner, tokenSigningKey, SystemTest) + if err != nil { + t.Fatal(err) + } + + if !key.IsActive { + t.Error("key is not active") + } +} + +func TestDatabase_PurgeTokenSigningKeys(t *testing.T) { + t.Parallel() + + ctx := project.TestContext(t) + db, _ := testDatabaseInstance.NewDatabase(t, nil) + keyManager := keys.TestKeyManager(t) + keyManagerSigner, ok := keyManager.(keys.SigningKeyManager) + if !ok { + t.Fatal("kms cannot manage signing keys") + } + + tokenSigningKey := keys.TestSigningKey(t, keyManager) + + for i := 0; i < 5; i++ { + tokenSigningKeyVersion, err := keyManagerSigner.CreateKeyVersion(ctx, tokenSigningKey) + if err != nil { + t.Fatal(err) + } + + key := &TokenSigningKey{ + KeyVersionID: tokenSigningKeyVersion, + } + if err := db.SaveTokenSigningKey(key, SystemTest); err != nil { + t.Fatal(err) + } + if err := db.ActivateTokenSigningKey(key.ID, SystemTest); err != nil { + t.Fatal(err) + } + } + + // Should not purge entries (too young). + { + n, err := db.PurgeTokenSigningKeys(ctx, keyManagerSigner, 24*time.Hour) + if err != nil { + t.Fatal(err) + } + if got, want := n, int64(0); got != want { + t.Errorf("expected %d to purge, got %d", want, got) + } + } + + // Purges entries. + { + n, err := db.PurgeTokenSigningKeys(ctx, keyManagerSigner, 1*time.Nanosecond) + if err != nil { + t.Fatal(err) + } + if got, want := n, int64(4); got != want { + t.Errorf("expected %d to purge, got %d", want, got) + } + } +} + +func TestDatabase_ActivateTokenSigningKey(t *testing.T) { + t.Parallel() + + ctx := project.TestContext(t) + + keyManager := keys.TestKeyManager(t) + keyManagerSigner, ok := keyManager.(keys.SigningKeyManager) + if !ok { + t.Fatal("kms cannot manage signing keys") + } + + tokenSigningKey := keys.TestSigningKey(t, keyManager) + tokenSigningKeyVersion, err := keyManagerSigner.CreateKeyVersion(ctx, tokenSigningKey) + if err != nil { + t.Fatal(err) + } + + t.Run("activates", func(t *testing.T) { + db, _ := testDatabaseInstance.NewDatabase(t, nil) + + key := &TokenSigningKey{ + KeyVersionID: tokenSigningKeyVersion, + } + if err := db.SaveTokenSigningKey(key, SystemTest); err != nil { + t.Fatal(err) + } + if err := db.ActivateTokenSigningKey(key.ID, SystemTest); err != nil { + t.Fatal(err) + } + + // Reload the key from the database. + updatedKey, err := db.FindTokenSigningKey(key.ID) + if err != nil { + t.Fatal(err) + } + if got, want := updatedKey.IsActive, true; got != want { + t.Errorf("expected is_active to be %t, got %t", want, got) + } + + // Do it again to test "already active" condition. + if err := db.ActivateTokenSigningKey(key.ID, SystemTest); err != nil { + t.Fatal(err) + } + }) + + t.Run("audits", func(t *testing.T) { + db, _ := testDatabaseInstance.NewDatabase(t, nil) + + key := &TokenSigningKey{ + KeyVersionID: tokenSigningKeyVersion, + } + if err := db.SaveTokenSigningKey(key, SystemTest); err != nil { + t.Fatal(err) + } + if err := db.ActivateTokenSigningKey(key.ID, SystemTest); err != nil { + t.Fatal(err) + } + + audits, _, err := db.ListAudits(nil) + if err != nil { + t.Fatal(err) + } + if got, want := len(audits), 1; got != want { + t.Errorf("expected %d audits, got %d: %#v", want, got, audits) + } + }) +} diff --git a/terraform/service_rotation.tf b/terraform/service_rotation.tf new file mode 100644 index 000000000..81a4dd45a --- /dev/null +++ b/terraform/service_rotation.tf @@ -0,0 +1,225 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "google_service_account" "rotation" { + project = var.project + account_id = "en-verification-rotation-sa" + display_name = "Verification rotation" +} + +resource "google_service_account_iam_member" "cloudbuild-deploy-rotation" { + service_account_id = google_service_account.rotation.id + role = "roles/iam.serviceAccountUser" + member = "serviceAccount:${data.google_project.project.number}@cloudbuild.gserviceaccount.com" + + depends_on = [ + google_project_service.services["cloudbuild.googleapis.com"], + google_project_service.services["iam.googleapis.com"], + ] +} + +resource "google_secret_manager_secret_iam_member" "rotation-db" { + for_each = toset([ + "sslcert", + "sslkey", + "sslrootcert", + "password", + ]) + + secret_id = google_secret_manager_secret.db-secret[each.key].id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.rotation.email}" +} + +resource "google_project_iam_member" "rotation-observability" { + for_each = toset([ + "roles/cloudtrace.agent", + "roles/logging.logWriter", + "roles/monitoring.metricWriter", + "roles/stackdriver.resourceMetadata.writer", + ]) + + project = var.project + role = each.key + member = "serviceAccount:${google_service_account.rotation.email}" +} + +resource "google_kms_crypto_key_iam_member" "rotation-database-encrypter" { + crypto_key_id = google_kms_crypto_key.database-encrypter.self_link + role = "roles/cloudkms.cryptoKeyEncrypterDecrypter" + member = "serviceAccount:${google_service_account.rotation.email}" +} + +resource "google_secret_manager_secret_iam_member" "rotation-db-apikey-db-hmac" { + secret_id = google_secret_manager_secret.db-apikey-db-hmac.id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.rotation.email}" +} + +resource "google_secret_manager_secret_iam_member" "rotation-db-apikey-sig-hmac" { + secret_id = google_secret_manager_secret.db-apikey-sig-hmac.id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.rotation.email}" +} + +resource "google_secret_manager_secret_iam_member" "rotation-db-verification-code-hmac" { + secret_id = google_secret_manager_secret.db-verification-code-hmac.id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.rotation.email}" +} + +resource "google_secret_manager_secret_iam_member" "rotation-cache-hmac-key" { + secret_id = google_secret_manager_secret.cache-hmac-key.id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.rotation.email}" +} + +resource "google_secret_manager_secret_iam_member" "rotation-ratelimit-hmac-key" { + secret_id = google_secret_manager_secret.ratelimit-hmac-key.id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.rotation.email}" +} + +resource "google_cloud_run_service" "rotation" { + name = "rotation" + location = var.region + + autogenerate_revision_name = true + + metadata { + annotations = merge( + local.default_service_annotations, + var.default_service_annotations_overrides, + lookup(var.service_annotations, "rotation", {}) + ) + } + template { + spec { + service_account_name = google_service_account.rotation.email + + containers { + image = "gcr.io/${var.project}/github.com/google/exposure-notifications-verification-server/rotation:initial" + + resources { + limits = { + cpu = "1" + memory = "512Mi" + } + } + + + dynamic "env" { + for_each = merge( + local.database_config, + local.gcp_config, + local.signing_config, + local.observability_config, + + // This MUST come last to allow overrides! + lookup(var.service_environment, "rotation", {}), + ) + + content { + name = env.key + value = env.value + } + } + } + } + + metadata { + annotations = merge( + local.default_revision_annotations, + var.default_revision_annotations_overrides, + lookup(var.revision_annotations, "rotation", {}) + ) + } + } + + depends_on = [ + google_project_service.services["run.googleapis.com"], + + google_secret_manager_secret_iam_member.rotation-db, + google_project_iam_member.rotation-observability, + google_kms_crypto_key_iam_member.rotation-database-encrypter, + google_secret_manager_secret_iam_member.rotation-db-apikey-db-hmac, + google_secret_manager_secret_iam_member.rotation-db-apikey-sig-hmac, + google_secret_manager_secret_iam_member.rotation-db-verification-code-hmac, + google_secret_manager_secret_iam_member.rotation-cache-hmac-key, + google_secret_manager_secret_iam_member.rotation-ratelimit-hmac-key, + + null_resource.build, + null_resource.migrate, + ] + + lifecycle { + ignore_changes = [ + template[0].metadata[0].annotations["client.knative.dev/user-image"], + template[0].metadata[0].annotations["run.googleapis.com/client-name"], + template[0].metadata[0].annotations["run.googleapis.com/client-version"], + template[0].spec[0].containers[0].image, + metadata[0].annotations["run.googleapis.com/ingress-status"], + metadata[0].labels["cloud.googleapis.com/location"], + ] + } +} + +output "rotation_url" { + value = google_cloud_run_service.rotation.status.0.url +} + +# +# Create scheduler job to invoke the service on a fixed interval. +# + +resource "google_service_account" "rotation-invoker" { + project = data.google_project.project.project_id + account_id = "en-rotation-invoker-sa" + display_name = "Verification rotation invoker" +} + +resource "google_cloud_run_service_iam_member" "rotation-invoker" { + project = google_cloud_run_service.rotation.project + location = google_cloud_run_service.rotation.location + service = google_cloud_run_service.rotation.name + role = "roles/run.invoker" + member = "serviceAccount:${google_service_account.rotation-invoker.email}" +} + +resource "google_cloud_scheduler_job" "rotation-worker" { + name = "rotation-worker" + region = var.cloudscheduler_location + schedule = "0 */6 * * *" + time_zone = "America/Los_Angeles" + attempt_deadline = "600s" + + retry_config { + retry_count = 1 + } + + http_target { + http_method = "GET" + uri = "${google_cloud_run_service.rotation.status.0.url}/" + oidc_token { + audience = google_cloud_run_service.rotation.status.0.url + service_account_email = google_service_account.rotation-invoker.email + } + } + + depends_on = [ + google_app_engine_application.app, + google_cloud_run_service_iam_member.rotation-invoker, + google_project_service.services["cloudscheduler.googleapis.com"], + ] +} From f979cda4225d5f6613b917231c9c8267fbf62c9a Mon Sep 17 00:00:00 2001 From: Seth Vargo Date: Thu, 14 Jan 2021 18:19:57 -0500 Subject: [PATCH 2/4] Return after ok --- pkg/controller/rotation/handle_rotate.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/controller/rotation/handle_rotate.go b/pkg/controller/rotation/handle_rotate.go index aeb286af8..ec9841bd6 100644 --- a/pkg/controller/rotation/handle_rotate.go +++ b/pkg/controller/rotation/handle_rotate.go @@ -55,6 +55,7 @@ func (c *Controller) HandleRotate() http.Handler { OK: false, Errors: []error{fmt.Errorf("too early")}, }) + return } // Token signing keys From ccc40f2e9bbb5063f03cad8db1e814a2d90febda Mon Sep 17 00:00:00 2001 From: Seth Vargo Date: Thu, 14 Jan 2021 18:20:14 -0500 Subject: [PATCH 3/4] Cleanup needs to return after not-ok too --- pkg/controller/cleanup/cleanup.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/controller/cleanup/cleanup.go b/pkg/controller/cleanup/cleanup.go index e77815c0e..f08d572fb 100644 --- a/pkg/controller/cleanup/cleanup.go +++ b/pkg/controller/cleanup/cleanup.go @@ -96,6 +96,7 @@ func (c *Controller) HandleCleanup() http.Handler { OK: false, Errors: []error{fmt.Errorf("too early")}, }) + return } // Construct a multi-error. If one of the purges fails, we still want to From 6ff548ab261363a5b9140b06758082f4d90f027a Mon Sep 17 00:00:00 2001 From: Seth Vargo Date: Thu, 14 Jan 2021 20:00:09 -0500 Subject: [PATCH 4/4] Add builders --- builders/build.yaml | 50 +++++++++++++++++++++++++++++++++++++++++++ builders/deploy.yaml | 23 ++++++++++++++++++++ builders/promote.yaml | 20 +++++++++++++++++ 3 files changed, 93 insertions(+) diff --git a/builders/build.yaml b/builders/build.yaml index d7fa0612f..b81ebc714 100644 --- a/builders/build.yaml +++ b/builders/build.yaml @@ -461,6 +461,56 @@ steps: - 'push-modeler' +# +# rotation +# +- id: 'build-rotation' + name: 'golang:1.15.2' + args: + - 'go' + - 'build' + - '-trimpath' + - '-ldflags=-s -w -X=${_REPO}/pkg/buildinfo.BuildID=${BUILD_ID} -X=${_REPO}/pkg/buildinfo.BuildTag=${_TAG} -extldflags=-static' + - '-o=./bin/rotation' + - './cmd/rotation' + waitFor: + - 'download-modules' + +- id: 'dockerize-rotation' + name: 'docker:19' + args: + - 'build' + - '--file=builders/service.dockerfile' + - '--tag=gcr.io/${PROJECT_ID}/${_REPO}/rotation:${_TAG}' + - '--build-arg=SERVICE=rotation' + - '.' + waitFor: + - 'build-rotation' + +- id: 'push-rotation' + name: 'docker:19' + args: + - 'push' + - 'gcr.io/${PROJECT_ID}/${_REPO}/rotation:${_TAG}' + +- id: 'attest-rotation' + name: 'gcr.io/google.com/cloudsdktool/cloud-sdk:307.0.0' + args: + - 'bash' + - '-eEuo' + - 'pipefail' + - '-c' + - |- + ARTIFACT_URL=$(docker inspect gcr.io/${PROJECT_ID}/${_REPO}/rotation:${_TAG} --format='{{index .RepoDigests 0}}') + gcloud beta container binauthz attestations sign-and-create \ + --project "${PROJECT_ID}" \ + --artifact-url "$${ARTIFACT_URL}" \ + --attestor "${_BINAUTHZ_ATTESTOR}" \ + --keyversion "${_BINAUTHZ_KEY_VERSION}" + waitFor: + - 'push-rotation' + + # # server # diff --git a/builders/deploy.yaml b/builders/deploy.yaml index fab285a2a..0a7f4e1eb 100644 --- a/builders/deploy.yaml +++ b/builders/deploy.yaml @@ -169,6 +169,29 @@ steps: waitFor: - '-' + +# +# rotation +# +- id: 'deploy-rotation' + name: 'gcr.io/google.com/cloudsdktool/cloud-sdk:307.0.0-alpine' + args: + - 'bash' + - '-eEuo' + - 'pipefail' + - '-c' + - |- + gcloud run deploy "rotation" \ + --quiet \ + --project "${PROJECT_ID}" \ + --platform "managed" \ + --region "${_REGION}" \ + --image "gcr.io/${PROJECT_ID}/${_REPO}/rotation:${_TAG}" \ + --no-traffic + waitFor: + - '-' + + # # server # diff --git a/builders/promote.yaml b/builders/promote.yaml index 4e937e17b..f94b218dd 100644 --- a/builders/promote.yaml +++ b/builders/promote.yaml @@ -162,6 +162,26 @@ steps: waitFor: - '-' +# +# rotation +# +- id: 'promote-rotation' + name: 'gcr.io/google.com/cloudsdktool/cloud-sdk:307.0.0-alpine' + args: + - 'bash' + - '-eEuo' + - 'pipefail' + - '-c' + - |- + gcloud run services update-traffic "rotation" \ + --quiet \ + --project "${PROJECT_ID}" \ + --platform "managed" \ + --region "${_REGION}" \ + --to-revisions "${_REVISION}=${_PERCENTAGE}" + waitFor: + - '-' + # # server #