From e6f39f9784ad1e7f7537b8da7d900985f13fc99e Mon Sep 17 00:00:00 2001 From: Yuchen Ying Date: Wed, 26 Aug 2020 15:14:16 -0700 Subject: [PATCH 1/5] Create e2e-runner service. This commit refactored the existing e2e-test CLI into a web service. It currently provides /default and /revise handlers, maps to the `-revise=false` and `-revise=true` flag of the original CLI. This service is currently guarded by an IAM policy to avoid hammering the API servers repeatedly. --- .gitignore | 2 +- .../e2e-test/main.go => cmd/e2e-runner/e2e.go | 66 ++---- cmd/e2e-runner/main.go | 173 ++++++++++++++++ pkg/config/e2e_runner_server_config.go | 70 +++++++ scripts/build | 2 +- scripts/deploy | 2 +- scripts/promote | 2 +- terraform/service_e2e_runner.tf | 193 ++++++++++++++++++ terraform/services.tf | 5 + 9 files changed, 457 insertions(+), 58 deletions(-) rename tools/e2e-test/main.go => cmd/e2e-runner/e2e.go (77%) create mode 100644 cmd/e2e-runner/main.go create mode 100644 pkg/config/e2e_runner_server_config.go create mode 100644 terraform/service_e2e_runner.tf diff --git a/.gitignore b/.gitignore index 302062a83..6654cab12 100644 --- a/.gitignore +++ b/.gitignore @@ -14,12 +14,12 @@ cmd/add-users/add-users cmd/adminapi/adminapi cmd/apiserver/apiserver cmd/cleanup/cleanup +cmd/e2e-runner/e2e-runner cmd/get-certificate/get-certificate cmd/get-code/get-code cmd/get-token/get-token cmd/migrate/migrate cmd/server/server -tools/e2e-test/e2e-test tools/gen-secret/gen-secret tools/seed/seed diff --git a/tools/e2e-test/main.go b/cmd/e2e-runner/e2e.go similarity index 77% rename from tools/e2e-test/main.go rename to cmd/e2e-runner/e2e.go index 4d4d3ebbe..377e70165 100644 --- a/tools/e2e-test/main.go +++ b/cmd/e2e-runner/e2e.go @@ -12,46 +12,28 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Command line test that exercises the verification and key server, -// simulating a mobile device uploading TEKs. -// +// E2E test code that exercises the verification and key server, simulating a +// mobile device uploading TEKs. // package main import ( "context" "encoding/base64" - "flag" "fmt" "net/http" - "os" - "strconv" "time" "github.com/google/exposure-notifications-verification-server/pkg/clients" + "github.com/google/exposure-notifications-verification-server/pkg/config" "github.com/google/exposure-notifications-verification-server/pkg/jsonclient" verifyapi "github.com/google/exposure-notifications-server/pkg/api/v1" "github.com/google/exposure-notifications-server/pkg/logging" "github.com/google/exposure-notifications-server/pkg/util" "github.com/google/exposure-notifications-server/pkg/verification" - - "github.com/sethvargo/go-envconfig" - "github.com/sethvargo/go-signalcontext" ) -type Config struct { - VerificationAdminAPIServer string `env:"VERIFICATION_ADMIN_API, default=http://localhost:8081"` - VerificationAdminAPIKey string `env:"VERIFICATION_ADMIN_API_KEY,required"` - VerificationAPIServer string `env:"VERIFICATION_SERVER_API, default=http://localhost:8082"` - VerificationAPIServerKey string `env:"VERIFICATION_SERVER_API_KEY,required"` - KeyServer string `env:"KEY_SERVER, default=http://localhost:8080"` - HealthAuthorityCode string `env:"HEALTH_AUTHORITY_CODE,required"` - - // Publish config - Region string `env:"REGION,default=US"` -} - const ( timeout = 2 * time.Second oneDay = 24 * time.Hour @@ -63,36 +45,12 @@ func timeToInterval(t time.Time) int32 { return int32(t.UTC().Truncate(oneDay).Unix() / int64(intervalLength.Seconds())) } -func main() { - ctx, done := signalcontext.OnInterrupt() - - debug, _ := strconv.ParseBool(os.Getenv("LOG_DEBUG")) - logger := logging.NewLogger(debug) - ctx = logging.WithLogger(ctx, logger) - - err := realMain(ctx) - done() - - if err != nil { - logger.Fatal(err) - } - logger.Info("successful shutdown") -} - -func realMain(ctx context.Context) error { +func e2e(ctx context.Context, config config.E2ERunnerConfig) error { logger := logging.FromContext(ctx) - var config Config - if err := envconfig.ProcessWith(ctx, &config, envconfig.OsLookuper()); err != nil { - return fmt.Errorf("unable to process environment: %w", err) - } - - doRevision := flag.Bool("revise", false, "--revise means to do a likely diagnosis and then revise to confirmed. one new key is added in between.") - verbose := flag.Bool("v", false, "ALL THE MESSAGES!") - flag.Parse() testType := "confirmed" iterations := 1 - if *doRevision { + if config.DoRevise { testType = "likely" iterations++ } @@ -122,7 +80,7 @@ func realMain(ctx context.Context) error { } else if code.Error != "" { return fmt.Errorf("issue API Error: %+v", code) } - if *verbose { + if config.Verbose { logger.Infof("Code Request: %+v", codeRequest) logger.Infof("Code Response: %+v", code) } @@ -135,7 +93,7 @@ func realMain(ctx context.Context) error { } else if token.Error != "" { return fmt.Errorf("verification API Error %+v", token) } - if *verbose { + if config.Verbose { logger.Infof("Token Request: %+v", tokenRequest) logger.Infof("Token Response: %+v", token) } @@ -147,7 +105,7 @@ func realMain(ctx context.Context) error { } else if codeStatus.Error != "" { return fmt.Errorf("check code status Error: %+v", codeStatus) } - if *verbose { + if config.Verbose { logger.Infof("Code Status Request: %+v", statusReq) logger.Infof("Code Status Response: %+v", codeStatus) } @@ -169,7 +127,7 @@ func realMain(ctx context.Context) error { } else if certificate.Error != "" { return fmt.Errorf("certificate API Error: %+v", certificate) } - if *verbose { + if config.Verbose { logger.Infof("Certificate Request: %+v", certRequest) logger.Infof("Certificate Response: %+v", certificate) } @@ -189,7 +147,7 @@ func realMain(ctx context.Context) error { client := &http.Client{ Timeout: timeout, } - if *verbose { + if config.Verbose { logger.Infof("Publish request: %+v", publish) } if err := jsonclient.MakeRequest(ctx, client, config.KeyServer, http.Header{}, &publish, &response); err != nil { @@ -198,11 +156,11 @@ func realMain(ctx context.Context) error { return fmt.Errorf("publish API error: %+v", response) } logger.Infof("Inserted %v exposures", response.InsertedExposures) - if *verbose { + if config.Verbose { logger.Infof("Publish response: %+v", response) } - if *doRevision { + if config.DoRevise { testType = "confirmed" revisionToken = response.RevisionToken diff --git a/cmd/e2e-runner/main.go b/cmd/e2e-runner/main.go new file mode 100644 index 000000000..b2da4ce5f --- /dev/null +++ b/cmd/e2e-runner/main.go @@ -0,0 +1,173 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This server is a simple webserver that triggers the e2e-test binary. +package main + +import ( + "context" + "fmt" + "math/rand" + "net/http" + "os" + "strconv" + "time" + + "github.com/google/exposure-notifications-server/pkg/logging" + "github.com/google/exposure-notifications-server/pkg/server" + "github.com/google/exposure-notifications-verification-server/pkg/config" + "github.com/google/exposure-notifications-verification-server/pkg/database" + + "github.com/gorilla/handlers" + "github.com/gorilla/mux" + "github.com/sethvargo/go-signalcontext" +) + +const ( + realmName = "e2e-test-realm" + realmRegionCode = "e2e-test" + adminKeyName = "e2e-admin-key." + deviceKeyName = "e2e-device-key." +) + +func main() { + ctx, done := signalcontext.OnInterrupt() + + debug, _ := strconv.ParseBool(os.Getenv("LOG_DEBUG")) + logger := logging.NewLogger(debug) + ctx = logging.WithLogger(ctx, logger) + + err := realMain(ctx) + done() + + if err != nil { + logger.Fatal(err) + } + logger.Info("successful shutdown") +} + +func randomString() string { + rand.Seed(time.Now().Unix()) + return fmt.Sprintf("%x", rand.Int63()) +} + +func realMain(ctx context.Context) error { + logger := logging.FromContext(ctx) + + // load configs + e2eConfig, err := config.NewE2ERunnerConfig(ctx) + if err != nil { + return fmt.Errorf("failed to process e2e-runner config: %w", err) + } + + db, err := e2eConfig.Database.Load(ctx) + if err != nil { + return fmt.Errorf("failed to load database config: %w", err) + } + if err := db.Open(ctx); err != nil { + return fmt.Errorf("failed to connect to database: %w", err) + } + defer db.Close() + + // Create or reuse the existing realm + realm, err := db.FindRealmByName(realmName) + if err != nil { + if !database.IsNotFound(err) { + return fmt.Errorf("error when finding the realm: %w", err) + } + realm = database.NewRealmWithDefaults(realmName) + realm.RegionCode = realmRegionCode + if err := db.SaveRealm(realm); err != nil { + return fmt.Errorf("failed to create realm: %w: %v", err, realm.ErrorMessages()) + } + } + + // Create new API keys + suffix := randomString() + + adminKey, err := realm.CreateAuthorizedApp(db, &database.AuthorizedApp{ + Name: adminKeyName + suffix, + APIKeyType: database.APIUserTypeAdmin, + }) + if err != nil { + return fmt.Errorf("error trying to create a new Admin API Key: %w", err) + } + + defer func() { + app, err := db.FindAuthorizedAppByAPIKey(adminKey) + if err != nil { + logger.Errorf("admin API key cleanup failed: %w", err) + } + if err := app.Disable(db); err != nil { + logger.Errorf("admin API key disable failed: %w", err) + } + logger.Info("successfully cleaned up e2e test admin key") + }() + + deviceKey, err := realm.CreateAuthorizedApp(db, &database.AuthorizedApp{ + Name: deviceKeyName + suffix, + APIKeyType: database.APIUserTypeDevice, + }) + if err != nil { + return fmt.Errorf("error trying to create a new Device API Key: %w", err) + } + + defer func() { + app, err := db.FindAuthorizedAppByAPIKey(deviceKey) + if err != nil { + logger.Errorf("device API key cleanup failed: %w", err) + } + if err := app.Disable(db); err != nil { + logger.Errorf("device API key disable failed: %w", err) + } + logger.Info("successfully cleaned up e2e test device key") + }() + + e2eConfig.VerificationAdminAPIKey = adminKey + e2eConfig.VerificationAPIServerKey = deviceKey + + // Create the router + r := mux.NewRouter() + r.HandleFunc("/default", defaultHandler(ctx, *e2eConfig)) + r.HandleFunc("/revise", reviseHandler(ctx, *e2eConfig)) + + srv, err := server.New(e2eConfig.Port) + if err != nil { + return fmt.Errorf("failed to create server: %w", err) + } + logger.Infow("server listening", "port", e2eConfig.Port) + return srv.ServeHTTPHandler(ctx, handlers.CombinedLoggingHandler(os.Stdout, r)) +} + +func defaultHandler(ctx context.Context, c config.E2ERunnerConfig) func(http.ResponseWriter, *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + c.DoRevise = false + if err := e2e(ctx, c); err != nil { + http.Error(w, "failed (check server logs for more details): "+err.Error(), http.StatusInternalServerError) + } else { + fmt.Fprint(w, "ok") + } + } +} + +func reviseHandler(ctx context.Context, c config.E2ERunnerConfig) func(http.ResponseWriter, *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + c.DoRevise = true + if err := e2e(ctx, c); err != nil { + http.Error(w, "failed (check server logs for more details): "+err.Error(), http.StatusInternalServerError) + } else { + fmt.Fprint(w, "ok") + } + } +} diff --git a/pkg/config/e2e_runner_server_config.go b/pkg/config/e2e_runner_server_config.go new file mode 100644 index 000000000..7d5745622 --- /dev/null +++ b/pkg/config/e2e_runner_server_config.go @@ -0,0 +1,70 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "context" + + "github.com/google/exposure-notifications-verification-server/pkg/database" + "github.com/sethvargo/go-envconfig" +) + +// E2ERunnerConfig represents the environment based configuration for the e2e-runner server. +type E2ERunnerConfig struct { + Database database.Config + + // DevMode produces additional debugging information. Do not enable in + // production environments. + DevMode bool `env:"DEV_MODE"` + + Port string `env:"PORT,default=8080"` + + // e2e-runner config + VerificationAdminAPIServer string `env:"VERIFICATION_ADMIN_API, default=http://localhost:8081"` + VerificationAdminAPIKey string + VerificationAPIServer string `env:"VERIFICATION_SERVER_API, default=http://localhost:8082"` + VerificationAPIServerKey string + KeyServer string `env:"KEY_SERVER, default=http://localhost:8080"` + HealthAuthorityCode string `env:"HEALTH_AUTHORITY_CODE,required"` + + // e2e-test config + DoRevise bool + Verbose bool +} + +// NewE2ERunnerConfig returns the environment config for the e2e-runner server. +// Only needs to be called once per instance, but may be called multiple times. +func NewE2ERunnerConfig(ctx context.Context) (*E2ERunnerConfig, error) { + var config E2ERunnerConfig + if err := ProcessWith(ctx, &config, envconfig.OsLookuper()); err != nil { + return nil, err + } + return &config, nil +} + +func (c *E2ERunnerConfig) Validate() error { + return nil +} + +func (c *E2ERunnerConfig) ToEnv() []string { + return []string{ + "VERIFICATION_ADMIN_API=" + c.VerificationAdminAPIServer, + "VERIFICATION_ADMIN_API_KEY=" + c.VerificationAdminAPIKey, + "VERIFICATION_SERVER_API=" + c.VerificationAPIServer, + "VERIFICATION_SERVER_API_KEY=" + c.VerificationAPIServerKey, + "KEY_SERVER=" + c.KeyServer, + "HEALTH_AUTHORITY_CODE=" + c.HealthAuthorityCode, + } +} diff --git a/scripts/build b/scripts/build index 10b42514f..e9868ad72 100755 --- a/scripts/build +++ b/scripts/build @@ -17,7 +17,7 @@ set -eEuo pipefail ROOT="$(cd "$(dirname "$0")/.." &>/dev/null; pwd -P)" -ALL_SERVICES="apiserver,cleanup,server,adminapi" +ALL_SERVICES="apiserver,cleanup,server,adminapi,e2e-runner" if [ -n "$(git status --porcelain)" ]; then echo "✋ Uncommitted local changes!" >&2 diff --git a/scripts/deploy b/scripts/deploy index 999465b47..845fc2183 100755 --- a/scripts/deploy +++ b/scripts/deploy @@ -17,7 +17,7 @@ set -eEuo pipefail ROOT="$(cd "$(dirname "$0")/.." &>/dev/null; pwd -P)" -ALL_SERVICES="apiserver,cleanup,server,adminapi" +ALL_SERVICES="apiserver,cleanup,server,adminapi,e2e-runner" if [ -z "${PROJECT_ID:-}" ]; then echo "✋ Missing PROJECT_ID!" >&2 diff --git a/scripts/promote b/scripts/promote index 13fcab3fa..4f3613fef 100755 --- a/scripts/promote +++ b/scripts/promote @@ -17,7 +17,7 @@ set -eEuo pipefail ROOT="$(cd "$(dirname "$0")/.." &>/dev/null; pwd -P)" -ALL_SERVICES="apiserver,cleanup,server,adminapi" +ALL_SERVICES="apiserver,cleanup,server,adminapi,e2e-runner" if [ -z "${PROJECT_ID:-}" ]; then echo "✋ Missing PROJECT_ID!" >&2 diff --git a/terraform/service_e2e_runner.tf b/terraform/service_e2e_runner.tf new file mode 100644 index 000000000..59f653a45 --- /dev/null +++ b/terraform/service_e2e_runner.tf @@ -0,0 +1,193 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "google_service_account" "e2e-runner" { + project = var.project + account_id = "en-verification-e2e-runner-sa" + display_name = "Verification e2e-runner" +} + +resource "google_service_account_iam_member" "cloudbuild-deploy-e2e-runner" { + service_account_id = google_service_account.e2e-runner.id + role = "roles/iam.serviceAccountUser" + member = "serviceAccount:${data.google_project.project.number}@cloudbuild.gserviceaccount.com" + + depends_on = [ + google_project_service.services["cloudbuild.googleapis.com"], + google_project_service.services["iam.googleapis.com"], + ] +} + +resource "google_secret_manager_secret_iam_member" "e2e-runner-db" { + for_each = toset([ + "sslcert", + "sslkey", + "sslrootcert", + "password", + ]) + + secret_id = google_secret_manager_secret.db-secret[each.key].id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.e2e-runner.email}" +} + +resource "google_project_iam_member" "e2e-runner-observability" { + for_each = toset([ + "roles/cloudtrace.agent", + "roles/logging.logWriter", + "roles/monitoring.metricWriter", + "roles/stackdriver.resourceMetadata.writer", + ]) + + project = var.project + role = each.key + member = "serviceAccount:${google_service_account.e2e-runner.email}" +} + +resource "google_kms_crypto_key_iam_member" "e2e-runner-database-encrypter" { + crypto_key_id = google_kms_crypto_key.database-encrypter.self_link + role = "roles/cloudkms.cryptoKeyEncrypterDecrypter" + member = "serviceAccount:${google_service_account.e2e-runner.email}" +} + +resource "google_secret_manager_secret_iam_member" "e2e-runner-db-apikey-db-hmac" { + secret_id = google_secret_manager_secret.db-apikey-db-hmac.id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.e2e-runner.email}" +} + +resource "google_secret_manager_secret_iam_member" "e2e-runner-db-apikey-sig-hmac" { + secret_id = google_secret_manager_secret.db-apikey-sig-hmac.id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.e2e-runner.email}" +} + +resource "google_secret_manager_secret_iam_member" "e2e-runner-db-verification-code-hmac" { + secret_id = google_secret_manager_secret.db-verification-code-hmac.id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.e2e-runner.email}" +} + +resource "google_cloud_run_service" "e2e-runner" { + name = "e2e-runner" + location = var.region + + autogenerate_revision_name = true + + template { + spec { + service_account_name = google_service_account.e2e-runner.email + + containers { + image = "gcr.io/${var.project}/github.com/google/exposure-notifications-verification-server/cmd/e2e-runner:initial" + + resources { + limits = { + cpu = "1" + memory = "512Mi" + } + } + + + dynamic "env" { + for_each = merge( + local.cache_config, + local.csrf_config, + local.database_config, + local.firebase_config, + local.gcp_config, + local.signing_config, + local.e2e_runner_config, + + // This MUST come last to allow overrides! + lookup(var.service_environment, "e2e-runner", {}), + ) + + content { + name = env.key + value = env.value + } + } + } + } + + metadata { + annotations = { + "run.googleapis.com/vpc-access-connector" : google_vpc_access_connector.connector.id + } + } + } + + depends_on = [ + google_project_service.services["run.googleapis.com"], + google_secret_manager_secret_iam_member.e2e-runner-db, + null_resource.build, + ] + + lifecycle { + ignore_changes = [ + template[0].metadata[0].annotations, + template[0].spec[0].containers[0].image, + ] + } +} + +output "e2e_runner_url" { + value = google_cloud_run_service.e2e-runner.status.0.url +} + +# +# Create scheduler job to invoke the service on a fixed interval. +# + +resource "google_service_account" "e2e-runner-invoker" { + project = data.google_project.project.project_id + account_id = "en-e2e-runner-invoker-sa" + display_name = "Verification e2e-runner invoker" +} + +resource "google_cloud_run_service_iam_member" "e2e-runner-invoker" { + project = google_cloud_run_service.e2e-runner.project + location = google_cloud_run_service.e2e-runner.location + service = google_cloud_run_service.e2e-runner.name + role = "roles/run.invoker" + member = "serviceAccount:${google_service_account.e2e-runner-invoker.email}" +} + +resource "google_cloud_scheduler_job" "e2e-runner-worker" { + name = "e2e-runner-worker" + region = var.cloudscheduler_location + schedule = "0 * * * *" + time_zone = "America/Los_Angeles" + attempt_deadline = "600s" + + retry_config { + retry_count = 1 + } + + http_target { + http_method = "GET" + uri = "${google_cloud_run_service.e2e-runner.status.0.url}/" + oidc_token { + audience = google_cloud_run_service.e2e-runner.status.0.url + service_account_email = google_service_account.e2e-runner-invoker.email + } + } + + depends_on = [ + google_app_engine_application.app, + google_cloud_run_service_iam_member.e2e-runner-invoker, + google_project_service.services["cloudscheduler.googleapis.com"], + ] +} diff --git a/terraform/services.tf b/terraform/services.tf index 9c2ff5f7e..daa327aeb 100644 --- a/terraform/services.tf +++ b/terraform/services.tf @@ -74,6 +74,11 @@ locals { TOKEN_KEY_MANAGER = "GOOGLE_CLOUD_KMS" TOKEN_SIGNING_KEY = trimprefix(data.google_kms_crypto_key_version.token-signer-version.id, "//cloudkms.googleapis.com/v1/") } + + e2e_runner_config = { + VERIFICATION_ADMIN_API = google_cloud_run_service.adminapi.status.0.url + VERIFICATION_SERVER_API = google_cloud_run_service.apiserver.status.0.url + } } output "cookie_keys" { From 56f1f36e74ccc4b504e835c6032b84dfa954f33d Mon Sep 17 00:00:00 2001 From: Yuchen Ying Date: Fri, 28 Aug 2020 20:02:33 -0700 Subject: [PATCH 2/5] small fix: remove unused code. --- pkg/config/e2e_runner_server_config.go | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/pkg/config/e2e_runner_server_config.go b/pkg/config/e2e_runner_server_config.go index 7d5745622..7139b3922 100644 --- a/pkg/config/e2e_runner_server_config.go +++ b/pkg/config/e2e_runner_server_config.go @@ -57,14 +57,3 @@ func NewE2ERunnerConfig(ctx context.Context) (*E2ERunnerConfig, error) { func (c *E2ERunnerConfig) Validate() error { return nil } - -func (c *E2ERunnerConfig) ToEnv() []string { - return []string{ - "VERIFICATION_ADMIN_API=" + c.VerificationAdminAPIServer, - "VERIFICATION_ADMIN_API_KEY=" + c.VerificationAdminAPIKey, - "VERIFICATION_SERVER_API=" + c.VerificationAPIServer, - "VERIFICATION_SERVER_API_KEY=" + c.VerificationAPIServerKey, - "KEY_SERVER=" + c.KeyServer, - "HEALTH_AUTHORITY_CODE=" + c.HealthAuthorityCode, - } -} From 490e022689c5a00ca14ee7476d3fb78490e3a4e8 Mon Sep 17 00:00:00 2001 From: Yuchen Ying Date: Sun, 30 Aug 2020 15:04:59 -0700 Subject: [PATCH 3/5] Replace Infof with Debugw per review suggestion. --- cmd/e2e-runner/e2e.go | 45 +++++++++++++------------- pkg/config/e2e_runner_server_config.go | 1 - 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/cmd/e2e-runner/e2e.go b/cmd/e2e-runner/e2e.go index 377e70165..354b9de39 100644 --- a/cmd/e2e-runner/e2e.go +++ b/cmd/e2e-runner/e2e.go @@ -80,10 +80,11 @@ func e2e(ctx context.Context, config config.E2ERunnerConfig) error { } else if code.Error != "" { return fmt.Errorf("issue API Error: %+v", code) } - if config.Verbose { - logger.Infof("Code Request: %+v", codeRequest) - logger.Infof("Code Response: %+v", code) - } + + logger.Debugw("Issue Code", + "request", codeRequest, + "response", code, + ) // Get the verification token logger.Infof("Verifying code and getting token") @@ -93,10 +94,10 @@ func e2e(ctx context.Context, config config.E2ERunnerConfig) error { } else if token.Error != "" { return fmt.Errorf("verification API Error %+v", token) } - if config.Verbose { - logger.Infof("Token Request: %+v", tokenRequest) - logger.Infof("Token Response: %+v", token) - } + logger.Debugw("getting token", + "request", tokenRequest, + "response", token, + ) logger.Infof("Check code status") statusReq, codeStatus, err := clients.CheckCodeStatus(ctx, config.VerificationAdminAPIServer, config.VerificationAdminAPIKey, code.UUID, timeout) @@ -105,10 +106,10 @@ func e2e(ctx context.Context, config config.E2ERunnerConfig) error { } else if codeStatus.Error != "" { return fmt.Errorf("check code status Error: %+v", codeStatus) } - if config.Verbose { - logger.Infof("Code Status Request: %+v", statusReq) - logger.Infof("Code Status Response: %+v", codeStatus) - } + logger.Debugw("check code status", + "request", statusReq, + "response", codeStatus, + ) if !codeStatus.Claimed { return fmt.Errorf("expected claimed OTP code for %s", statusReq.UUID) } @@ -127,10 +128,10 @@ func e2e(ctx context.Context, config config.E2ERunnerConfig) error { } else if certificate.Error != "" { return fmt.Errorf("certificate API Error: %+v", certificate) } - if config.Verbose { - logger.Infof("Certificate Request: %+v", certRequest) - logger.Infof("Certificate Response: %+v", certificate) - } + logger.Debugw("get certificate", + "request", certRequest, + "response", certificate, + ) // Upload the TEKs publish := verifyapi.Publish{ @@ -147,18 +148,18 @@ func e2e(ctx context.Context, config config.E2ERunnerConfig) error { client := &http.Client{ Timeout: timeout, } - if config.Verbose { - logger.Infof("Publish request: %+v", publish) - } + logger.Debugw("publish", + "request", publish, + ) if err := jsonclient.MakeRequest(ctx, client, config.KeyServer, http.Header{}, &publish, &response); err != nil { return fmt.Errorf("error publishing teks: %w", err) } else if response.ErrorMessage != "" { return fmt.Errorf("publish API error: %+v", response) } logger.Infof("Inserted %v exposures", response.InsertedExposures) - if config.Verbose { - logger.Infof("Publish response: %+v", response) - } + logger.Debugw("publish", + "response", response, + ) if config.DoRevise { testType = "confirmed" diff --git a/pkg/config/e2e_runner_server_config.go b/pkg/config/e2e_runner_server_config.go index 7139b3922..6baba9bcc 100644 --- a/pkg/config/e2e_runner_server_config.go +++ b/pkg/config/e2e_runner_server_config.go @@ -41,7 +41,6 @@ type E2ERunnerConfig struct { // e2e-test config DoRevise bool - Verbose bool } // NewE2ERunnerConfig returns the environment config for the e2e-runner server. From 04aaff3d2041694b6c725b7caa11a1b132d291e5 Mon Sep 17 00:00:00 2001 From: Yuchen Ying Date: Sun, 30 Aug 2020 15:30:09 -0700 Subject: [PATCH 4/5] Address additional comment from review. --- cloudbuild.yaml | 24 +++++++++++++++++++++++- cmd/e2e-runner/main.go | 5 +++-- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 5aad257be..1c1eab841 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -82,11 +82,32 @@ steps: 'push', 'gcr.io/${PROJECT_ID}/github.com/google/exposure-notifications-verification-server/cmd/adminapi:${SHORT_SHA}', ] +- id: 'e2e-runner' + name: 'docker' + timeout: 10m + waitFor: ["-"] + args: [ + 'build', + '--tag', + 'gcr.io/${PROJECT_ID}/github.com/google/exposure-notifications-verification-server/cmd/apiserver:${SHORT_SHA}', + '--build-arg', + 'SERVICE=e2e-runner', + '.', + ] +- id: 'e2e-runner:publish' + name: 'docker' + waitFor: + - e2e-runner + args: [ + 'push', + 'gcr.io/${PROJECT_ID}/github.com/google/exposure-notifications-verification-server/cmd/apiserver:${SHORT_SHA}', + ] - id: 'deploy' waitFor: - 'adminapi:publish' - 'apiserver:publish' - 'cleanup:publish' + - 'e2e-runner:publish' - 'server:publish' name: 'gcr.io/google.com/cloudsdktool/cloud-sdk:alpine' args: @@ -96,7 +117,7 @@ steps: - '-c' - |- gcloud components install --quiet beta; - SERVICES=(adminapi apiserver cleanup server); + SERVICES=(adminapi apiserver cleanup server e2e-runner); for s in "${SERVICES[@]}"; do gcloud beta run deploy "${s}" \ --quiet \ @@ -112,4 +133,5 @@ images: [ 'gcr.io/${PROJECT_ID}/github.com/google/exposure-notifications-verification-server/cmd/apiserver:${SHORT_SHA}', 'gcr.io/${PROJECT_ID}/github.com/google/exposure-notifications-verification-server/cmd/cleanup:${SHORT_SHA}', 'gcr.io/${PROJECT_ID}/github.com/google/exposure-notifications-verification-server/cmd/adminapi:${SHORT_SHA}', + 'gcr.io/${PROJECT_ID}/github.com/google/exposure-notifications-verification-server/cmd/e2e-runner:${SHORT_SHA}', ] diff --git a/cmd/e2e-runner/main.go b/cmd/e2e-runner/main.go index b2da4ce5f..1cb22f72e 100644 --- a/cmd/e2e-runner/main.go +++ b/cmd/e2e-runner/main.go @@ -26,6 +26,7 @@ import ( "github.com/google/exposure-notifications-server/pkg/logging" "github.com/google/exposure-notifications-server/pkg/server" + "github.com/google/exposure-notifications-verification-server/pkg/config" "github.com/google/exposure-notifications-verification-server/pkg/database" @@ -84,12 +85,12 @@ func realMain(ctx context.Context) error { realm, err := db.FindRealmByName(realmName) if err != nil { if !database.IsNotFound(err) { - return fmt.Errorf("error when finding the realm: %w", err) + return fmt.Errorf("error when finding the realm %q: %w", realmName, err) } realm = database.NewRealmWithDefaults(realmName) realm.RegionCode = realmRegionCode if err := db.SaveRealm(realm); err != nil { - return fmt.Errorf("failed to create realm: %w: %v", err, realm.ErrorMessages()) + return fmt.Errorf("failed to create realm %+v: %w: %v", realm, err, realm.ErrorMessages()) } } From 6bfe8170ddf76d4837c2ca4860bbbbd871daa7b4 Mon Sep 17 00:00:00 2001 From: Yuchen Ying Date: Sun, 30 Aug 2020 15:40:00 -0700 Subject: [PATCH 5/5] Add observability configs. --- cmd/e2e-runner/main.go | 13 +++++++++++++ pkg/config/e2e_runner_server_config.go | 4 +++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/cmd/e2e-runner/main.go b/cmd/e2e-runner/main.go index 1cb22f72e..1f72e1976 100644 --- a/cmd/e2e-runner/main.go +++ b/cmd/e2e-runner/main.go @@ -25,6 +25,7 @@ import ( "time" "github.com/google/exposure-notifications-server/pkg/logging" + "github.com/google/exposure-notifications-server/pkg/observability" "github.com/google/exposure-notifications-server/pkg/server" "github.com/google/exposure-notifications-verification-server/pkg/config" @@ -72,6 +73,18 @@ func realMain(ctx context.Context) error { return fmt.Errorf("failed to process e2e-runner config: %w", err) } + // Setup monitoring + logger.Info("configuring observability exporter") + oe, err := observability.NewFromEnv(ctx, e2eConfig.Observability) + if err != nil { + return fmt.Errorf("unable to create ObservabilityExporter provider: %w", err) + } + if err := oe.StartExporter(); err != nil { + return fmt.Errorf("error initializing observability exporter: %w", err) + } + defer oe.Close() + logger.Infow("observability exporter", "config", e2eConfig.Observability) + db, err := e2eConfig.Database.Load(ctx) if err != nil { return fmt.Errorf("failed to load database config: %w", err) diff --git a/pkg/config/e2e_runner_server_config.go b/pkg/config/e2e_runner_server_config.go index 6baba9bcc..1ffa85093 100644 --- a/pkg/config/e2e_runner_server_config.go +++ b/pkg/config/e2e_runner_server_config.go @@ -17,13 +17,15 @@ package config import ( "context" + "github.com/google/exposure-notifications-server/pkg/observability" "github.com/google/exposure-notifications-verification-server/pkg/database" "github.com/sethvargo/go-envconfig" ) // E2ERunnerConfig represents the environment based configuration for the e2e-runner server. type E2ERunnerConfig struct { - Database database.Config + Database database.Config + Observability *observability.Config // DevMode produces additional debugging information. Do not enable in // production environments.