Metrics

cmd/adminapi/main.go

@@ -134,7 +134,10 @@ func realMain(ctx context.Context) error {
 	r.Use(requireAPIKey)
 	r.Use(rateLimit)
 
-	issueapiController := issueapi.New(ctx, config, db, h)
+	issueapiController, err := issueapi.New(ctx, config, db, h)
+	if err != nil {
+		return fmt.Errorf("issueapi.New: %w", err)
+	}
 	r.Handle("/api/issue", issueapiController.HandleIssue()).Methods("POST")
 
 	codeStatusController := codestatus.NewAPI(ctx, config, db, h)

cmd/apiserver/main.go

@@ -153,7 +153,10 @@ func realMain(ctx context.Context) error {
 	// POST /api/verify
 	verifyChaff := chaff.New()
 	defer verifyChaff.Close()
-	verifyapiController := verifyapi.New(ctx, config, db, h, tokenSigner)
+	verifyapiController, err := verifyapi.New(ctx, config, db, h, tokenSigner)
+	if err != nil {
+		return fmt.Errorf("failed to create verify api controller: %w", err)
+	}
 	r.Handle("/api/verify", handleChaff(verifyChaff, verifyapiController.HandleVerify())).Methods("POST")
 
 	// POST /api/certificate

cmd/server/main.go

@@ -225,7 +225,10 @@ func realMain(ctx context.Context) error {
 		sub.Handle("", homeController.HandleHome()).Methods("GET")
 
 		// API for creating new verification codes. Called via AJAX.
-		issueapiController := issueapi.New(ctx, config, db, h)
+		issueapiController, err := issueapi.New(ctx, config, db, h)
+		if err != nil {
+			return fmt.Errorf("issueapi.New: %w", err)
+		}
 		sub.Handle("/issue", issueapiController.HandleIssue()).Methods("POST")
 	}
 

pkg/controller/certapi/certapi.go

@@ -24,6 +24,7 @@ import (
 	"github.com/google/exposure-notifications-verification-server/pkg/database"
 	"github.com/google/exposure-notifications-verification-server/pkg/keyutils"
 	"github.com/google/exposure-notifications-verification-server/pkg/render"
+	"go.opencensus.io/stats"
 
 	verifyapi "github.com/google/exposure-notifications-server/pkg/api/v1"
 	"github.com/google/exposure-notifications-server/pkg/cache"
@@ -42,6 +43,8 @@ type Controller struct {
 	pubKeyCache *keyutils.PublicKeyCache // Cache of public keys for verification token verification.
 	signerCache *cache.Cache             // Cache signers on a per-realm basis.
 	kms         keys.KeyManager
+
+	metrics *Metrics
 }
 
 func New(ctx context.Context, config *config.APIServerConfig, db *database.Database, h *render.Renderer, kms keys.KeyManager) (*Controller, error) {
@@ -57,6 +60,11 @@ func New(ctx context.Context, config *config.APIServerConfig, db *database.Datab
 		return nil, fmt.Errorf("cannot create signer cache, likely invalid duration: %w", err)
 	}
 
+	metrics, err := registerMetrics()
+	if err != nil {
+		return nil, err
+	}
+
 	return &Controller{
 		config:      config,
 		db:          db,
@@ -65,12 +73,13 @@ func New(ctx context.Context, config *config.APIServerConfig, db *database.Datab
 		pubKeyCache: pubKeyCache,
 		signerCache: signerCache,
 		kms:         kms,
+		metrics:     metrics,
 	}, nil
 }
 
 // Parses and validates the token against the configured keyID and public key.
 // If the token si valid the token id (`tid') and subject (`sub`) claims are returned.
-func (c *Controller) validateToken(verToken string, publicKey crypto.PublicKey) (string, *database.Subject, error) {
+func (c *Controller) validateToken(ctx context.Context, verToken string, publicKey crypto.PublicKey) (string, *database.Subject, error) {
 	// Parse and validate the verification token.
 	token, err := jwt.ParseWithClaims(verToken, &jwt.StandardClaims{}, func(token *jwt.Token) (interface{}, error) {
 		kidHeader := token.Header[verifyapi.KeyIDHeader]
@@ -84,24 +93,29 @@ func (c *Controller) validateToken(verToken string, publicKey crypto.PublicKey)
 		return nil, fmt.Errorf("no public key for specified 'kid' not found: %v", kid)
 	})
 	if err != nil {
+		stats.Record(ctx, c.metrics.TokenInvalid.M(1), c.metrics.CertificateErrors.M(1))
 		c.logger.Errorf("invalid verification token: %v", err)
 		return "", nil, fmt.Errorf("invalid verification token")
 	}
 	tokenClaims, ok := token.Claims.(*jwt.StandardClaims)
 	if !ok {
+		stats.Record(ctx, c.metrics.TokenInvalid.M(1), c.metrics.CertificateErrors.M(1))
 		c.logger.Errorf("invalid claims in verification token")
 		return "", nil, fmt.Errorf("invalid verification token")
 	}
 	if err := tokenClaims.Valid(); err != nil {
+		stats.Record(ctx, c.metrics.TokenInvalid.M(1), c.metrics.CertificateErrors.M(1))
 		c.logger.Errorf("JWT is invalid: %v", err)
 		return "", nil, fmt.Errorf("verification token expired")
 	}
 	if !tokenClaims.VerifyIssuer(c.config.TokenSigning.TokenIssuer, true) || !tokenClaims.VerifyAudience(c.config.TokenSigning.TokenIssuer, true) {
+		stats.Record(ctx, c.metrics.TokenInvalid.M(1), c.metrics.CertificateErrors.M(1))
 		c.logger.Errorf("jwt contains invalid iss/aud: iss %v aud: %v", tokenClaims.Issuer, tokenClaims.Audience)
 		return "", nil, fmt.Errorf("verification token not valid")
 	}
 	subject, err := database.ParseSubject(tokenClaims.Subject)
 	if err != nil {
+		stats.Record(ctx, c.metrics.TokenInvalid.M(1), c.metrics.CertificateErrors.M(1))
 		return "", nil, fmt.Errorf("invalid subject: %w", err)
 	}
 	return tokenClaims.Id, subject, nil

pkg/controller/certapi/certificate.go

@@ -25,6 +25,9 @@ import (
 	"github.com/google/exposure-notifications-verification-server/pkg/controller"
 	"github.com/google/exposure-notifications-verification-server/pkg/database"
 	"github.com/google/exposure-notifications-verification-server/pkg/jwthelper"
+	"github.com/google/exposure-notifications-verification-server/pkg/observability"
+	"go.opencensus.io/stats"
+	"go.opencensus.io/tag"
 
 	verifyapi "github.com/google/exposure-notifications-server/pkg/api/v1"
 )
@@ -35,41 +38,60 @@ func (c *Controller) HandleCertificate() http.Handler {
 
 		authApp := controller.AuthorizedAppFromContext(ctx)
 		if authApp == nil {
+			stats.Record(ctx, c.metrics.CertificateErrors.M(1))
 			c.logger.Errorf("missing authorized app")
 			controller.MissingAuthorizedApp(w, r, c.h)
 			return
 		}
 
+		// This is a non terminal error, as we're only using the realm for stats.
+		realm, err := authApp.Realm(c.db)
+		if err != nil {
+			c.logger.Errorf("unable to load realm", "error", err)
+		} else {
+			ctx, err = tag.New(ctx,
+				tag.Upsert(observability.RealmTagKey, realm.Name))
+			if err != nil {
+				c.logger.Errorw("unable to record metrics for realm", "realmID", realm.ID, "error", err)
+			}
+		}
+		stats.Record(ctx, c.metrics.Attempts.M(1))
+
 		// Get the public key for the token.
 		publicKey, err := c.pubKeyCache.GetPublicKey(ctx, c.config.TokenSigning.TokenSigningKey, c.kms)
 		if err != nil {
 			c.logger.Errorw("failed to get public key", "error", err)
+			stats.Record(ctx, c.metrics.CertificateErrors.M(1))
 			c.h.RenderJSON(w, http.StatusInternalServerError, api.InternalError())
 			return
 		}
 
 		var request api.VerificationCertificateRequest
 		if err := controller.BindJSON(w, r, &request); err != nil {
 			c.logger.Errorf("failed to parse json request", "error", err)
+			stats.Record(ctx, c.metrics.CertificateErrors.M(1))
 			c.h.RenderJSON(w, http.StatusBadRequest, api.Error(err).WithCode(api.ErrTokenInvalid))
 			return
 		}
 
 		// Parse and validate the verification token.
-		tokenID, subject, err := c.validateToken(request.VerificationToken, publicKey)
+		tokenID, subject, err := c.validateToken(ctx, request.VerificationToken, publicKey)
 		if err != nil {
+			stats.Record(ctx, c.metrics.CertificateErrors.M(1))
 			c.h.RenderJSON(w, http.StatusBadRequest, api.Error(err).WithCode(api.ErrTokenInvalid))
 			return
 		}
 
 		// Validate the HMAC length. SHA 256 HMAC must be 32 bytes in length.
 		hmacBytes, err := base64util.DecodeString(request.ExposureKeyHMAC)
 		if err != nil {
+			stats.Record(ctx, c.metrics.CertificateErrors.M(1))
 			c.h.RenderJSON(w, http.StatusBadRequest,
 				api.Errorf("exposure key HMAC is not a valid base64: %v", err).WithCode(api.ErrHMACInvalid))
 			return
 		}
 		if len(hmacBytes) != 32 {
+			stats.Record(ctx, c.metrics.CertificateErrors.M(1))
 			c.h.RenderJSON(w, http.StatusBadRequest,
 				api.Errorf("exposure key HMAC is not the correct length, want: 32 got: %v", len(hmacBytes)).WithCode(api.ErrHMACInvalid))
 			return
@@ -78,6 +100,7 @@ func (c *Controller) HandleCertificate() http.Handler {
 		// determine the correct signing key to use.
 		signerInfo, err := c.getSignerForRealm(ctx, authApp)
 		if err != nil {
+			stats.Record(ctx, c.metrics.CertificateErrors.M(1))
 			c.logger.Errorw("failed to get signer", "error", err)
 			c.h.RenderJSON(w, http.StatusInternalServerError, api.InternalError())
 			return
@@ -103,6 +126,7 @@ func (c *Controller) HandleCertificate() http.Handler {
 		certToken.Header[verifyapi.KeyIDHeader] = signerInfo.KeyID
 		certificate, err := jwthelper.SignJWT(certToken, signerInfo.Signer)
 		if err != nil {
+			stats.Record(ctx, c.metrics.CertificateErrors.M(1))
 			c.logger.Errorw("failed to sign certificate", "error", err)
 			c.h.RenderJSON(w, http.StatusBadRequest, api.Error(err).WithCode(api.ErrInternal))
 			return
@@ -114,17 +138,22 @@ func (c *Controller) HandleCertificate() http.Handler {
 			c.logger.Errorw("failed to claim token", "tokenID", tokenID, "error", err)
 			switch {
 			case errors.Is(err, database.ErrTokenExpired):
+				stats.Record(ctx, c.metrics.TokenExpired.M(1), c.metrics.CertificateErrors.M(1))
 				c.h.RenderJSON(w, http.StatusBadRequest, api.Error(err).WithCode(api.ErrTokenExpired))
 			case errors.Is(err, database.ErrTokenUsed):
+				stats.Record(ctx, c.metrics.TokenUsed.M(1), c.metrics.CertificateErrors.M(1))
 				c.h.RenderJSON(w, http.StatusBadRequest, api.Errorf("verification token invalid").WithCode(api.ErrTokenExpired))
 			case errors.Is(err, database.ErrTokenMetadataMismatch):
+				stats.Record(ctx, c.metrics.TokenInvalid.M(1), c.metrics.CertificateErrors.M(1))
 				c.h.RenderJSON(w, http.StatusBadRequest, api.Errorf("verification token invalid").WithCode(api.ErrTokenExpired))
 			default:
+				stats.Record(ctx, c.metrics.TokenInvalid.M(1), c.metrics.CertificateErrors.M(1))
 				c.h.RenderJSON(w, http.StatusBadRequest, api.Error(err))
 			}
 			return
 		}
 
+		stats.Record(ctx, c.metrics.CertificateIssued.M(1))
 		c.h.RenderJSON(w, http.StatusOK, &api.VerificationCertificateResponse{
 			Certificate: certificate,
 		})

pkg/controller/certapi/metrics.go

@@ -0,0 +1,110 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package certapi
+
+import (
+	"fmt"
+
+	"github.com/google/exposure-notifications-verification-server/pkg/observability"
+
+	"go.opencensus.io/stats"
+	"go.opencensus.io/stats/view"
+	"go.opencensus.io/tag"
+)
+
+var (
+	MetricPrefix = observability.MetricRoot + "/api/certificate"
+)
+
+type Metrics struct {
+	Attempts          *stats.Int64Measure
+	TokenExpired      *stats.Int64Measure
+	TokenUsed         *stats.Int64Measure
+	TokenInvalid      *stats.Int64Measure
+	CertificateIssued *stats.Int64Measure
+	CertificateErrors *stats.Int64Measure
+}
+
+func registerMetrics() (*Metrics, error) {
+	mAttempts := stats.Int64(MetricPrefix+"/attempts", "certificate issue attempts", stats.UnitDimensionless)
+	if err := view.Register(&view.View{
+		Name:        MetricPrefix + "/attempt_count",
+		Measure:     mAttempts,
+		Description: "The count of certificate issue attempts",
+		TagKeys:     []tag.Key{observability.RealmTagKey},
+		Aggregation: view.Count(),
+	}); err != nil {
+		return nil, fmt.Errorf("stat view registration failure: %w", err)
+	}
+	mTokenExpired := stats.Int64(MetricPrefix+"/token_expired", "expired tokens on certificate issue", stats.UnitDimensionless)
+	if err := view.Register(&view.View{
+		Name:        MetricPrefix + "/token_expired_count",
+		Measure:     mTokenExpired,
+		Description: "The count of expired tokens on certificate issue",
+		TagKeys:     []tag.Key{observability.RealmTagKey},
+		Aggregation: view.Count(),
+	}); err != nil {
+		return nil, fmt.Errorf("stat view registration failure: %w", err)
+	}
+	mTokenUsed := stats.Int64(MetricPrefix+"/token_used", "already used tokens on certificate issue", stats.UnitDimensionless)
+	if err := view.Register(&view.View{
+		Name:        MetricPrefix + "/token_used_count",
+		Measure:     mTokenUsed,
+		Description: "The count of already used tokens on certificate issue",
+		TagKeys:     []tag.Key{observability.RealmTagKey},
+		Aggregation: view.Count(),
+	}); err != nil {
+		return nil, fmt.Errorf("stat view registration failure: %w", err)
+	}
+	mTokenInvalid := stats.Int64(MetricPrefix+"/invalid_token", "invalid tokens on certificate issue", stats.UnitDimensionless)
+	if err := view.Register(&view.View{
+		Name:        MetricPrefix + "/invalid_token_count",
+		Measure:     mTokenInvalid,
+		Description: "The count of invalid tokens on certificate issue",
+		TagKeys:     []tag.Key{observability.RealmTagKey},
+		Aggregation: view.Count(),
+	}); err != nil {
+		return nil, fmt.Errorf("stat view registration failure: %w", err)
+	}
+	mCertificateIssued := stats.Int64(MetricPrefix+"/issue", "certificates issued", stats.UnitDimensionless)
+	if err := view.Register(&view.View{
+		Name:        MetricPrefix + "/issue_count",
+		Measure:     mCertificateIssued,
+		Description: "The count of certificates issued",
+		TagKeys:     []tag.Key{observability.RealmTagKey},
+		Aggregation: view.Count(),
+	}); err != nil {
+		return nil, fmt.Errorf("stat view registration failure: %w", err)
+	}
+	mCertificateErrors := stats.Int64(MetricPrefix+"/errors", "certificate issue errors", stats.UnitDimensionless)
+	if err := view.Register(&view.View{
+		Name:        MetricPrefix + "/error_count",
+		Measure:     mCertificateErrors,
+		Description: "The count of certificate issue errors",
+		TagKeys:     []tag.Key{observability.RealmTagKey},
+		Aggregation: view.Count(),
+	}); err != nil {
+		return nil, fmt.Errorf("stat view registration failure: %w", err)
+	}
+
+	return &Metrics{
+		Attempts:          mAttempts,
+		TokenExpired:      mTokenExpired,
+		TokenUsed:         mTokenUsed,
+		TokenInvalid:      mTokenInvalid,
+		CertificateIssued: mCertificateIssued,
+		CertificateErrors: mCertificateErrors,
+	}, nil
+}