diff --git a/pkg/controller/appsync/appsync.go b/pkg/controller/appsync/appsync.go index 04ed32aa5..deb5551bd 100644 --- a/pkg/controller/appsync/appsync.go +++ b/pkg/controller/appsync/appsync.go @@ -40,8 +40,11 @@ func (c *Controller) HandleSync() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { ctx := r.Context() + logger := logging.FromContext(ctx).Named("appsync.HandleSync") + ok, err := c.db.TryLock(ctx, appSyncLock, c.config.AppSyncMinPeriod) if err != nil { + logger.Errorw("failed to acquire lock", "error", err) c.h.RenderJSON(w, http.StatusInternalServerError, &AppSyncResult{ OK: false, Errors: []string{err.Error()}, @@ -49,6 +52,7 @@ func (c *Controller) HandleSync() http.Handler { return } if !ok { + logger.Debugw("skipping (too early)") c.h.RenderJSON(w, http.StatusOK, &AppSyncResult{ OK: false, Errors: []string{"too early"}, diff --git a/pkg/controller/cleanup/handle_cleanup.go b/pkg/controller/cleanup/handle_cleanup.go index 29fc8df24..b9e876487 100644 --- a/pkg/controller/cleanup/handle_cleanup.go +++ b/pkg/controller/cleanup/handle_cleanup.go @@ -23,7 +23,6 @@ import ( "github.com/google/exposure-notifications-verification-server/internal/project" "github.com/google/exposure-notifications-verification-server/pkg/observability" "github.com/hashicorp/go-multierror" - "go.opencensus.io/stats" "go.opencensus.io/tag" ) @@ -42,7 +41,7 @@ func (c *Controller) HandleCleanup() http.Handler { ok, err := c.db.TryLock(ctx, cleanupName, c.config.CleanupMinPeriod) if err != nil { - logger.Errorw("failed to run shouldCleanup", "error", err) + logger.Errorw("failed to acquire lock", "error", err) c.h.RenderJSON(w, http.StatusInternalServerError, &CleanupResult{ OK: false, Errors: []string{err.Error()}, @@ -50,14 +49,13 @@ func (c *Controller) HandleCleanup() http.Handler { return } if !ok { - stats.RecordWithTags(ctx, []tag.Mutator{observability.ResultNotOK()}, mClaimRequests.M(1)) + logger.Debugw("skipping (too early)") c.h.RenderJSON(w, http.StatusOK, &CleanupResult{ OK: false, Errors: []string{"too early"}, }) return } - stats.RecordWithTags(ctx, []tag.Mutator{observability.ResultOK()}, mClaimRequests.M(1)) // Construct a multi-error. If one of the purges fails, we still want to // attempt the other purges. diff --git a/pkg/controller/rotation/handle_token_rotate.go b/pkg/controller/rotation/handle_token_rotate.go index e6c447297..99f12657e 100644 --- a/pkg/controller/rotation/handle_token_rotate.go +++ b/pkg/controller/rotation/handle_token_rotate.go @@ -24,7 +24,6 @@ import ( "github.com/google/exposure-notifications-verification-server/pkg/database" "github.com/google/exposure-notifications-verification-server/pkg/observability" "github.com/hashicorp/go-multierror" - "go.opencensus.io/stats" "go.opencensus.io/tag" ) @@ -44,7 +43,7 @@ func (c *Controller) HandleRotate() http.Handler { ok, err := c.db.TryLock(ctx, tokenRotationLock, c.config.MinTTL) if err != nil { - logger.Errorw("failed to run shouldRotate", "error", err) + logger.Errorw("failed to acquire lock", "error", err) c.h.RenderJSON(w, http.StatusInternalServerError, &Result{ OK: false, Errors: []string{err.Error()}, @@ -52,14 +51,13 @@ func (c *Controller) HandleRotate() http.Handler { return } if !ok { - stats.RecordWithTags(ctx, []tag.Mutator{observability.ResultNotOK()}, mClaimRequests.M(1)) + logger.Debugw("skipping (too early)") c.h.RenderJSON(w, http.StatusOK, &Result{ OK: false, Errors: []string{"too early"}, }) return } - stats.RecordWithTags(ctx, []tag.Mutator{observability.ResultOK()}, mClaimRequests.M(1)) // Token signing keys func() { diff --git a/pkg/controller/rotation/handle_verification_rotate.go b/pkg/controller/rotation/handle_verification_rotate.go index e4da63691..f2367cf56 100644 --- a/pkg/controller/rotation/handle_verification_rotate.go +++ b/pkg/controller/rotation/handle_verification_rotate.go @@ -22,14 +22,11 @@ import ( "github.com/google/exposure-notifications-verification-server/internal/project" "github.com/google/exposure-notifications-verification-server/pkg/database" - "github.com/google/exposure-notifications-verification-server/pkg/observability" "github.com/google/exposure-notifications-verification-server/pkg/pagination" "github.com/google/exposure-notifications-server/pkg/logging" "github.com/hashicorp/go-multierror" - "go.opencensus.io/stats" - "go.opencensus.io/tag" ) // HandleVerificationRotate handles verification certificate key rotation. @@ -47,7 +44,7 @@ func (c *Controller) HandleVerificationRotate() http.Handler { ok, err := c.db.TryLock(ctx, verificationRotationLock, c.config.MinTTL) if err != nil { - logger.Errorw("failed to obtain lock", "lock", verificationRotationLock, "error", err) + logger.Errorw("failed to acquire lock", "error", err) c.h.RenderJSON(w, http.StatusInternalServerError, &Result{ OK: false, Errors: []string{err.Error()}, @@ -55,14 +52,13 @@ func (c *Controller) HandleVerificationRotate() http.Handler { return } if !ok { - stats.RecordWithTags(ctx, []tag.Mutator{observability.ResultNotOK()}, mClaimRequests.M(1)) + logger.Debugw("skipping (too early)") c.h.RenderJSON(w, http.StatusOK, &Result{ OK: false, Errors: []string{"too early"}, }) return } - stats.RecordWithTags(ctx, []tag.Mutator{observability.ResultOK()}, mClaimRequests.M(1)) var merr *multierror.Error diff --git a/pkg/controller/statspuller/handle_pull.go b/pkg/controller/statspuller/handle_pull.go index 8a36bd041..7b11f3fc0 100644 --- a/pkg/controller/statspuller/handle_pull.go +++ b/pkg/controller/statspuller/handle_pull.go @@ -42,8 +42,11 @@ func (c *Controller) HandlePullStats() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { ctx := r.Context() + logger := logging.FromContext(ctx).Named("statspuller.HandlePullStats") + ok, err := c.db.TryLock(ctx, statsPullerLock, c.config.StatsPullerMinPeriod) if err != nil { + logger.Errorw("failed to acquite lock", "error", err) c.h.RenderJSON(w, http.StatusInternalServerError, &Result{ OK: false, Errors: []string{err.Error()}, @@ -51,6 +54,7 @@ func (c *Controller) HandlePullStats() http.Handler { return } if !ok { + logger.Debugw("skipping (too early)") c.h.RenderJSON(w, http.StatusOK, &Result{ OK: false, Errors: []string{"too early"}, @@ -65,7 +69,6 @@ func (c *Controller) HandlePullStats() http.Handler { return } - logger := logging.FromContext(ctx).Named("rotation.HandlePullStats") for _, realmStat := range statsConfigs { realmID := realmStat.RealmID