Add and connect recovery middleware

cmd/appsync/main.go

@@ -110,6 +110,10 @@ func realMain(ctx context.Context) error {
 	populateLogger := middleware.PopulateLogger(logger)
 	r.Use(populateLogger)
 
+	// Recovery injection
+	recovery := middleware.Recovery(h)
+	r.Use(recovery)
+
 	appSyncController, err := appsync.New(cfg, db, h)
 	if err != nil {
 		return fmt.Errorf("failed to create cleanup controller: %w", err)

cmd/cleanup/main.go

@@ -122,6 +122,10 @@ func realMain(ctx context.Context) error {
 	populateLogger := middleware.PopulateLogger(logger)
 	r.Use(populateLogger)
 
+	// Recovery injection
+	recovery := middleware.Recovery(h)
+	r.Use(recovery)
+
 	cleanupController := cleanup.New(cfg, db, tokenSignerTyp, h)
 	r.Handle("/", cleanupController.HandleCleanup()).Methods(http.MethodGet)
 

cmd/e2e-runner/main.go

@@ -137,6 +137,10 @@ func realMain(ctx context.Context) error {
 	populateLogger := middleware.PopulateLogger(logger)
 	r.Use(populateLogger)
 
+	// Recovery injection
+	recovery := middleware.Recovery(h)
+	r.Use(recovery)
+
 	r.Handle("/default", handleDefault(cfg, h))
 	r.Handle("/revise", handleRevise(cfg, h))
 	r.Handle("/enx-redirect", handleENXRedirect(enxRedirectClient, h))

cmd/modeler/main.go

@@ -121,6 +121,10 @@ func realMain(ctx context.Context) error {
 	populateLogger := middleware.PopulateLogger(logger)
 	r.Use(populateLogger)
 
+	// Recovery injection
+	recovery := middleware.Recovery(h)
+	r.Use(recovery)
+
 	// Rate limiting
 	limiterStore, err := ratelimit.RateLimiterFor(ctx, &cfg.RateLimit)
 	if err != nil {

cmd/rotation/main.go

@@ -122,6 +122,10 @@ func realMain(ctx context.Context) error {
 	populateLogger := middleware.PopulateLogger(logger)
 	r.Use(populateLogger)
 
+	// Recovery injection
+	recovery := middleware.Recovery(h)
+	r.Use(recovery)
+
 	rotationController := rotation.New(cfg, db, tokenSignerTyp, h)
 	r.Handle("/", rotationController.HandleRotate()).Methods(http.MethodGet)
 	r.Handle("/realms", rotationController.HandleVerificationRotate()).Methods(http.MethodGet)

cmd/stats-puller/main.go

@@ -113,6 +113,10 @@ func realMain(ctx context.Context) error {
 	populateLogger := middleware.PopulateLogger(logger)
 	r.Use(populateLogger)
 
+	// Recovery injection
+	recovery := middleware.Recovery(h)
+	r.Use(recovery)
+
 	client, err := clients.NewKeyServerClient(cfg.KeyServerURL,
 		clients.WithTimeout(cfg.DownloadTimeout),
 		clients.WithMaxBodySize(cfg.FileSizeLimitBytes))

internal/routes/adminapi.go

@@ -62,10 +62,14 @@ func AdminAPI(
 	populateRequestID := middleware.PopulateRequestID(h)
 	r.Use(populateRequestID)
 
-	// Logger injection.
+	// Logger injection
 	populateLogger := middleware.PopulateLogger(logging.FromContext(ctx))
 	r.Use(populateLogger)
 
+	// Recovery injection
+	recovery := middleware.Recovery(h)
+	r.Use(recovery)
+
 	httplimiter, err := limitware.NewMiddleware(ctx, limiterStore,
 		limitware.APIKeyFunc(ctx, db, "adminapi:ratelimit:", cfg.RateLimit.HMACKey),
 		limitware.AllowOnError(false))

internal/routes/apiserver.go

@@ -66,10 +66,14 @@ func APIServer(
 	populateRequestID := middleware.PopulateRequestID(h)
 	r.Use(populateRequestID)
 
-	// Logger injection.
+	// Logger injection
 	populateLogger := middleware.PopulateLogger(logging.FromContext(ctx))
 	r.Use(populateLogger)
 
+	// Recovery injection
+	recovery := middleware.Recovery(h)
+	r.Use(recovery)
+
 	// Note that rate limiting is installed _after_ the chaff middleware because
 	// we do not want chaff requests to count towards rate-limiting quota.
 	httplimiter, err := limitware.NewMiddleware(ctx, limiterStore,

internal/routes/enx_redirect.go

@@ -61,6 +61,10 @@ func ENXRedirect(
 	populateLogger := middleware.PopulateLogger(logging.FromContext(ctx))
 	r.Use(populateLogger)
 
+	// Recovery injection
+	recovery := middleware.Recovery(h)
+	r.Use(recovery)
+
 	// Install common security headers
 	r.Use(middleware.SecureHeaders(cfg.DevMode, "html"))
 

internal/routes/server.go

@@ -103,10 +103,14 @@ func Server(
 	populateRequestID := middleware.PopulateRequestID(h)
 	r.Use(populateRequestID)
 
-	// Logger injection.
+	// Logger injection
 	populateLogger := middleware.PopulateLogger(logging.FromContext(ctx))
 	r.Use(populateLogger)
 
+	// Recovery injection
+	recovery := middleware.Recovery(h)
+	r.Use(recovery)
+
 	httplimiter, err := limitware.NewMiddleware(ctx, limiterStore,
 		limitware.UserIDKeyFunc(ctx, "server:ratelimit:", cfg.RateLimit.HMACKey),
 		limitware.AllowOnError(false))

pkg/controller/middleware/recovery.go

@@ -0,0 +1,50 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package middleware
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/google/exposure-notifications-server/pkg/logging"
+	"github.com/google/exposure-notifications-verification-server/pkg/controller"
+	"github.com/google/exposure-notifications-verification-server/pkg/render"
+
+	"github.com/gorilla/mux"
+)
+
+// Recovery recovers from panics and other fatal errors. It keeps the server and
+// service running, returning 500 to the caller while also logging the error in
+// a structured format.
+func Recovery(h *render.Renderer) mux.MiddlewareFunc {
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			ctx := r.Context()
+
+			logger := logging.FromContext(ctx).Named("middleware.Recover")
+
+			defer func() {
+				if p := recover(); p != nil {
+					logger.Errorw("http handler panic", "panic", p)
+					controller.InternalError(w, r, h, fmt.Errorf("panic: %v", p))
+					return
+				}
+			}()
+
+			next.ServeHTTP(w, r)
+			return
+		})
+	}
+}

pkg/controller/middleware/recovery_test.go

@@ -0,0 +1,78 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package middleware_test
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/google/exposure-notifications-verification-server/internal/envstest"
+	"github.com/google/exposure-notifications-verification-server/internal/project"
+	"github.com/google/exposure-notifications-verification-server/pkg/controller/middleware"
+	"github.com/google/exposure-notifications-verification-server/pkg/render"
+)
+
+func TestRecovery(t *testing.T) {
+	t.Parallel()
+
+	ctx := project.TestContext(t)
+	h, err := render.New(ctx, envstest.ServerAssetsPath(), true)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	m := middleware.Recovery(h)
+
+	cases := []struct {
+		name    string
+		handler http.Handler
+		code    int
+	}{
+		{
+			name:    "default",
+			handler: emptyHandler(),
+			code:    200,
+		},
+		{
+			name: "panic",
+			handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				panic("oops")
+			}),
+			code: 500,
+		},
+	}
+
+	for _, tc := range cases {
+		tc := tc
+
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			r, err := http.NewRequestWithContext(ctx, http.MethodGet, "/", nil)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			w := httptest.NewRecorder()
+
+			m(tc.handler).ServeHTTP(w, r)
+
+			if got, want := w.Code, tc.code; got != want {
+				t.Errorf("expected %d to be %d", got, want)
+			}
+		})
+	}
+}