Add statistics endpoints to adminapi

cmd/server/assets/realmadmin/_stats_codes.html

@@ -12,8 +12,8 @@
     <a href="#" data-toggle="modal" data-target="#realm-modal">Learn more about this chart</a>
     <span>
       <span class="mr-1">Export as:</span>
-      <a href="/realm/stats.csv" class="mr-1">CSV</a>
-      <a href="/realm/stats.json">JSON</a>
+      <a href="/stats/realm.csv" class="mr-1">CSV</a>
+      <a href="/stats/realm.json">JSON</a>
     </span>
   </small>
 </div>
@@ -59,7 +59,7 @@ <h5 class="modal-title">Codes issued &amp; claimed</h5>
 
   function drawRealmCharts() {
     $.ajax({
-      url: '/realm/stats.json',
+      url: '/stats/realm.json',
       dataType: 'json',
     })
     .done(function(data, status, xhr) {

cmd/server/assets/realmadmin/_stats_daily_active_users.html

@@ -12,8 +12,8 @@
     <a href="#" data-toggle="modal" data-target="#daily-active-users-modal">Learn more about this chart</a>
     <span>
       <span class="mr-1">Export as:</span>
-      <a href="/realm/stats.csv" class="mr-1">CSV</a>
-      <a href="/realm/stats.json">JSON</a>
+      <a href="/stats/realm.csv" class="mr-1">CSV</a>
+      <a href="/stats/realm.json">JSON</a>
     </span>
   </small>
 </div>
@@ -55,7 +55,7 @@ <h5 class="modal-title">Daily active users</h5>
 
   function drawDailyActiveUsersChart() {
     $.ajax({
-      url: '/realm/stats.json',
+      url: '/stats/realm.json',
       dataType: 'json',
     })
     .done(function(data, status, xhr) {

cmd/server/assets/realmadmin/_stats_external_issuers.html

@@ -14,8 +14,8 @@
     <a href="#" data-toggle="modal" data-target="#per-external-issuer-table-modal">Learn more about this table</a>
     <span>
       <span class="mr-1">Export as:</span>
-      <a href="/realm/stats.csv?scope=external" class="mr-1">CSV</a>
-      <a href="/realm/stats.json?scope=external">JSON</a>
+      <a href="/stats/realm-external-issuer.csv" class="mr-1">CSV</a>
+      <a href="/stats/realm-external-issuer.json">JSON</a>
     </span>
   </small>
 </div>
@@ -68,7 +68,7 @@ <h5 class="modal-title">Codes issued by external issuers by day</h5>
 
   function drawExternalIssuersTable() {
     $.ajax({
-      url: '/realm/stats.json',
+      url: '/stats/realm-external-issuer.json',
       data: { scope: 'external' },
       dataType: 'json',
     })

cmd/server/assets/realmadmin/_stats_users.html

@@ -14,8 +14,8 @@
     <a href="#" data-toggle="modal" data-target="#per-user-table-modal">Learn more about this table</a>
     <span>
       <span class="mr-1">Export as:</span>
-      <a href="/realm/stats.csv?scope=user" class="mr-1">CSV</a>
-      <a href="/realm/stats.json?scope=user">JSON</a>
+      <a href="/stats/realm-user.csv" class="mr-1">CSV</a>
+      <a href="/stats/realm-user.json">JSON</a>
     </span>
   </small>
 </div>
@@ -60,7 +60,7 @@ <h5 class="modal-title">Codes issued by user by day</h5>
 
   function drawUsersTable() {
     $.ajax({
-      url: '/realm/stats.json',
+      url: '/stats/realm-user.json',
       data: { scope: 'user' },
       dataType: 'json',
     })

docs/api.md

@@ -448,6 +448,27 @@ Expires an unclaimed code. If the code has been claimed an error is returned.
 The timestamps are updated to the new expiration time (which will be in the
 past).
 
+
+## `/api/stats/*` (preview)
+
+**The statistics API are currently in preview. They are not covered by our
+backwards-compatibility promise and the APIs are subject to change without
+notice!**
+
+This path includes realm-level statistics for the past 30 days.
+
+-   `/api/stats/realm.{csv,json}` - Daily statistics for the realm, including
+    codes issued, codes claimed, and daily active users (if enabled).
+
+-   `/api/stats/realm-user.{csv,json}` - Daily statistics for codes issued by
+    realm user. These statistics only include codes issued by humans logged into
+    the verification system.
+
+-   `/api/stats/realm-external-issuser.{csv,json}` - Daily statistics for codes
+    issued by external issuers. These statistics only include codes issued by
+    the API where an `externalIssuer` field was provided.
+
+
 # Chaffing requests
 
 In addition to "real" requests, the server also accepts chaff (fake) requests.

internal/routes/adminapi.go

@@ -26,6 +26,7 @@ import (
 	"github.com/google/exposure-notifications-verification-server/pkg/controller/codes"
 	"github.com/google/exposure-notifications-verification-server/pkg/controller/issueapi"
 	"github.com/google/exposure-notifications-verification-server/pkg/controller/middleware"
+	"github.com/google/exposure-notifications-verification-server/pkg/controller/stats"
 	"github.com/google/exposure-notifications-verification-server/pkg/database"
 	"github.com/google/exposure-notifications-verification-server/pkg/ratelimit/limitware"
 	"github.com/google/exposure-notifications-verification-server/pkg/render"
@@ -104,6 +105,18 @@ func AdminAPI(
 		codesController := codes.NewAPI(ctx, cfg, db, h)
 		sub.Handle("/checkcodestatus", codesController.HandleCheckCodeStatus()).Methods("POST")
 		sub.Handle("/expirecode", codesController.HandleExpireAPI()).Methods("POST")
+
+		{
+			sub := sub.PathPrefix("/stats").Subrouter()
+
+			statsController := stats.New(ctx, cacher, db, h)
+			sub.Handle("/realm.csv", statsController.HandleRealmStats(stats.StatsTypeCSV)).Methods("GET")
+			sub.Handle("/realm.json", statsController.HandleRealmStats(stats.StatsTypeJSON)).Methods("GET")
+			sub.Handle("/realm-user.csv", statsController.HandleRealmUserStats(stats.StatsTypeCSV)).Methods("GET")
+			sub.Handle("/realm-user.json", statsController.HandleRealmUserStats(stats.StatsTypeJSON)).Methods("GET")
+			sub.Handle("/realm-external-issuer.csv", statsController.HandleRealmExternalIssuerStats(stats.StatsTypeCSV)).Methods("GET")
+			sub.Handle("/realm-external-issuer.json", statsController.HandleRealmExternalIssuerStats(stats.StatsTypeJSON)).Methods("GET")
+		}
 	}
 
 	// Wrap the main router in the mutating middleware method. This cannot be

internal/routes/server.go

@@ -35,6 +35,7 @@ import (
 	"github.com/google/exposure-notifications-verification-server/pkg/controller/mobileapps"
 	"github.com/google/exposure-notifications-verification-server/pkg/controller/realmadmin"
 	"github.com/google/exposure-notifications-verification-server/pkg/controller/realmkeys"
+	"github.com/google/exposure-notifications-verification-server/pkg/controller/stats"
 	"github.com/google/exposure-notifications-verification-server/pkg/controller/user"
 	"github.com/google/exposure-notifications-verification-server/pkg/database"
 	"github.com/google/exposure-notifications-verification-server/pkg/ratelimit/limitware"
@@ -261,6 +262,21 @@ func Server(
 		userRoutes(sub, userController)
 	}
 
+	// stats
+	{
+		sub := r.PathPrefix("/stats").Subrouter()
+		sub.Use(requireAuth)
+		sub.Use(loadCurrentMembership)
+		sub.Use(requireMembership)
+		sub.Use(processFirewall)
+		sub.Use(requireVerified)
+		sub.Use(requireMFA)
+		sub.Use(rateLimit)
+
+		statsController := stats.New(ctx, cacher, db, h)
+		statsRoutes(sub, statsController)
+	}
+
 	// realms
 	{
 		sub := r.PathPrefix("/realm").Subrouter()
@@ -376,14 +392,22 @@ func realmkeysRoutes(r *mux.Router, c *realmkeys.Controller) {
 	r.Handle("/keys/activate", c.HandleActivate()).Methods("POST")
 }
 
+// statsRoutes are the statistics routes, rooted at /stats.
+func statsRoutes(r *mux.Router, c *stats.Controller) {
+	r.Handle("/realm.csv", c.HandleRealmStats(stats.StatsTypeCSV)).Methods("GET")
+	r.Handle("/realm.json", c.HandleRealmStats(stats.StatsTypeJSON)).Methods("GET")
+	r.Handle("/realm-user.csv", c.HandleRealmUserStats(stats.StatsTypeCSV)).Methods("GET")
+	r.Handle("/realm-user.json", c.HandleRealmUserStats(stats.StatsTypeJSON)).Methods("GET")
+	r.Handle("/realm-external-issuer.csv", c.HandleRealmExternalIssuerStats(stats.StatsTypeCSV)).Methods("GET")
+	r.Handle("/realm-external-issuer.json", c.HandleRealmExternalIssuerStats(stats.StatsTypeJSON)).Methods("GET")
+}
+
 // realmadminRoutes are the realm admin routes.
 func realmadminRoutes(r *mux.Router, c *realmadmin.Controller) {
 	r.Handle("/settings", c.HandleSettings()).Methods("GET", "POST")
 	r.Handle("/settings/enable-express", c.HandleEnableExpress()).Methods("POST")
 	r.Handle("/settings/disable-express", c.HandleDisableExpress()).Methods("POST")
 	r.Handle("/stats", c.HandleStats()).Methods("GET")
-	r.Handle("/stats.csv", c.HandleStats()).Methods("GET")
-	r.Handle("/stats.json", c.HandleStats()).Methods("GET")
 	r.Handle("/events", c.HandleEvents()).Methods("GET")
 }
 

internal/routes/server_test.go

@@ -215,6 +215,41 @@ func TestRoutes_realmkeysRoutes(t *testing.T) {
 	}
 }
 
+func TestRoutes_statsRoutes(t *testing.T) {
+	t.Parallel()
+
+	m := mux.NewRouter()
+	statsRoutes(m, nil)
+
+	cases := []struct {
+		req  *http.Request
+		vars map[string]string
+	}{
+		{
+			req: httptest.NewRequest("GET", "/realm.csv", nil),
+		},
+		{
+			req: httptest.NewRequest("GET", "/realm.json", nil),
+		},
+		{
+			req: httptest.NewRequest("GET", "/realm-user.csv", nil),
+		},
+		{
+			req: httptest.NewRequest("GET", "/realm-user.json", nil),
+		},
+		{
+			req: httptest.NewRequest("GET", "/realm-external-issuer.csv", nil),
+		},
+		{
+			req: httptest.NewRequest("GET", "/realm-external-issuer.json", nil),
+		},
+	}
+
+	for _, tc := range cases {
+		testRoute(t, m, tc.req, tc.vars)
+	}
+}
+
 func TestRoutes_realmadminRoutes(t *testing.T) {
 	t.Parallel()
 
@@ -240,12 +275,6 @@ func TestRoutes_realmadminRoutes(t *testing.T) {
 		{
 			req: httptest.NewRequest("GET", "/stats", nil),
 		},
-		{
-			req: httptest.NewRequest("GET", "/stats.json", nil),
-		},
-		{
-			req: httptest.NewRequest("GET", "/stats.csv", nil),
-		},
 		{
 			req: httptest.NewRequest("GET", "/events", nil),
 		},

pkg/controller/middleware/membership.go

@@ -81,7 +81,7 @@ func LoadCurrentMembership(cacher cache.Cacher, db *database.Database, h render.
 				return
 			}
 
-			// Save the membership on the context.
+			// Save the membership and realm on the context.
 			ctx = controller.WithMembership(ctx, membership)
 			r = r.Clone(ctx)
 

pkg/controller/realmadmin/stats.go

@@ -15,24 +15,12 @@
 package realmadmin
 
 import (
-	"context"
-	"encoding/json"
-	"fmt"
 	"net/http"
-	"strconv"
-	"strings"
-	"time"
 
-	"github.com/google/exposure-notifications-verification-server/internal/icsv"
-	"github.com/google/exposure-notifications-verification-server/internal/project"
-	"github.com/google/exposure-notifications-verification-server/pkg/cache"
 	"github.com/google/exposure-notifications-verification-server/pkg/controller"
-	"github.com/google/exposure-notifications-verification-server/pkg/database"
 	"github.com/google/exposure-notifications-verification-server/pkg/rbac"
 )
 
-const cacheTimeout = 30 * time.Minute
-
 func (c *Controller) HandleStats() http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		ctx := r.Context()
@@ -47,112 +35,8 @@ func (c *Controller) HandleStats() http.Handler {
 			return
 		}
 
-		currentRealm := membership.Realm
-
-		now := time.Now().UTC()
-		past := now.Add(-30 * 24 * time.Hour)
-
-		pth := r.URL.Path
-		switch {
-		case strings.HasSuffix(pth, ".csv"):
-			var filename string
-			var stats icsv.Marshaler
-			var err error
-
-			nowFormatted := now.Format(project.RFC3339Squish)
-
-			switch r.URL.Query().Get("scope") {
-			case "external":
-				filename = fmt.Sprintf("%s-external-issuer-stats.csv", nowFormatted)
-				stats, err = c.getExternalIssuerStats(ctx, currentRealm, now, past)
-			case "user":
-				filename = fmt.Sprintf("%s-user-stats.csv", nowFormatted)
-				stats, err = c.getUserStats(ctx, currentRealm, now, past)
-			default:
-				filename = fmt.Sprintf("%s-realm-stats.csv", nowFormatted)
-				stats, err = c.getRealmStats(ctx, currentRealm, now, past)
-			}
-
-			if err != nil {
-				controller.InternalError(w, r, c.h, err)
-				return
-			}
-
-			c.h.RenderCSV(w, http.StatusOK, filename, stats)
-		case strings.HasSuffix(pth, ".json"):
-			var stats json.Marshaler
-			var err error
-
-			switch r.URL.Query().Get("scope") {
-			case "external":
-				stats, err = c.getExternalIssuerStats(ctx, currentRealm, now, past)
-			case "user":
-				stats, err = c.getUserStats(ctx, currentRealm, now, past)
-			default:
-				stats, err = c.getRealmStats(ctx, currentRealm, now, past)
-			}
-
-			if err != nil {
-				controller.InternalError(w, r, c.h, err)
-				return
-			}
-
-			c.h.RenderJSON(w, http.StatusOK, stats)
-		default:
-			// Fallback to HTML
-			c.renderHTML(ctx, w, currentRealm)
-			return
-		}
+		m := controller.TemplateMapFromContext(ctx)
+		m.Title("Realm stats")
+		c.h.RenderHTML(w, "realmadmin/stats", m)
 	})
 }
-
-func (c *Controller) renderHTML(ctx context.Context, w http.ResponseWriter, realm *database.Realm) {
-	m := controller.TemplateMapFromContext(ctx)
-	m.Title("Realm stats")
-	c.h.RenderHTML(w, "realmadmin/stats", m)
-}
-
-// getRealmStats returns the realm stats for a given date range.
-func (c *Controller) getRealmStats(ctx context.Context, realm *database.Realm, now, past time.Time) (database.RealmStats, error) {
-	var stats database.RealmStats
-	cacheKey := &cache.Key{
-		Namespace: "stats:realm",
-		Key:       strconv.FormatUint(uint64(realm.ID), 10),
-	}
-	if err := c.cacher.Fetch(ctx, cacheKey, &stats, cacheTimeout, func() (interface{}, error) {
-		return realm.Stats(c.db, past, now)
-	}); err != nil {
-		return nil, err
-	}
-	return stats, nil
-}
-
-// getUserStats gets the per-user realm stats for a given date range.
-func (c *Controller) getUserStats(ctx context.Context, realm *database.Realm, now, past time.Time) (database.RealmUserStats, error) {
-	var userStats database.RealmUserStats
-	cacheKey := &cache.Key{
-		Namespace: "stats:realm:per_user",
-		Key:       strconv.FormatUint(uint64(realm.ID), 10),
-	}
-	if err := c.cacher.Fetch(ctx, cacheKey, &userStats, cacheTimeout, func() (interface{}, error) {
-		return realm.UserStats(c.db, past, now)
-	}); err != nil {
-		return nil, err
-	}
-	return userStats, nil
-}
-
-// getExternalIssuerStats gets the external issuer stats for a given date range.
-func (c *Controller) getExternalIssuerStats(ctx context.Context, realm *database.Realm, now, past time.Time) (database.ExternalIssuerStats, error) {
-	var stats database.ExternalIssuerStats
-	cacheKey := &cache.Key{
-		Namespace: "stats:realm:per_external_issuer",
-		Key:       strconv.FormatUint(uint64(realm.ID), 10),
-	}
-	if err := c.cacher.Fetch(ctx, cacheKey, &stats, cacheTimeout, func() (interface{}, error) {
-		return realm.ExternalIssuerStats(c.db, past, now)
-	}); err != nil {
-		return nil, err
-	}
-	return stats, nil
-}