Add per-realm stats (#514)

* Add per realm stats Adds per realm stats, and a simple visualization of the codes issued and claimed for the past month. * Handle review feedback * More Co-authored-by: Jeremy Faller <jeremy@golang.org>
google · Sep 10, 2020 · 6f4db71 · 6f4db71
1 parent 19145ac
commit 6f4db71
Show file tree

Hide file tree

Showing 12 changed files with 273 additions and 16 deletions.
diff --git a/cmd/server/assets/apikeys/show.html b/cmd/server/assets/apikeys/show.html
@@ -64,7 +64,7 @@ <h1>{{$authApp.Name}} API key</h1>
       <div class="card-header">Statistics</div>
       <div class="card-body table-responsive">
         {{if $stats}}
-        <div id="chart" style="height: 300px;" class="m-3">
+        <div id="chart" class="mb-3" style="height: 300px;">
           <span>Loading chart...</span>
         </div>
         <table class="table table-bordered table-striped">
@@ -122,7 +122,7 @@ <h1>{{$authApp.Name}} API key</h1>
       };
 
       var chart = new google.charts.Line(document.getElementById('chart'));
-      chart.draw(data, options);
+      chart.draw(data, google.charts.Line.convertOptions(options));
     }
   </script>
   {{end}}

diff --git a/cmd/server/assets/header.html b/cmd/server/assets/header.html
@@ -197,6 +197,7 @@
               <h6 class="dropdown-header">Manage realm</h6>
               <a class="dropdown-item" href="/apikeys">API keys</a>
               <a class="dropdown-item" href="/realm/keys">Signing keys</a>
+              <a class="dropdown-item" href="/realm/stats">Statistics</a>
               <a class="dropdown-item" href="/users">Users</a>
               <a class="dropdown-item" href="/realm/settings">Settings</a>
               <div class="dropdown-divider"></div>

diff --git a/cmd/server/assets/realmstats.html b/cmd/server/assets/realmstats.html
@@ -0,0 +1,85 @@
+{{define "realmstats"}}
+
+{{$realm := .realm}}
+{{$stats := .stats}}
+
+<!doctype html>
+<html lang="en">
+<head>
+  {{template "head" .}}
+</head>
+
+<body>
+  {{template "navbar" .}}
+
+  <main role="main" class="container">
+    {{template "flash" .}}
+
+    <h1>Realm stats</h1>
+
+    <div class="card mb-3">
+      <div class="card-header">Statistics</div>
+      <div class="card-body table-responsive">
+        {{if $stats}}
+        <div id="chart" class="mb-3" style="height: 300px;">
+          <span>Loading chart...</span>
+        </div>
+        <table class="table table-bordered table-striped">
+          <thead>
+            <tr>
+              <th scope="col">Date</th>
+              <th scope="col" width="125">Issued</th>
+              <th scope="col" width="125">Claimed</th>
+            </tr>
+          </thead>
+          <tbody>
+            {{range $stat := $stats}}
+            <tr>
+              <td>{{$stat.Date.Format "2006-01-02"}}</td>
+              <td>{{$stat.CodesIssued}}</td>
+              <td>{{$stat.CodesClaimed}}</td>
+            </tr>
+            {{end}}
+          </tbody>
+        </table>
+        <div class="font-italic">
+          This data is refreshed every 5 minutes.
+        </div>
+        {{else}}
+          <p>No codes have been issued in this realm.</p>
+        {{end}}
+      </div>
+    </div>
+
+  </main>
+
+  {{template "scripts" .}}
+
+  {{if $stats}}
+  <script src="https://www.gstatic.com/charts/loader.js"></script>
+  <script>
+    google.charts.load('current', {packages: ['line']});
+    google.charts.setOnLoadCallback(drawChart)
+
+    function drawChart() {
+      var data = google.visualization.arrayToDataTable([
+        ['Date', 'Issued', "Claimed"],
+        {{range $stat := $stats}}
+        ['{{$stat.Date.Format "Jan 2"}}', {{$stat.CodesIssued}}, {{$stat.CodesClaimed}}],
+        {{end}}
+      ]);
+
+      var options = {
+        colors: ['#007bff', '#ff7b00'],
+        legend: {position: 'bottom'},
+        tooltip: {trigger: 'focus'},
+      };
+
+      var chart = new google.charts.Line(document.getElementById('chart'));
+      chart.draw(data, google.charts.Line.convertOptions(options));
+    }
+  </script>
+  {{end}}
+</body>
+</html>
+{{end}}
diff --git a/cmd/server/assets/users/show.html b/cmd/server/assets/users/show.html
@@ -48,7 +48,7 @@ <h1>{{$user.Name}}</h1>
       <div class="card-header">Statistics</div>
       <div class="card-body table-responsive">
         {{if $stats}}
-        <div id="chart" style="height: 300px;">
+        <div id="chart" class="mb-3" style="height: 300px;">
           <span>Loading chart...</span>
         </div>
         <table class="table table-bordered table-striped">
@@ -106,7 +106,7 @@ <h1>{{$user.Name}}</h1>
       };
 
       var chart = new google.charts.Line(document.getElementById('chart'));
-      chart.draw(data, options);
+      chart.draw(data, google.charts.Line.convertOptions(options));
     }
   </script>
   {{end}}

diff --git a/cmd/server/main.go b/cmd/server/main.go
@@ -323,11 +323,12 @@ func realMain(ctx context.Context) error {
 		realmSub.Use(requireAdmin)
 		realmSub.Use(rateLimit)
 
-		realmadminController := realmadmin.New(ctx, config, db, h)
+		realmadminController := realmadmin.New(ctx, cacher, config, db, h)
 		realmSub.Handle("/settings", realmadminController.HandleIndex()).Methods("GET")
 		realmSub.Handle("/settings/save", realmadminController.HandleSave()).Methods("POST")
 		realmSub.Handle("/settings/enable-express", realmadminController.HandleEnableExpress()).Methods("POST")
 		realmSub.Handle("/settings/disable-express", realmadminController.HandleDisableExpress()).Methods("POST")
+		realmSub.Handle("/stats", realmadminController.HandleShow()).Methods("GET")
 
 		realmKeysController, err := realmkeys.New(ctx, config, db, certificateSigner, h)
 		if err != nil {

diff --git a/pkg/controller/realmadmin/realmadmin.go b/pkg/controller/realmadmin/realmadmin.go
@@ -18,6 +18,7 @@ package realmadmin
 import (
 	"context"
 
+	"github.com/google/exposure-notifications-verification-server/pkg/cache"
 	"github.com/google/exposure-notifications-verification-server/pkg/config"
 	"github.com/google/exposure-notifications-verification-server/pkg/database"
 	"github.com/google/exposure-notifications-verification-server/pkg/render"
@@ -28,16 +29,18 @@ import (
 )
 
 type Controller struct {
+	cacher cache.Cacher
 	config *config.ServerConfig
 	db     *database.Database
 	h      *render.Renderer
 	logger *zap.SugaredLogger
 }
 
-func New(ctx context.Context, config *config.ServerConfig, db *database.Database, h *render.Renderer) *Controller {
+func New(ctx context.Context, cacher cache.Cacher, config *config.ServerConfig, db *database.Database, h *render.Renderer) *Controller {
 	logger := logging.FromContext(ctx)
 
 	return &Controller{
+		cacher: cacher,
 		config: config,
 		db:     db,
 		h:      h,

diff --git a/pkg/controller/realmadmin/show.go b/pkg/controller/realmadmin/show.go
@@ -0,0 +1,58 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package realmadmin
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/google/exposure-notifications-verification-server/pkg/controller"
+	"github.com/google/exposure-notifications-verification-server/pkg/database"
+)
+
+func (c *Controller) HandleShow() http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		ctx := r.Context()
+
+		realm := controller.RealmFromContext(ctx)
+		if realm == nil {
+			controller.MissingRealm(w, r, c.h)
+			return
+		}
+
+		// Get and cache the stats for this user.
+		var stats []*database.RealmStats
+		cacheKey := fmt.Sprintf("stats:realm:%d", realm.ID)
+		if err := c.cacher.Fetch(ctx, cacheKey, &stats, 5*time.Minute, func() (interface{}, error) {
+			now := time.Now().UTC()
+			past := now.Add(-30 * 24 * time.Hour)
+			return realm.Stats(c.db, past, now)
+		}); err != nil {
+			controller.InternalError(w, r, c.h, err)
+			return
+		}
+
+		c.renderStats(ctx, w, realm, stats)
+	})
+}
+
+func (c *Controller) renderStats(ctx context.Context, w http.ResponseWriter, realm *database.Realm, stats []*database.RealmStats) {
+	m := controller.TemplateMapFromContext(ctx)
+	m["user"] = realm
+	m["stats"] = stats
+	c.h.RenderHTML(w, "realmstats", m)
+}
diff --git a/pkg/database/migrations.go b/pkg/database/migrations.go
@@ -868,6 +868,31 @@ func (db *Database) getMigrations(ctx context.Context) *gormigrate.Gormigrate {
 				return tx.Exec("ALTER TABLE realms DROP COLUMN IF EXISTS mfa_mode").Error
 			},
 		},
+		{
+			ID: "00036-AddRealmStats",
+			Migrate: func(tx *gorm.DB) error {
+				logger.Debugw("db migrations: adding realm stats")
+				if err := tx.AutoMigrate(&RealmStats{}).Error; err != nil {
+					return err
+				}
+				statements := []string{
+					"CREATE UNIQUE INDEX IF NOT EXISTS idx_realm_stats_stats_date_realm_id ON realm_stats (date, realm_id)",
+					"CREATE INDEX IF NOT EXISTS idx_realm_stats_date ON realm_stats (date)",
+				}
+				for _, sql := range statements {
+					if err := tx.Exec(sql).Error; err != nil {
+						return err
+					}
+				}
+				return nil
+			},
+			Rollback: func(tx *gorm.DB) error {
+				if err := tx.DropTable(&RealmStats{}).Error; err != nil {
+					return err
+				}
+				return nil
+			},
+		},
 	})
 }
 

diff --git a/pkg/database/realm.go b/pkg/database/realm.go
@@ -623,3 +623,27 @@ func (r *Realm) DestroySigningKeyVersion(ctx context.Context, db *Database, id i
 
 	return nil
 }
+
+// Stats returns the usage statistics for this realm. If no stats exist,
+// returns an empty array.
+func (r *Realm) Stats(db *Database, start, stop time.Time) ([]*RealmStats, error) {
+	var stats []*RealmStats
+
+	start = start.Truncate(24 * time.Hour)
+	stop = stop.Truncate(24 * time.Hour)
+
+	if err := db.db.
+		Model(&RealmStats{}).
+		Where("realm_id = ?", r.ID).
+		Where("(date >= ? AND date <= ?)", start, stop).
+		Order("date ASC").
+		Find(&stats).
+		Error; err != nil {
+		if IsNotFound(err) {
+			return stats, nil
+		}
+		return nil, err
+	}
+
+	return stats, nil
+}
diff --git a/pkg/database/realm_stats.go b/pkg/database/realm_stats.go
@@ -0,0 +1,32 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package database
+
+import (
+	"time"
+)
+
+// RealmStats represents statistics related to a user in the database.
+type RealmStats struct {
+	Date         time.Time `gorm:"date; not null"`
+	RealmID      uint      `gorm:"realm_id; not null"`
+	CodesIssued  uint      `gorm:"codes_issued; default: 0"`
+	CodesClaimed uint      `gorm:"codes_claimed; default: 0"`
+}
+
+// TableName sets the RealmStats table name
+func (RealmStats) TableName() string {
+	return "realm_stats"
+}
diff --git a/pkg/database/token.go b/pkg/database/token.go
@@ -175,13 +175,15 @@ func (db *Database) VerifyCodeAndIssueToken(realmID uint, verCode string, accept
 			return err
 		}
 
-		if expired, err := db.IsCodeExpired(&vc, verCode); err != nil {
-			db.logger.Debugw("error checking code expiration", "error", err)
+		// Validation
+		expired, err := db.IsCodeExpired(&vc, verCode)
+		if err != nil {
+			db.logger.Errorw("failed to check code expiration", "error", err)
 			return ErrVerificationCodeExpired
-		} else if expired {
+		}
+		if expired {
 			return ErrVerificationCodeExpired
 		}
-
 		if vc.Claimed {
 			return ErrVerificationCodeUsed
 		}
@@ -190,10 +192,22 @@ func (db *Database) VerifyCodeAndIssueToken(realmID uint, verCode string, accept
 			return ErrUnsupportedTestType
 		}
 
-		// Mark claimed. Transactional update.
+		// Mark as claimed
 		vc.Claimed = true
 		if err := tx.Save(&vc).Error; err != nil {
-			return err
+			return fmt.Errorf("failed to claim token: %w", err)
+		}
+
+		// Update statistics
+		now := time.Now().Truncate(24 * time.Hour)
+		sql := `
+			INSERT INTO realm_stats(date, realm_id, codes_claimed)
+				VALUES ($1, $2, 1)
+			ON CONFLICT (date, realm_id) DO UPDATE
+				SET codes_claimed = realm_stats.codes_claimed + 1
+		`
+		if err := tx.Exec(sql, now, vc.RealmID).Error; err != nil {
+			return fmt.Errorf("failed to update stats: %w", err)
 		}
 
 		buffer := make([]byte, tokenBytes)