Add modeling service for abuse prevention

builders/build.yaml

@@ -192,6 +192,32 @@ steps:
   waitFor:
   - 'build-migrate'
 
+#
+# modeler
+#
+- id: 'build-modeler'
+  name: 'golang:1.15.1'
+  args:
+  - 'go'
+  - 'build'
+  - '-trimpath'
+  - '-ldflags=-s -w -X=${_REPO}/pkg/buildinfo.BuildID=${BUILD_ID} -X=${_REPO}/pkg/buildinfo.BuildTag=${_TAG} -extldflags=-static'
+  - '-o=./bin/modeler'
+  - './cmd/modeler'
+  waitFor:
+  - 'download-modules'
+
+- id: 'dockerize-modeler'
+  name: 'docker:19'
+  args:
+  - 'build'
+  - '--file=builders/service.dockerfile'
+  - '--tag=gcr.io/${PROJECT_ID}/${_REPO}/modeler:${_TAG}'
+  - '--build-arg=SERVICE=modeler'
+  - '.'
+  waitFor:
+  - 'build-modeler'
+
 #
 # server
 #
@@ -224,4 +250,5 @@ images:
 - 'gcr.io/${PROJECT_ID}/${_REPO}/cleanup:${_TAG}'
 - 'gcr.io/${PROJECT_ID}/${_REPO}/e2e-runner:${_TAG}'
 - 'gcr.io/${PROJECT_ID}/${_REPO}/migrate:${_TAG}'
+- 'gcr.io/${PROJECT_ID}/${_REPO}/modeler:${_TAG}'
 - 'gcr.io/${PROJECT_ID}/${_REPO}/server:${_TAG}'

builders/deploy.yaml

@@ -106,6 +106,26 @@ steps:
   waitFor:
   - '-'
 
+#
+# modeler
+#
+- id: 'deploy-modeler'
+  name: 'gcr.io/google.com/cloudsdktool/cloud-sdk:307.0.0-alpine'
+  args:
+  - 'bash'
+  - '-eEuo'
+  - 'pipefail'
+  - '-c'
+  - |-
+    gcloud run deploy "modeler" \
+      --quiet \
+      --project "${PROJECT_ID}" \
+      --platform "managed" \
+      --region "${_REGION}" \
+      --image "gcr.io/${PROJECT_ID}/${_REPO}/modeler:${_TAG}" \
+      --no-traffic
+  waitFor:
+  - '-'
 
 #
 # server

builders/promote.yaml

@@ -102,6 +102,26 @@ steps:
   waitFor:
   - '-'
 
+#
+# modeler
+#
+- id: 'promote-modeler'
+  name: 'gcr.io/google.com/cloudsdktool/cloud-sdk:307.0.0-alpine'
+  args:
+  - 'bash'
+  - '-eEuo'
+  - 'pipefail'
+  - '-c'
+  - |-
+    gcloud run services update-traffic "modeler" \
+      --quiet \
+      --project "${PROJECT_ID}" \
+      --platform "managed" \
+      --region "${_REGION}" \
+      --to-revisions "${_REVISION}=${_PERCENTAGE}"
+  waitFor:
+  - '-'
+
 #
 # server
 #

cmd/modeler/main.go

@@ -0,0 +1,114 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This server builds or re-builds the statistical models for predicting the
+// future number of codes a realm with generate for abuse prevention.
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strconv"
+
+	"github.com/google/exposure-notifications-verification-server/pkg/buildinfo"
+	"github.com/google/exposure-notifications-verification-server/pkg/config"
+	"github.com/google/exposure-notifications-verification-server/pkg/controller/modeler"
+	"github.com/google/exposure-notifications-verification-server/pkg/ratelimit"
+	"github.com/google/exposure-notifications-verification-server/pkg/render"
+	"github.com/sethvargo/go-signalcontext"
+
+	"github.com/google/exposure-notifications-server/pkg/logging"
+	"github.com/google/exposure-notifications-server/pkg/observability"
+	"github.com/google/exposure-notifications-server/pkg/server"
+
+	"github.com/gorilla/mux"
+)
+
+func main() {
+	ctx, done := signalcontext.OnInterrupt()
+
+	debug, _ := strconv.ParseBool(os.Getenv("LOG_DEBUG"))
+	logger := logging.NewLogger(debug)
+	logger = logger.With("build_id", buildinfo.BuildID)
+	logger = logger.With("build_tag", buildinfo.BuildTag)
+
+	ctx = logging.WithLogger(ctx, logger)
+
+	err := realMain(ctx)
+	done()
+
+	if err != nil {
+		logger.Fatal(err)
+	}
+	logger.Info("successful shutdown")
+}
+
+func realMain(ctx context.Context) error {
+	logger := logging.FromContext(ctx)
+
+	config, err := config.NewModeler(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to process config: %w", err)
+	}
+
+	// Setup monitoring
+	logger.Info("configuring observability exporter")
+	oeConfig := config.ObservabilityExporterConfig()
+	oe, err := observability.NewFromEnv(ctx, oeConfig)
+	if err != nil {
+		return fmt.Errorf("unable to create ObservabilityExporter provider: %w", err)
+	}
+	if err := oe.StartExporter(); err != nil {
+		return fmt.Errorf("error initializing observability exporter: %w", err)
+	}
+	defer oe.Close()
+	logger.Infow("observability exporter", "config", oeConfig)
+
+	// Setup database
+	db, err := config.Database.Load(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to load database config: %w", err)
+	}
+	if err := db.Open(ctx); err != nil {
+		return fmt.Errorf("failed to connect to database: %w", err)
+	}
+	defer db.Close()
+
+	// Create the renderer
+	h, err := render.New(ctx, "", config.DevMode)
+	if err != nil {
+		return fmt.Errorf("failed to create renderer: %w", err)
+	}
+
+	// Create the router
+	r := mux.NewRouter()
+
+	// Rate limiting
+	limiterStore, err := ratelimit.RateLimiterFor(ctx, &config.RateLimit)
+	if err != nil {
+		return fmt.Errorf("failed to create limiter: %w", err)
+	}
+	defer limiterStore.Close(ctx)
+
+	modelerController := modeler.New(ctx, config, db, limiterStore, h)
+	r.Handle("/", modelerController.HandleModel()).Methods("POST")
+
+	srv, err := server.New(config.Port)
+	if err != nil {
+		return fmt.Errorf("failed to create server: %w", err)
+	}
+	logger.Infow("server listening", "port", config.Port)
+	return srv.ServeHTTPHandler(ctx, r)
+}

cmd/server/assets/header.html

@@ -409,3 +409,10 @@ <h6 class="dropdown-header">Actions</h6>
   </a>
 </div>
 {{end}}
+
+{{define "beta-notice"}}
+<div class="alert alert-warning">
+  <span class="oi oi-beaker"></span> This feature is still under active
+  development.
+</div>
+{{end}}

cmd/server/assets/realm.html

@@ -356,17 +356,20 @@ <h1>Realm settings</h1>
           Abuse prevention
         </div>
         <div class="card-body">
-          <div class="alert alert-warning">
-            <span class="oi oi-beaker"></span> This feature is still under
-            active development.
-          </div>
+          {{template "beta-notice" .}}
 
           <p>
             Abuse prevention uses the historical record of your realm's past
             daily code issuances to build a predictive model of future use,
             rejecting requests that fall outside of the predicted model.
           </p>
 
+          <p>
+            Without abuse protection, an attacker with a compromised credential
+            could generate many fake codes and then use those codes to
+            subsequently upload many fake keys to the system.
+          </p>
+
           <div class="form-group form-check">
             <input class="form-check-input" type="checkbox" name="abuse_prevention_enabled" id="abuse_prevention_enabled" value="1" {{if $realm.AbusePreventionEnabled}} checked{{end}}>
             <label class="form-check-label" for="abuse_prevention_enabled">
@@ -407,6 +410,17 @@ <h1>Realm settings</h1>
                 applying your limit factor.
               </small>
             </div>
+
+            <div class="form-label-group">
+              <input type="text" id="abuse_prevention_burst" name="abuse_prevention_burst" class="form-control" placeholder="Temporary burst" value="0" />
+              <label for="abuse_prevention_burst">Temporary burst</label>
+              <small class="form-text text-muted">
+                Set this value to temporarily add quota to your realm. You
+                should only do this when you've exceeded your daily quota and
+                still need to issue new codes. After 00:00 UTC, the quota will
+                reset back to the predicted model automatically.
+              </small>
+            </div>
           </div>
         </div>
       </div>

cmd/server/assets/users/import.html

@@ -24,10 +24,7 @@ <h1>Import users</h1>
     <div class="card mb-3 shadow-sm">
       <div class="card-header">Import options</div>
       <div class="card-body">
-        <div class="alert alert-warning">
-          <span class="oi oi-beaker"></span> This feature is still under
-          active development.
-        </div>
+        {{template "beta-notice" .}}
         <small class="form-text text-muted">Coming soon</small>
       </div>
     </div>

cmd/server/main.go

@@ -328,7 +328,7 @@ func realMain(ctx context.Context) error {
 		realmSub.Use(requireMFA)
 		realmSub.Use(rateLimit)
 
-		realmadminController := realmadmin.New(ctx, cacher, config, db, h)
+		realmadminController := realmadmin.New(ctx, cacher, config, db, limiterStore, h)
 		realmSub.Handle("/settings", realmadminController.HandleIndex()).Methods("GET")
 		realmSub.Handle("/settings/save", realmadminController.HandleSave()).Methods("POST")
 		realmSub.Handle("/settings/enable-express", realmadminController.HandleEnableExpress()).Methods("POST")

go.mod

@@ -14,13 +14,19 @@ require (
 	github.com/client9/misspell v0.3.4
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/frankban/quicktest v1.8.1 // indirect
+	github.com/gonum/blas v0.0.0-20181208220705-f22b278b28ac // indirect
+	github.com/gonum/floats v0.0.0-20181209220543-c233463c7e82 // indirect
+	github.com/gonum/internal v0.0.0-20181124074243-f884aa714029 // indirect
+	github.com/gonum/lapack v0.0.0-20181123203213-e4cdc5a0bff9 // indirect
+	github.com/gonum/matrix v0.0.0-20181209220409-c518dec07be9
 	github.com/google/exposure-notifications-server v0.8.0
 	github.com/google/go-cmp v0.5.2
 	github.com/gorilla/csrf v1.7.0
 	github.com/gorilla/handlers v1.5.0
 	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/schema v1.2.0
 	github.com/gorilla/sessions v1.2.1
+	github.com/hashicorp/go-multierror v1.1.0
 	github.com/jinzhu/gorm v1.9.16
 	github.com/jinzhu/now v1.1.1 // indirect
 	github.com/kelseyhightower/envconfig v1.4.0 // indirect
@@ -31,9 +37,9 @@ require (
 	github.com/ory/dockertest v3.3.5+incompatible
 	github.com/rakutentech/jwk-go v1.0.1
 	github.com/sethvargo/go-envconfig v0.3.1
-	github.com/sethvargo/go-limiter v0.5.1
+	github.com/sethvargo/go-limiter v0.5.2
 	github.com/sethvargo/go-password v0.2.0
-	github.com/sethvargo/go-redisstore v0.2.0-opencensus
+	github.com/sethvargo/go-redisstore v0.2.1-opencensus
 	github.com/sethvargo/go-retry v0.1.0
 	github.com/sethvargo/go-signalcontext v0.1.0
 	github.com/smartystreets/assertions v1.0.0 // indirect

go.sum

@@ -283,6 +283,7 @@ github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:z
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
 github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 h1:sDMmm+q/3+BukdIpxwO365v/Rbspp2Nt5XntgQRXq8Q=
 github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
+github.com/containerd/containerd v1.3.3 h1:LoIzb5y9x5l8VKAlyrbusNPXqBY0+kviRloxFUMFwKc=
 github.com/containerd/containerd v1.3.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
 github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
 github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe h1:PEmIrUvwG9Yyv+0WKZqjXfSFDeZjs/q15g0m08BYS9k=
@@ -321,7 +322,9 @@ github.com/dimchansky/utfbom v1.1.0 h1:FcM3g+nofKgUteL8dm/UpdRXNC9KmADgTpLKsu0TR
 github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8=
 github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
 github.com/docker/distribution v2.7.0+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
+github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
+github.com/docker/docker v1.4.2-0.20200213202729-31a86c4ab209 h1:tmV+YbYOUAYDmAiamzhRKqQXaAUyUY2xVt27Rv7rCzA=
 github.com/docker/docker v1.4.2-0.20200213202729-31a86c4ab209/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
 github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
@@ -486,6 +489,16 @@ github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8l
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/gonum/blas v0.0.0-20181208220705-f22b278b28ac h1:Q0Jsdxl5jbxouNs1TQYt0gxesYMU4VXRbsTlgDloZ50=
+github.com/gonum/blas v0.0.0-20181208220705-f22b278b28ac/go.mod h1:P32wAyui1PQ58Oce/KYkOqQv8cVw1zAapXOl+dRFGbc=
+github.com/gonum/floats v0.0.0-20181209220543-c233463c7e82 h1:EvokxLQsaaQjcWVWSV38221VAK7qc2zhaO17bKys/18=
+github.com/gonum/floats v0.0.0-20181209220543-c233463c7e82/go.mod h1:PxC8OnwL11+aosOB5+iEPoV3picfs8tUpkVd0pDo+Kg=
+github.com/gonum/internal v0.0.0-20181124074243-f884aa714029 h1:8jtTdc+Nfj9AR+0soOeia9UZSvYBvETVHZrugUowJ7M=
+github.com/gonum/internal v0.0.0-20181124074243-f884aa714029/go.mod h1:Pu4dmpkhSyOzRwuXkOgAvijx4o+4YMUJJo9OvPYMkks=
+github.com/gonum/lapack v0.0.0-20181123203213-e4cdc5a0bff9 h1:7qnwS9+oeSiOIsiUMajT+0R7HR6hw5NegnKPmn/94oI=
+github.com/gonum/lapack v0.0.0-20181123203213-e4cdc5a0bff9/go.mod h1:XA3DeT6rxh2EAE789SSiSJNqxPaC0aE9J8NTOI0Jo/A=
+github.com/gonum/matrix v0.0.0-20181209220409-c518dec07be9 h1:V2IgdyerlBa/MxaEFRbV5juy/C3MGdj4ePi+g6ePIp4=
+github.com/gonum/matrix v0.0.0-20181209220409-c518dec07be9/go.mod h1:0EXg4mc1CNP0HCqCz+K4ts155PXIlUywf0wqN+GfPZw=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@@ -925,6 +938,7 @@ github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHX
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
+github.com/mattn/go-shellwords v1.0.5 h1:JhhFTIOslh5ZsPrpa3Wdg8bF0WI3b44EMblmU9wIsXc=
 github.com/mattn/go-shellwords v1.0.5/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
 github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
@@ -975,6 +989,7 @@ github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/mongodb/go-client-mongodb-atlas v0.1.2 h1:qmUme1TlQBPZupmXMnpD8DxnfGXLVGs3w+0Z17HBiSA=
 github.com/mongodb/go-client-mongodb-atlas v0.1.2/go.mod h1:LS8O0YLkA+sbtOb3fZLF10yY3tJM+1xATXMJ3oU35LU=
+github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
 github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwielbut/pointy v1.1.0 h1:U5/YEfoIkaGCHv0St3CgjduqXID4FNRoyZgLM1kY9vg=
@@ -1164,12 +1179,12 @@ github.com/sethvargo/go-envconfig v0.3.1 h1:OUnL02SWTz+t8XtxTO6YQuLMi3StljJHzmtP
 github.com/sethvargo/go-envconfig v0.3.1/go.mod h1:XZ2JRR7vhlBEO5zMmOpLgUhgYltqYqq4d4tKagtPUv0=
 github.com/sethvargo/go-gcpkms v0.1.0 h1:pyjDLqLwpk9pMjDSTilPpaUjgP1AfSjX9WGzitZwGUY=
 github.com/sethvargo/go-gcpkms v0.1.0/go.mod h1:33BuvqUjsYk0bpMgn+WCclCYtMLOyaqtn5j0fCo4vvk=
-github.com/sethvargo/go-limiter v0.5.1 h1:3Ss4/AC1hXrHwQv75xMjGYJDCeMGeHv8B5tqKIi5+Pc=
-github.com/sethvargo/go-limiter v0.5.1/go.mod h1:C0kbSFbiriE5k2FFOe18M1YZbAR2Fiwf72uGu0CXCcU=
+github.com/sethvargo/go-limiter v0.5.2 h1:NIFp7xy3NyE2+mEHbengdLQF0C0STOpwF5Qw5JtayIs=
+github.com/sethvargo/go-limiter v0.5.2/go.mod h1:C0kbSFbiriE5k2FFOe18M1YZbAR2Fiwf72uGu0CXCcU=
 github.com/sethvargo/go-password v0.2.0 h1:BTDl4CC/gjf/axHMaDQtw507ogrXLci6XRiLc7i/UHI=
 github.com/sethvargo/go-password v0.2.0/go.mod h1:Ym4Mr9JXLBycr02MFuVQ/0JHidNetSgbzutTr3zsYXE=
-github.com/sethvargo/go-redisstore v0.2.0-opencensus h1:GoKpIA/d0KDotrHhk18syOoDthgbBMzdhe0iguvyXbQ=
-github.com/sethvargo/go-redisstore v0.2.0-opencensus/go.mod h1:ovmJaSVc9yhiG+7is6gf3uSTC92kzAbrJsx6L0vnPZU=
+github.com/sethvargo/go-redisstore v0.2.1-opencensus h1:EAwZAuZr5DJdLmEruJTj2zeBvmZsIXI7wqgMueuaxks=
+github.com/sethvargo/go-redisstore v0.2.1-opencensus/go.mod h1:TMFAy7azG5hDd/Hb5ng2CDsawcxg1+oEuGhuVp7eycI=
 github.com/sethvargo/go-retry v0.1.0 h1:8sPqlWannzcReEcYjHSNw9becsiYudcwTD7CasGjQaI=
 github.com/sethvargo/go-retry v0.1.0/go.mod h1:JzIOdZqQDNpPkQDmcqgtteAcxFLtYpNF/zJCM1ysDg8=
 github.com/sethvargo/go-signalcontext v0.1.0 h1:3IU7HOlmRXF0PSDf85C4nJ/zjYDjF+DS+LufcKfLvyk=