diff --git a/docs/api.md b/docs/api.md index 47163c4e0..306e998bf 100644 --- a/docs/api.md +++ b/docs/api.md @@ -336,7 +336,7 @@ eg. error: "the first code failed", errorCode: "missing_date", } -`` +``` **BatchIssueCodeRequest** @@ -468,7 +468,7 @@ notice!** This path includes realm-level statistics for the past 30 days. - `/api/stats/realm.{csv,json}` - Daily statistics for the realm, including - codes issued, codes claimed, and daily active users (if enabled). + codes issued, codes claimed, tokens claimed, and invalid attempts. - `/api/stats/realm/users.{csv,json}` - Daily statistics for codes issued by realm user. These statistics only include codes issued by humans logged into diff --git a/pkg/controller/apikey/stats.go b/pkg/controller/apikey/stats.go index d3f452f3d..5b14732d0 100644 --- a/pkg/controller/apikey/stats.go +++ b/pkg/controller/apikey/stats.go @@ -15,23 +15,18 @@ package apikey import ( - "context" "fmt" "net/http" - "strconv" "strings" "time" "github.com/google/exposure-notifications-verification-server/internal/project" - "github.com/google/exposure-notifications-verification-server/pkg/cache" "github.com/google/exposure-notifications-verification-server/pkg/controller" "github.com/google/exposure-notifications-verification-server/pkg/database" "github.com/google/exposure-notifications-verification-server/pkg/rbac" "github.com/gorilla/mux" ) -const statsCacheTimeout = 30 * time.Minute - func (c *Controller) HandleStats() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { ctx := r.Context() @@ -66,7 +61,7 @@ func (c *Controller) HandleStats() http.Handler { return } - stats, err := c.getStats(ctx, authApp, currentRealm) + stats, err := authApp.StatsCached(ctx, c.db, c.cacher) if err != nil { controller.InternalError(w, r, c.h, err) return @@ -88,20 +83,3 @@ func (c *Controller) HandleStats() http.Handler { } }) } - -func (c *Controller) getStats(ctx context.Context, authApp *database.AuthorizedApp, realm *database.Realm) (database.AuthorizedAppStats, error) { - now := time.Now().UTC() - past := now.Add(-30 * 24 * time.Hour) - - var stats database.AuthorizedAppStats - cacheKey := &cache.Key{ - Namespace: "stats:app", - Key: strconv.FormatUint(uint64(authApp.ID), 10), - } - if err := c.cacher.Fetch(ctx, cacheKey, &stats, statsCacheTimeout, func() (interface{}, error) { - return authApp.Stats(c.db, past, now) - }); err != nil { - return nil, err - } - return stats, nil -} diff --git a/pkg/database/authorized_app.go b/pkg/database/authorized_app.go index 77a136120..731dada1c 100644 --- a/pkg/database/authorized_app.go +++ b/pkg/database/authorized_app.go @@ -15,6 +15,7 @@ package database import ( + "context" "crypto/hmac" "crypto/rand" "crypto/sha512" @@ -27,6 +28,7 @@ import ( "github.com/google/exposure-notifications-server/pkg/base64util" "github.com/google/exposure-notifications-server/pkg/timeutils" "github.com/google/exposure-notifications-verification-server/internal/project" + "github.com/google/exposure-notifications-verification-server/pkg/cache" "github.com/jinzhu/gorm" ) @@ -192,10 +194,9 @@ func (db *Database) FindAuthorizedAppByAPIKey(apiKey string) (*AuthorizedApp, er // Stats returns the usage statistics for this app. If no stats exist, it // returns an empty array. -func (a *AuthorizedApp) Stats(db *Database, start, stop time.Time) (AuthorizedAppStats, error) { - start = timeutils.UTCMidnight(start) - stop = timeutils.UTCMidnight(stop) - +func (a *AuthorizedApp) Stats(db *Database) (AuthorizedAppStats, error) { + stop := timeutils.Midnight(time.Now().UTC()) + start := stop.Add(30 * -24 * time.Hour) if start.After(stop) { return nil, ErrBadDateRange } @@ -208,7 +209,11 @@ func (a *AuthorizedApp) Stats(db *Database, start, stop time.Time) (AuthorizedAp d.date AS date, $1 AS authorized_app_id, $2 AS authorized_app_name, - COALESCE(s.codes_issued, 0) AS codes_issued + COALESCE(s.codes_issued, 0) AS codes_issued, + COALESCE(s.codes_claimed, 0) AS codes_claimed, + COALESCE(s.codes_invalid, 0) AS codes_invalid, + COALESCE(s.tokens_claimed, 0) AS tokens_claimed, + COALESCE(s.tokens_invalid, 0) AS tokens_invalid FROM ( SELECT date::date FROM generate_series($3, $4, '1 day'::interval) date ) d @@ -225,6 +230,25 @@ func (a *AuthorizedApp) Stats(db *Database, start, stop time.Time) (AuthorizedAp return stats, nil } +// StatsCached is stats, but cached. +func (a *AuthorizedApp) StatsCached(ctx context.Context, db *Database, cacher cache.Cacher) (AuthorizedAppStats, error) { + if cacher == nil { + return nil, fmt.Errorf("cacher cannot be nil") + } + + var stats AuthorizedAppStats + cacheKey := &cache.Key{ + Namespace: "stats:app", + Key: strconv.FormatUint(uint64(a.ID), 10), + } + if err := cacher.Fetch(ctx, cacheKey, &stats, 30*time.Minute, func() (interface{}, error) { + return a.Stats(db) + }); err != nil { + return nil, err + } + return stats, nil +} + // SaveAuthorizedApp saves the authorized app. func (db *Database) SaveAuthorizedApp(a *AuthorizedApp, actor Auditable) error { if a == nil { diff --git a/pkg/database/authorized_app_stats.go b/pkg/database/authorized_app_stats.go index c6c4181b2..c34cdf23a 100644 --- a/pkg/database/authorized_app_stats.go +++ b/pkg/database/authorized_app_stats.go @@ -66,7 +66,11 @@ func (s AuthorizedAppStats) MarshalCSV() ([]byte, error) { var b bytes.Buffer w := csv.NewWriter(&b) - if err := w.Write([]string{"date", "authorized_app_id", "authorized_app_name", "codes_issued"}); err != nil { + if err := w.Write([]string{ + "date", "authorized_app_id", "authorized_app_name", + "codes_issued", "codes_claimed", "codes_invalid", + "tokens_claimed", "tokens_invalid", + }); err != nil { return nil, fmt.Errorf("failed to write CSV header: %w", err) } @@ -76,6 +80,10 @@ func (s AuthorizedAppStats) MarshalCSV() ([]byte, error) { strconv.FormatUint(uint64(stat.AuthorizedAppID), 10), stat.AuthorizedAppName, strconv.FormatUint(uint64(stat.CodesIssued), 10), + strconv.FormatUint(uint64(stat.CodesClaimed), 10), + strconv.FormatUint(uint64(stat.CodesInvalid), 10), + strconv.FormatUint(uint64(stat.TokensClaimed), 10), + strconv.FormatUint(uint64(stat.TokensInvalid), 10), }); err != nil { return nil, fmt.Errorf("failed to write CSV entry %d: %w", i, err) } @@ -101,7 +109,11 @@ type jsonAuthorizedAppStatstats struct { } type jsonAuthorizedAppStatstatsData struct { - CodesIssued uint `json:"codes_issued"` + CodesIssued uint `json:"codes_issued"` + CodesClaimed uint `json:"codes_claimed"` + CodesInvalid uint `json:"codes_invalid"` + TokensClaimed uint `json:"tokens_claimed"` + TokensInvalid uint `json:"tokens_invalid"` } // MarshalJSON is a custom JSON marshaller. @@ -116,7 +128,11 @@ func (s AuthorizedAppStats) MarshalJSON() ([]byte, error) { stats = append(stats, &jsonAuthorizedAppStatstats{ Date: stat.Date, Data: &jsonAuthorizedAppStatstatsData{ - CodesIssued: stat.CodesIssued, + CodesIssued: stat.CodesIssued, + CodesClaimed: stat.CodesClaimed, + CodesInvalid: stat.CodesInvalid, + TokensClaimed: stat.TokensClaimed, + TokensInvalid: stat.TokensInvalid, }, }) } @@ -154,6 +170,10 @@ func (s *AuthorizedAppStats) UnmarshalJSON(b []byte) error { AuthorizedAppID: result.AuthorizedAppID, AuthorizedAppName: result.AuthorizedAppName, CodesIssued: stat.Data.CodesIssued, + CodesClaimed: stat.Data.CodesClaimed, + CodesInvalid: stat.Data.CodesInvalid, + TokensClaimed: stat.Data.TokensClaimed, + TokensInvalid: stat.Data.TokensInvalid, }) } diff --git a/pkg/database/authorized_app_stats_test.go b/pkg/database/authorized_app_stats_test.go index 1a8f3512f..6b6d770d2 100644 --- a/pkg/database/authorized_app_stats_test.go +++ b/pkg/database/authorized_app_stats_test.go @@ -41,11 +41,15 @@ func TestAuthorizedAppStats_MarshalCSV(t *testing.T) { Date: time.Date(2020, 2, 3, 0, 0, 0, 0, time.UTC), AuthorizedAppID: 1, CodesIssued: 10, + CodesClaimed: 4, + CodesInvalid: 2, + TokensClaimed: 3, + TokensInvalid: 1, AuthorizedAppName: "Appy", }, }, - exp: `date,authorized_app_id,authorized_app_name,codes_issued -2020-02-03,1,Appy,10 + exp: `date,authorized_app_id,authorized_app_name,codes_issued,codes_claimed,codes_invalid,tokens_claimed,tokens_invalid +2020-02-03,1,Appy,10,4,2,3,1 `, }, { @@ -55,25 +59,37 @@ func TestAuthorizedAppStats_MarshalCSV(t *testing.T) { Date: time.Date(2020, 2, 3, 0, 0, 0, 0, time.UTC), AuthorizedAppID: 1, CodesIssued: 10, + CodesClaimed: 10, + CodesInvalid: 2, + TokensClaimed: 4, + TokensInvalid: 2, AuthorizedAppName: "Appy", }, { Date: time.Date(2020, 2, 4, 0, 0, 0, 0, time.UTC), AuthorizedAppID: 1, CodesIssued: 45, + CodesClaimed: 44, + CodesInvalid: 5, + TokensClaimed: 3, + TokensInvalid: 2, AuthorizedAppName: "Mc", }, { Date: time.Date(2020, 2, 5, 0, 0, 0, 0, time.UTC), AuthorizedAppID: 1, CodesIssued: 15, + CodesClaimed: 13, + CodesInvalid: 4, + TokensClaimed: 6, + TokensInvalid: 2, AuthorizedAppName: "Apperson", }, }, - exp: `date,authorized_app_id,authorized_app_name,codes_issued -2020-02-03,1,Appy,10 -2020-02-04,1,Mc,45 -2020-02-05,1,Apperson,15 + exp: `date,authorized_app_id,authorized_app_name,codes_issued,codes_claimed,codes_invalid,tokens_claimed,tokens_invalid +2020-02-03,1,Appy,10,10,2,4,2 +2020-02-04,1,Mc,45,44,5,3,2 +2020-02-05,1,Apperson,15,13,4,6,2 `, }, } diff --git a/pkg/database/authorized_app_test.go b/pkg/database/authorized_app_test.go index ab95802c8..47d878983 100644 --- a/pkg/database/authorized_app_test.go +++ b/pkg/database/authorized_app_test.go @@ -21,7 +21,6 @@ import ( "testing" "time" - "github.com/google/exposure-notifications-server/pkg/timeutils" "github.com/google/exposure-notifications-verification-server/pkg/pagination" "github.com/jinzhu/gorm" ) @@ -164,13 +163,11 @@ func TestAuthorizedApp_Stats(t *testing.T) { // Ensure graph is contiguous. { - stop := timeutils.Midnight(time.Now().UTC()) - start := stop.Add(6 * -24 * time.Hour) - stats, err := authorizedApp.Stats(db, start, stop) + stats, err := authorizedApp.Stats(db) if err != nil { t.Fatal(err) } - if got, want := len(stats), 7; got != want { + if got, want := len(stats), 30; got < want { t.Errorf("expected stats for %d days, got %d", want, got) } } diff --git a/pkg/database/realm.go b/pkg/database/realm.go index 403986202..08cce5d21 100644 --- a/pkg/database/realm.go +++ b/pkg/database/realm.go @@ -1443,7 +1443,10 @@ func (r *Realm) Stats(db *Database) (RealmStats, error) { d.date AS date, $1 AS realm_id, COALESCE(s.codes_issued, 0) AS codes_issued, - COALESCE(s.codes_claimed, 0) AS codes_claimed + COALESCE(s.codes_claimed, 0) AS codes_claimed, + COALESCE(s.codes_invalid, 0) AS codes_invalid, + COALESCE(s.tokens_claimed, 0) AS tokens_claimed, + COALESCE(s.tokens_invalid, 0) AS tokens_invalid FROM ( SELECT date::date FROM generate_series($2, $3, '1 day'::interval) date ) d diff --git a/pkg/database/realm_stats.go b/pkg/database/realm_stats.go index 5833e3d2a..e0984ba1c 100644 --- a/pkg/database/realm_stats.go +++ b/pkg/database/realm_stats.go @@ -61,7 +61,11 @@ func (s RealmStats) MarshalCSV() ([]byte, error) { var b bytes.Buffer w := csv.NewWriter(&b) - if err := w.Write([]string{"date", "codes_issued", "codes_claimed"}); err != nil { + if err := w.Write([]string{ + "date", + "codes_issued", "codes_claimed", "codes_invalid", + "tokens_claimed", "tokens_invalid", + }); err != nil { return nil, fmt.Errorf("failed to write CSV header: %w", err) } @@ -70,6 +74,9 @@ func (s RealmStats) MarshalCSV() ([]byte, error) { stat.Date.Format(project.RFC3339Date), strconv.FormatUint(uint64(stat.CodesIssued), 10), strconv.FormatUint(uint64(stat.CodesClaimed), 10), + strconv.FormatUint(uint64(stat.CodesInvalid), 10), + strconv.FormatUint(uint64(stat.TokensClaimed), 10), + strconv.FormatUint(uint64(stat.TokensInvalid), 10), }); err != nil { return nil, fmt.Errorf("failed to write CSV entry %d: %w", i, err) } @@ -94,8 +101,11 @@ type jsonRealmStatStats struct { } type jsonRealmStatStatsData struct { - CodesIssued uint `json:"codes_issued"` - CodesClaimed uint `json:"codes_claimed"` + CodesIssued uint `json:"codes_issued"` + CodesClaimed uint `json:"codes_claimed"` + CodesInvalid uint `json:"codes_invalid"` + TokensClaimed uint `json:"tokens_claimed"` + TokensInvalid uint `json:"tokens_invalid"` } // MarshalJSON is a custom JSON marshaller. @@ -110,8 +120,11 @@ func (s RealmStats) MarshalJSON() ([]byte, error) { stats = append(stats, &jsonRealmStatStats{ Date: stat.Date, Data: &jsonRealmStatStatsData{ - CodesIssued: stat.CodesIssued, - CodesClaimed: stat.CodesClaimed, + CodesIssued: stat.CodesIssued, + CodesClaimed: stat.CodesClaimed, + CodesInvalid: stat.CodesInvalid, + TokensClaimed: stat.TokensClaimed, + TokensInvalid: stat.TokensInvalid, }, }) } @@ -144,10 +157,13 @@ func (s *RealmStats) UnmarshalJSON(b []byte) error { for _, stat := range result.Stats { *s = append(*s, &RealmStat{ - Date: stat.Date, - RealmID: result.RealmID, - CodesIssued: stat.Data.CodesIssued, - CodesClaimed: stat.Data.CodesClaimed, + Date: stat.Date, + RealmID: result.RealmID, + CodesIssued: stat.Data.CodesIssued, + CodesClaimed: stat.Data.CodesClaimed, + CodesInvalid: stat.Data.CodesInvalid, + TokensClaimed: stat.Data.TokensClaimed, + TokensInvalid: stat.Data.TokensInvalid, }) } diff --git a/pkg/database/realm_stats_test.go b/pkg/database/realm_stats_test.go index b1ab411f2..9bd05c277 100644 --- a/pkg/database/realm_stats_test.go +++ b/pkg/database/realm_stats_test.go @@ -38,42 +38,54 @@ func TestRealmStats_MarshalCSV(t *testing.T) { name: "single", stats: []*RealmStat{ { - Date: time.Date(2020, 2, 3, 0, 0, 0, 0, time.UTC), - RealmID: 1, - CodesIssued: 10, - CodesClaimed: 9, + Date: time.Date(2020, 2, 3, 0, 0, 0, 0, time.UTC), + RealmID: 1, + CodesIssued: 10, + CodesClaimed: 9, + CodesInvalid: 1, + TokensClaimed: 7, + TokensInvalid: 2, }, }, - exp: `date,codes_issued,codes_claimed -2020-02-03,10,9 + exp: `date,codes_issued,codes_claimed,codes_invalid,tokens_claimed,tokens_invalid +2020-02-03,10,9,1,7,2 `, }, { name: "multi", stats: []*RealmStat{ { - Date: time.Date(2020, 2, 3, 0, 0, 0, 0, time.UTC), - RealmID: 1, - CodesIssued: 10, - CodesClaimed: 9, + Date: time.Date(2020, 2, 3, 0, 0, 0, 0, time.UTC), + RealmID: 1, + CodesIssued: 10, + CodesClaimed: 9, + CodesInvalid: 1, + TokensClaimed: 7, + TokensInvalid: 2, }, { - Date: time.Date(2020, 2, 4, 0, 0, 0, 0, time.UTC), - RealmID: 1, - CodesIssued: 45, - CodesClaimed: 30, + Date: time.Date(2020, 2, 4, 0, 0, 0, 0, time.UTC), + RealmID: 1, + CodesIssued: 45, + CodesClaimed: 30, + CodesInvalid: 29, + TokensClaimed: 27, + TokensInvalid: 2, }, { - Date: time.Date(2020, 2, 5, 0, 0, 0, 0, time.UTC), - RealmID: 1, - CodesIssued: 15, - CodesClaimed: 2, + Date: time.Date(2020, 2, 5, 0, 0, 0, 0, time.UTC), + RealmID: 1, + CodesIssued: 15, + CodesClaimed: 2, + CodesInvalid: 0, + TokensClaimed: 2, + TokensInvalid: 0, }, }, - exp: `date,codes_issued,codes_claimed -2020-02-03,10,9 -2020-02-04,45,30 -2020-02-05,15,2 + exp: `date,codes_issued,codes_claimed,codes_invalid,tokens_claimed,tokens_invalid +2020-02-03,10,9,1,7,2 +2020-02-04,45,30,29,27,2 +2020-02-05,15,2,0,2,0 `, }, }