Skip to content

Commit

Permalink
Completely redo stats system as there was an issue where it loses all…
Browse files Browse the repository at this point in the history
… data on restart
  • Loading branch information
JasonLovesDoggo committed Jun 6, 2024
1 parent 35530c7 commit e90e752
Show file tree
Hide file tree
Showing 7 changed files with 196 additions and 24 deletions.
48 changes: 35 additions & 13 deletions docs/index.html
Original file line number Diff line number Diff line change
Expand Up @@ -236,7 +236,7 @@ <h1 id="abacus">Abacus</h1>

<p>
So far, <code data-stat-id="keys">...</code> keys have been created and there have been <code
data-stat-id="commands_processed">...</code> requests served. Abacus is currently on version <code
data-stat-id="command_total">...</code> requests served. <code data-stat-id="hits">...</code> of those have been hits and <code data-stat-id="gets">...</code> have been gets. Abacus is currently on version <code
data-stat-id="version">...</code>.<br>
This page has been visited <code id="visits">...</code> times.
</p>
Expand Down Expand Up @@ -554,13 +554,21 @@ <h3 class="endpoint">/stats</h3>
<pre class="success">
GET /stats/
⇒ 200 {
"commands_processed": "14911", // total requests
"db_uptime": "527154", // db uptime in seconds
"db_version": "6.0.16", // DB version
"expired_keys": "438", // N expired keys
"key_misses": "468", // Requests that returned 404 (key not found)
"keys": "75", // N active keys
"version": "1.0.0" // Abacus's current version
"commands": {
"create": 20394, // total number of /create's
"get": 403232, // total number of /get's
"hit": 703232, // total number of /hit's
"total": 2003058 // total number of requests served
},
"db_uptime": "5558", // database uptime in seconds
"db_version": "6.0.16", // database version
"expired_keys__since_restart": "130", // number of keys expired since db's last restart
"key_misses__since_restart": "205", // number of keys not found since db's last restart
"total_keys": 87904, // total number of keys created
"version": "1.1.0", // Abacus's version
"shard": "boujee-coorgi", // Handler shard
"uptime": "1h23m45s" // shard uptime

}
</pre>

Expand Down Expand Up @@ -592,14 +600,28 @@ <h1 style="all: unset">
function stats() {
request("https://abacus.jasoncameron.dev/stats", function (obj) {
if (obj) {
const commands = parseInt(obj["commands_processed"]);
const keys = parseInt(obj["keys"]) + parseInt(obj["expired_keys"]);
const commands = obj["commands"];
const keys = obj["total_keys"];

Array.from(document.querySelectorAll(`*[data-stat-id="version"]`)).map(o => o.innerText = obj["version"]);
Array.from(document.querySelectorAll(`*[data-stat-id="commands_processed"]`)).map(o => {
o.innerText = formatter.format(commands)
o.title = `${commands.toLocaleString()} requests served`


Array.from(document.querySelectorAll(`*[data-stat-id="command_total"]`)).map(o => {
o.innerText = formatter.format(commands["total"])
o.title = `${commands["total"].toLocaleString()} requests served`
});

Array.from(document.querySelectorAll(`*[data-stat-id="hits"]`)).map(o => {
o.innerText = formatter.format(commands["hit"])
o.title = `${commands["hit"].toLocaleString()} hits`
});

Array.from(document.querySelectorAll(`*[data-stat-id="gets"]`)).map(o => {
o.innerText = formatter.format(commands["get"])
o.title = `${commands["get"].toLocaleString()} gets`
});


Array.from(document.querySelectorAll(`*[data-stat-id="keys"]`)).map(o => {
o.innerText = formatter.format(keys)
o.title = `${keys.toLocaleString()} keys created`
Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ require (
)

require (
github.com/anandvarma/namegen v1.0.1 // indirect
github.com/bytedance/sonic v1.11.2 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
github.com/JGLTechnologies/gin-rate-limit v1.5.4 h1:1hIaXIdGM9MZFZlXgjWJLpxaK0WHEa5MeloK49nmQsc=
github.com/JGLTechnologies/gin-rate-limit v1.5.4/go.mod h1:mGEhNzlHEg/Tk+KH/mKylZLTfDjACnx7MVYaAlj07eU=
github.com/anandvarma/namegen v1.0.1 h1:ACLz+jxBaH2YJcsgxmmYvNDz74ykFmfArsSH0jo++e4=
github.com/anandvarma/namegen v1.0.1/go.mod h1:MFyILur9tG8PxaCXGZVr/2BOnHtRIgxYejYFZdWLxr0=
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
Expand Down
58 changes: 52 additions & 6 deletions main.go
Original file line number Diff line number Diff line change
@@ -1,13 +1,20 @@
package main

import (
"context"
"errors"
"fmt"
"log"
"net/http"
"os"
"os/signal"
"strconv"
"strings"
"syscall"
"time"

"github.com/anandvarma/namegen"

"github.com/redis/go-redis/v9"

"github.com/jasonlovesdoggo/abacus/middleware"
Expand All @@ -28,12 +35,14 @@ const (
var (
Client *redis.Client
RateLimitClient *redis.Client
DbNum uint8 = 0 // 0-16
startTime time.Time
DbNum = 0 // 0-16
StartTime time.Time
Shard string
)

func init() {
// Connect to Redis
Shard = namegen.New().Get()
utils.LoadEnv()

if strings.ToLower(os.Getenv("DEBUG")) == "true" {
Expand All @@ -44,7 +53,7 @@ func init() {

ADDR := os.Getenv("REDIS_HOST") + ":" + os.Getenv("REDIS_PORT")
fmt.Println("Listening to redis on: " + ADDR)
DbNum, _ := strconv.Atoi(os.Getenv("REDIS_DB"))
DbNum, _ = strconv.Atoi(os.Getenv("REDIS_DB"))
Client = redis.NewClient(&redis.Options{
Addr: ADDR, // Redis server address
Username: os.Getenv("REDIS_USERNAME"),
Expand All @@ -61,8 +70,12 @@ func init() {

func main() {
// only run the following if .env is present
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()

utils.LoadEnv()
startTime = time.Now()
StartTime = time.Now()
utils.InitializeStats(Client)
// Initialize the Gin router
r := gin.Default()
r.Use(cors.Default())
Expand All @@ -72,6 +85,7 @@ func main() {
}
route := r.Group("")
route.Use(middleware.RateLimit(RateLimitClient))
route.Use(middleware.Stats())

// Define routes
r.NoRoute(func(c *gin.Context) {
Expand All @@ -82,7 +96,7 @@ func main() {
r.StaticFile("/favicon.ico", "./assets/favicon.ico")
route.GET("/healthcheck", func(context *gin.Context) {
context.JSON(http.StatusOK, gin.H{
"status": "ok", "uptime": time.Since(startTime).String()})
"status": "ok", "uptime": time.Since(StartTime).String()})
})
route.GET("/docs", func(context *gin.Context) {
context.Redirect(http.StatusPermanentRedirect, DocsUrl)
Expand Down Expand Up @@ -112,5 +126,37 @@ func main() {
authorized.POST("/update/:namespace/*key", UpdateByView)

// Run the server
_ = r.Run("0.0.0.0:" + os.Getenv("PORT"))

srv := &http.Server{
Addr: ":" + os.Getenv("PORT"),
Handler: r,
}

go func() {
// service connections
if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
log.Fatalf("listen: %s\n", err)
}
}()

// Wait for interrupt signal to gracefully shutdown the server with
// a timeout of 5 seconds.
quit := make(chan os.Signal, 1)
// kill (no param) default send syscall.SIGTERM
// kill -2 is syscall.SIGINT
// kill -9 is syscall. SIGKILL but can"t be catch, so don't need add it
signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)
<-quit
close(utils.ServerClose)

ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := srv.Shutdown(ctx); err != nil {
log.Fatal("Server Shutdown:", err)
}
select {
case <-ctx.Done():
log.Println("timeout of 5 seconds.")
}
log.Println("Server exiting")
}
34 changes: 34 additions & 0 deletions middleware/stats.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
package middleware

import (
"strings"

"github.com/jasonlovesdoggo/abacus/utils"

"github.com/gin-gonic/gin"
)

func formatPath(path string) string {
path = path[1:]
path = strings.Split(path, "/")[0]
return path
}

//func shouldSkip(path string) bool {
// switch path {
//
// case "/favicon.ico", "/docs", "/", "favicon.svg":
// return true
// }
// return false
//}

func Stats() gin.HandlerFunc {
return func(c *gin.Context) {
route := formatPath(c.Request.URL.Path)
utils.Total++
utils.CommonStats[route]++
c.Next()

}
}
30 changes: 25 additions & 5 deletions routes.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"strconv"
"strings"
"sync"
"time"

"github.com/redis/go-redis/v9"

Expand Down Expand Up @@ -362,10 +363,29 @@ func StatsView(c *gin.Context) {
}
}

dbInfo := strings.Split(infoDict["Keyspace"]["db"+strconv.Itoa(int(DbNum))], ",")
keys := strings.Split(dbInfo[0], "=")
c.JSON(http.StatusOK, gin.H{"version": Version, "db_uptime": infoDict["Server"]["uptime_in_seconds"], "db_version": infoDict["Server"]["redis_version"], "expired_keys": infoDict["Stats"]["expired_keys"],
"key_misses": infoDict["Stats"]["keyspace_misses"],
"commands_processed": infoDict["Stats"]["total_commands_processed"], "keys": keys[1]})
total, _ := strconv.Atoi(Client.Get(ctx, "stats:Total").Val())

hits, _ := strconv.Atoi(Client.Get(ctx, "stats:hit").Val())
gets, _ := strconv.Atoi(Client.Get(ctx, "stats:get").Val())

create, _ := strconv.Atoi(Client.Get(ctx, "stats:create").Val())

totalKeys := create + (hits / 8)

c.JSON(http.StatusOK, gin.H{
"version": Version,
"uptime": time.Since(StartTime).String(),
"db_uptime": infoDict["Server"]["uptime_in_seconds"],
"db_version": infoDict["Server"]["redis_version"],
"expired_keys__since_restart": infoDict["Stats"]["expired_keys"],
"key_misses__since_restart": infoDict["Stats"]["keyspace_misses"],
"commands": gin.H{
"total": total,
"get": gets,
"hit": hits,
"create": create,
},
"total_keys": totalKeys,
"shard": Shard,
})
}
47 changes: 47 additions & 0 deletions utils/stats.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
package utils

import (
"context"
"log"
"time"

"github.com/redis/go-redis/v9"
)

var Total int64 = 0

var CommonStats = map[string]int64{}

var ServerClose = make(chan struct{})

func saveStats(client *redis.Client) {
newTotal := Total
Total = 0 // reset the total

newStats := CommonStats
CommonStats = map[string]int64{} // reset the map

client.IncrBy(context.Background(), "stats:Total", newTotal) // Capitalized to avoid conflict with a potential key named "total"
for key, value := range newStats {
client.IncrBy(context.Background(), "stats:"+key, value)
}
}

func InitializeStats(client *redis.Client) {
ticker := time.NewTicker(30 * time.Second)
go func() {
for {
select {
case <-ticker.C:
saveStats(client)

case <-ServerClose:
ticker.Stop()
log.Println("Saving stats... Closing stats goroutine. Goodbye!")
saveStats(client) // save the stats one last time before closing

return
}
}
}()
}

0 comments on commit e90e752

Please sign in to comment.