rget: server: add metrics endpoint

Add a metrics endpoint on :2112.

This is needed to get a count of the projects and releases that are
popular with users.

go.mod

@@ -37,6 +37,7 @@ require (
 	github.com/nmrshll/oauth2-noserver v0.0.0-20190221200101-9bf017bef639
 	github.com/nmrshll/rndm-go v0.0.0-20170430161430-8da3024e53de // indirect
 	github.com/palantir/stacktrace v0.0.0-20161112013806-78658fd2d177 // indirect
+	github.com/prometheus/client_golang v1.0.0
 	github.com/rogpeppe/fastuuid v1.2.0 // indirect
 	github.com/shurcooL/go v0.0.0-20190704215121-7189cc372560 // indirect
 	github.com/skratchdot/open-golang v0.0.0-20190402232053-79abb63cd66e // indirect

rget/cmd/server.go

@@ -22,6 +22,9 @@ import (
 	"net/http"
 	"os"
 
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/spf13/cobra"
 	githttp "gopkg.in/src-d/go-git.v4/plumbing/transport/http"
 
@@ -44,10 +47,20 @@ func init() {
 	rootCmd.AddCommand(serverCmd)
 }
 
-type sumRepo gitcache.GitCache
+type rgetServer struct {
+	*gitcache.GitCache
+	projReqs *prometheus.CounterVec
+}
 
-func (r sumRepo) handler(resp http.ResponseWriter, req *http.Request) {
+func (r rgetServer) handler(resp http.ResponseWriter, req *http.Request) {
 	if req.Method != "POST" {
+		domain, err := rgetwellknown.TrimDigestDomain(req.Host)
+		if err != nil {
+			fmt.Printf("request for unknown host %v unable to parse: %v\n", req.Host, err)
+		}
+		if len(domain) > 0 {
+			r.projReqs.WithLabelValues(req.Method, domain).Inc()
+		}
 		http.Error(resp, "only POST is supported", http.StatusBadRequest)
 		return
 	}
@@ -70,6 +83,8 @@ func (r sumRepo) handler(resp http.ResponseWriter, req *http.Request) {
 		return
 	}
 
+	r.projReqs.WithLabelValues(req.Method, domain).Inc()
+
 	// Step 1: Download the SHA256SUMS that is correct for the URL
 	response, err := http.Get(sumsURL)
 	var sha256file []byte
@@ -89,9 +104,7 @@ func (r sumRepo) handler(resp http.ResponseWriter, req *http.Request) {
 	sums := rgethash.FromSHA256SumFile(string(sha256file))
 
 	// Step 2: Save the file contents to the git repo by domain
-	gc := gitcache.GitCache(r)
-
-	_, err = gc.Get(context.Background(), sums.Domain())
+	_, err = r.GitCache.Get(context.Background(), sums.Domain())
 	if err == nil {
 		// TODO(philips): add rate limiting and DDoS protections here
 		fmt.Printf("cache hit: %v\n", sumsURL)
@@ -101,7 +114,7 @@ func (r sumRepo) handler(resp http.ResponseWriter, req *http.Request) {
 
 	// Step 3. Create the Certificate object for the domain and save that as well
 	ctdomain := sums.Domain() + "." + domain
-	err = gc.Put(context.Background(), ctdomain, sha256file)
+	err = r.GitCache.Put(context.Background(), ctdomain, sha256file)
 	if err != nil {
 		fmt.Printf("git put error: %v", err)
 		http.Error(resp, "internal service error", http.StatusInternalServerError)
@@ -137,7 +150,17 @@ func server(cmd *cobra.Command, args []string) {
 	if err != nil {
 		panic(err)
 	}
-	http.HandleFunc("/", sumRepo(*pubgc).handler)
+	rr := promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "rget_project_requests",
+		Help: "Total number of requests for a particular project",
+	}, []string{"method", "project"})
+
+	rs := rgetServer{
+		GitCache: pubgc,
+		projReqs: rr,
+	}
+
+	http.HandleFunc("/", rs.handler)
 
 	privgc, err := gitcache.NewGitCache(privgit, &auth, "private")
 	if err != nil {
@@ -169,5 +192,17 @@ func server(cmd *cobra.Command, args []string) {
 		}
 	}()
 
+	ms := &http.Server{
+		Addr:    ":2112",
+		Handler: promhttp.Handler(),
+	}
+
+	go func() {
+		err := ms.ListenAndServe()
+		if err != nil {
+			panic(err)
+		}
+	}()
+
 	log.Fatal(http.ListenAndServe(":http", nil))
 }

rgetwellknown/wellknown.go

@@ -99,6 +99,26 @@ func SumPrefix(target string) (string, error) {
 	return match["sumPrefix"], nil
 }
 
+// TrimDigest removes the two 16 digit hex subdomains and the record.merklecounty.com
+// parts to make a domain slug that can be used for project tracking
+func TrimDigestDomain(domain string) (string, error) {
+	if !strings.HasSuffix(domain, "."+PublicServiceHost) {
+		return "", errors.New("incorrect domain suffix")
+	}
+	domain = strings.TrimSuffix(domain, "."+PublicServiceHost)
+
+	parts := strings.Split(domain, ".")
+	if len(parts) < 3 {
+		return "", errors.New("domain too short")
+	}
+
+	if len(parts[0]) != 32 && len(parts[1]) != 32 {
+		return "", errors.New("digest part too short")
+	}
+
+	return strings.Join(parts[2:], "."), nil
+}
+
 var errUnknownSite = errors.New("no domain translation logic for this URL")
 
 // expand rewrites s to replace {k} with match[k] for each key k in match.

rgetwellknown/wellknown_test.go

@@ -51,3 +51,38 @@ func TestSumPrefix(t *testing.T) {
 		}
 	}
 }
+
+func TestTrimDigestDomain(t *testing.T) {
+	testCases := []struct {
+		domain  string
+		want    string
+		wantErr bool
+	}{
+		{"2fcd82bbae7bcf7c0b0c5a2f91d3dd93.1e7c7be8587808ee85b347412ffa7514.v0-0-4.rget.merklecounty.github.com.recorder.merklecounty.com", "v0-0-4.rget.merklecounty.github.com", false},
+		// digest too short
+		{"1.2.v0-0-4.rget.merklecounty.github.com.recorder.merklecounty.com", "", true},
+		// domain too short
+		{"2fcd82bbae7bcf7c0b0c5a2f91d3dd93.1e7c7be8587808ee85b347412ffa7514.recorder.merklecounty.com", "", true},
+		// wrong domain
+		{"2fcd82bbae7bcf7c0b0c5a2f91d3dd93.1e7c7be8587808ee85b347412ffa7514.example.com", "", true},
+	}
+
+	for ti, tt := range testCases {
+		dd, err := TrimDigestDomain(tt.domain)
+		if !tt.wantErr && err != nil {
+			t.Errorf("%d: error from TrimDigestDomain %v: %v", ti, tt.domain, err)
+		}
+
+		if tt.wantErr {
+			if err == nil {
+				t.Errorf("%d: wanted err got nil", ti)
+			}
+			continue
+		}
+
+		if dd != tt.want {
+			t.Errorf("%d: domain %v != %v", ti, dd, tt.want)
+		}
+	}
+
+}