Skip to content

Commit

Permalink
Wrap the server with the Prometheus so we get metrics + add an e2e te…
Browse files Browse the repository at this point in the history
…st for it.

I wrapped it in a job, since otherwise it will be trickier to get access to the
metrics from outside the kind cluster.

Fix the dangling pointer from sigstore#262 and fix a tiny typo, though I kind of do
like exposing a singingCert that might come handy in karaoke?

Signed-off-by: Ville Aikas <vaikas@chainguard.dev>
  • Loading branch information
vaikas committed Dec 8, 2021
1 parent f4746cc commit e2e1bd0
Show file tree
Hide file tree
Showing 7 changed files with 156 additions and 17 deletions.
19 changes: 19 additions & 0 deletions .github/workflows/verify-k8s.yml
Original file line number Diff line number Diff line change
Expand Up @@ -225,6 +225,25 @@ jobs:
kubectl wait --for=condition=Complete --timeout=90s job/check-oidc
- name: Validate prometheus metrics exported and look correct
run: |
cat <<EOF | kubectl apply -f -
apiVersion: batch/v1
kind: Job
metadata:
name: check-prometheus-metrics
spec:
template:
spec:
restartPolicy: Never
automountServiceAccountToken: false
containers:
- name: check-metrics
image: ko://github.com/sigstore/fulcio/test/prometheus/
EOF
kubectl wait --for=condition=Complete --timeout=90s job/check-prometheus-metrics
- name: Collect logs
if: ${{ always() }}
run: |
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ There are other targets available in the [`Makefile`](Makefile), check it out.

## API

The API is defined via OpenAPI, defined [here](openapi.yaml).
The API is defined [here](./pkg/api/client.go).

## Transparency

Expand Down
30 changes: 17 additions & 13 deletions cmd/app/serve.go
Original file line number Diff line number Diff line change
Expand Up @@ -100,19 +100,23 @@ var serveCmd = &cobra.Command{
}

decorateHandler := func(h http.Handler) http.Handler {
return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
ctx := r.Context()

// For each request, infuse context with our snapshot of the FulcioConfig.
// TODO(mattmoor): Consider periodically (every minute?) refreshing the ConfigMap
// from disk, so that we don't need to cycle pods to pick up config updates.
// Alternately we could take advantage of Knative's configmap watcher.
ctx = config.With(ctx, cfg)
ctx = api.WithCA(ctx, baseca)
ctx = api.WithCTLogURL(ctx, viper.GetString("ct-log-url"))

h.ServeHTTP(rw, r.WithContext(ctx))
})
// Wrap the inner func with instrumentation to get latencies
// that get partitioned by 'code' and 'method'.
return promhttp.InstrumentHandlerDuration(
api.MetricLatency,
http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
ctx := r.Context()

// For each request, infuse context with our snapshot of the FulcioConfig.
// TODO(mattmoor): Consider periodically (every minute?) refreshing the ConfigMap
// from disk, so that we don't need to cycle pods to pick up config updates.
// Alternately we could take advantage of Knative's configmap watcher.
ctx = config.With(ctx, cfg)
ctx = api.WithCA(ctx, baseca)
ctx = api.WithCTLogURL(ctx, viper.GetString("ct-log-url"))

h.ServeHTTP(rw, r.WithContext(ctx))
}))
}

prom := http.Server{
Expand Down
3 changes: 2 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@ require (
github.com/mitchellh/mapstructure v1.4.3 // indirect
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.11.0
github.com/prometheus/common v0.29.0 // indirect
github.com/prometheus/client_model v0.2.0
github.com/prometheus/common v0.29.0
github.com/prometheus/procfs v0.7.0 // indirect
github.com/sigstore/sigstore v1.0.1
github.com/spf13/cobra v1.2.1
Expand Down
2 changes: 1 addition & 1 deletion pkg/api/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ const SigstorePublicServerURL = "https://fulcio.sigstore.dev"

// Client is the interface for accessing the Fulcio API.
type Client interface {
// SigningCert sends the provided CertificateRequest to the /api/v1/singingCert
// SigningCert sends the provided CertificateRequest to the /api/v1/signingCert
// endpoint of a Fulcio API, authenticated with the provided bearer token.
SigningCert(cr CertificateRequest, token string) (*CertificateResponse, error)
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/api/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,5 +29,5 @@ var (
MetricLatency = promauto.NewHistogramVec(prometheus.HistogramOpts{
Name: "fulcio_api_latency",
Help: "API Latency on calls",
}, []string{"path", "code"})
}, []string{"code", "method"})
)
115 changes: 115 additions & 0 deletions test/prometheus/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
// Copyright 2021 The Sigstore Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package main

import (
"flag"
"fmt"
"log"
"net/http"

dto "github.com/prometheus/client_model/go"
"github.com/prometheus/common/expfmt"
)

func fatal(err error) {
if err != nil {
log.Fatalln(err)
}
}

func parseMF(url string) (map[string]*dto.MetricFamily, error) {
resp, err := http.Get(url)
if err != nil {
return nil, err
}
defer resp.Body.Close()

var parser expfmt.TextParser
return parser.TextToMetricFamilies(resp.Body)
}

func main() {
f := flag.String("url", "http://fulcio-server.fulcio-dev.svc:2112/metrics", "set url to fetch metrics from")
flag.Parse()

mf, err := parseMF(*f)
fatal(err)

// Just grab the api_latency metric, make sure it's a histogram
// and just make sure there is at least one 200, and no errors there.
latency, ok := mf["fulcio_api_latency"]
if !ok || latency == nil {
log.Fatal("Did not get fulcio_api_latency metric")
}
if err := checkLatency(latency); err != nil {
log.Fatalf("fulcio_api_latency metric failed: %s", err)
}

// Then make sure the cert counter went up.
certCount, ok := mf["fulcio_new_certs"]
if !ok || certCount == nil {
log.Fatal("Did not get fulcio_new_certs metric")
}
if err := checkCertCount(certCount); err != nil {
log.Fatalf("fulcio_new_certs metric failed: %s", err)
}
}

// Make sure latency is a Histogram, and it has a POST with a 200.
func checkLatency(latency *dto.MetricFamily) error {
if *latency.Type != *dto.MetricType_HISTOGRAM.Enum() {
return fmt.Errorf("Wrong type, wanted %+v, got: %+v", dto.MetricType_HISTOGRAM.Enum(), latency.Type)
}
if len(latency.Metric) != 1 {
return fmt.Errorf("Got multiple entries, or none for metric, wanted one, got: %+v", latency.Metric)
}
// Make sure there's a 'post' and it's a 200.
var code string
var method string
for _, value := range latency.Metric[0].Label {
if *value.Name == "code" {
code = *value.Value
}
if *value.Name == "method" {
method = *value.Value
}
}
if code != "200" {
return fmt.Errorf("unexpected code, wanted 200, got %s", code)
}
if method != "post" {
return fmt.Errorf("unexpected method, wanted post, got %s", method)
}

if *latency.Metric[0].Histogram.SampleCount != 1 {
return fmt.Errorf("Unexpected samplecount, wanted 1, got %d", *latency.Metric[0].Histogram.SampleCount)
}
return nil
}

func checkCertCount(certCount *dto.MetricFamily) error {
if *certCount.Type != *dto.MetricType_COUNTER.Enum() {
return fmt.Errorf("Wrong type, wanted %+v, got: %+v", dto.MetricType_COUNTER.Enum(), certCount.Type)
}
if len(certCount.Metric) != 1 {
return fmt.Errorf("Got multiple entries, or none for metric, wanted one, got: %+v", certCount.Metric)
}
if *certCount.Metric[0].Counter.Value < 1 {
return fmt.Errorf("Got incorrect cert count, wanted one, got: %f", *certCount.Metric[0].Counter.Value)
}
return nil
}

0 comments on commit e2e1bd0

Please sign in to comment.