Merge pull request #27 from patoarvizu/enable_metric_endpoints_creation

Enable metric endpoints creation

.circleci/config.yml

@@ -66,10 +66,12 @@ jobs:
             k3d create --image rancher/k3s:v1.0.0 -t=0
             export KUBECONFIG="$(k3d get-kubeconfig)"
             k3d import-images patoarvizu/vault-dynamic-configuration-operator:latest
+            kubectl apply -f https://raw.githubusercontent.com/patoarvizu/common-manifests/master/prometheus-operator/crds.yaml
             kubectl apply -f https://raw.githubusercontent.com/patoarvizu/common-manifests/master/vault/vault-operator.yaml
             kubectl apply -f https://raw.githubusercontent.com/patoarvizu/common-manifests/master/vault/vault-cluster-kubernetes-and-db.yaml
             kubectl apply -f test/manifests/namespaces/test.yaml
             kubectl apply -f deploy/
+            operator-sdk test local ./test/e2e/ --namespace "vault" --go-test-flags '-v -run TestMonitoringObjectsCreated'
             operator-sdk test local ./test/e2e/ --namespace "vault" --go-test-flags '-v -run TestSingleNamespace'
             kubectl -n vault patch deployment vault-dynamic-configuration-operator --type='json' -p='[{"op":"add", "path":"/spec/template/spec/containers/0/command/-", "value":"--bound-roles-to-all-namespaces"}]'
             operator-sdk test local ./test/e2e/ --namespace "vault" --go-test-flags '-v -run TestAllNamespaces'

README.md

@@ -12,6 +12,7 @@
         - [Operator command-line flags](#operator-command-line-flags)
         - [Operator permissions](#operator-permissions)
     - [Vault agent sidecar auto-inject mutating webhook](#vault-agent-sidecar-auto-inject-mutating-webhook)
+    - [Monitoring](#monitoring)
     - [For security nerds](#for-security-nerds)
         - [Docker images are signed and published to Docker Hub's Notary server](#docker-images-are-signed-and-published-to-docker-hubs-notary-server)
         - [Docker images are labeled with Git and GPG metadata](#docker-images-are-labeled-with-git-and-gpg-metadata)
@@ -69,6 +70,10 @@ Since the operator is **not** operating on the Vault cluster directly, it doesn'
 
 The [`vault-agent-auto-inject-webhook`](https://github.com/patoarvizu/vault-agent-auto-inject-webhook) is a companion component to this operator. It can be deployed as a Kubernetes [Mutating Webhook](https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/) to automatically inject a Vault agent sidecar into `Pod`s, using annotations. Read the component's documentation on how to deploy it.
 
+## Monitoring
+
+If your Kubernetes cluster is running the Prometheus [operator](https://github.com/coreos/prometheus-operator), this operator will automatically create an additional `Service` called `vault-dynamic-configuration-operator-metrics` and a corresponding `ServiceMonitor` of the same name. This monitor will scrape the operator for metrics on two different ports. Port 8383 will post general metrics about the running process, while port 8686 will post metrics about the custom resources managed by the operator. More information can be found on the Operator SDK [website](https://sdk.operatorframework.io/docs/golang/monitoring/prometheus/).
+
 ## For security nerds
 
 ### Docker images are signed and published to Docker Hub's Notary server

cmd/manager/main.go

@@ -126,7 +126,11 @@ func main() {
 	}
 
 	services := []*v1.Service{service}
-	_, err = metrics.CreateServiceMonitors(cfg, namespace, services)
+	operatorNs, err := k8sutil.GetOperatorNamespace()
+	if err != nil {
+		log.Info("Could not get operator namespace", "error", err.Error())
+	}
+	_, err = metrics.CreateServiceMonitors(cfg, operatorNs, services)
 	if err != nil {
 		log.Info("Could not create ServiceMonitor object", "error", err.Error())
 		if err == metrics.ErrServiceMonitorNotPresent {

deploy/cluster_role.yaml

@@ -35,4 +35,41 @@ rules:
   - get
   - list
   - watch
-  - update
+  - update
+- apiGroups:
+  - monitoring.coreos.com
+  resources:
+  - servicemonitors
+  verbs:
+  - get
+  - create
+- apiGroups:
+  - apps
+  resourceNames:
+  - vault-dynamic-configuration-operator
+  resources:
+  - deployments/finalizers
+  verbs:
+  - update
+- apiGroups:
+  - apps
+  resources:
+  - replicasets
+  - deployments
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - services
+  - services/finalizers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch

go.sum

@@ -1000,6 +1000,7 @@ google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRn
 google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610 h1:Ygq9/SRJX9+dU0WCIICM8RkWvDw03lvB77hrhJnpxfU=
 google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20191028173616-919d9bdd9fe6 h1:UXl+Zk3jqqcbEVV7ace5lrt4YdA4tXiz3f/KbmD29Vo=
 google.golang.org/genproto v0.0.0-20191028173616-919d9bdd9fe6/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
 google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
@@ -1010,6 +1011,7 @@ google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.24.0 h1:vb/1TCsVn3DcJlQ0Gs1yB1pKI6Do2/QNwxdKqmc/b0s=
 google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA=
 gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=

test/e2e/operator_test.go

@@ -1,11 +1,83 @@
 package e2e
 
 import (
+	"context"
 	"testing"
+	"time"
 
+	monitoringv1 "github.com/coreos/prometheus-operator/pkg/apis/monitoring/v1"
 	framework "github.com/operator-framework/operator-sdk/pkg/test"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	dynclient "sigs.k8s.io/controller-runtime/pkg/client"
 )
 
+func TestMonitoringObjectsCreated(t *testing.T) {
+	ctx := framework.NewTestCtx(t)
+	defer ctx.Cleanup()
+	setup(t, ctx)
+	metricsService := &v1.Service{}
+	err := wait.Poll(time.Second*2, time.Second*60, func() (done bool, err error) {
+		err = framework.Global.Client.Get(context.TODO(), dynclient.ObjectKey{Namespace: "vault", Name: "vault-dynamic-configuration-operator-metrics"}, metricsService)
+		if err != nil {
+			return false, err
+		}
+		return true, nil
+	})
+	if err != nil {
+		t.Fatal("Could not get metrics Service")
+	}
+	httpMetricsPortFound := false
+	crMetricsPortFound := false
+	for _, p := range metricsService.Spec.Ports {
+		if p.Name == "http-metrics" && p.Port == 8383 {
+			httpMetricsPortFound = true
+			continue
+		}
+		if p.Name == "cr-metrics" && p.Port == 8686 {
+			crMetricsPortFound = true
+			continue
+		}
+	}
+	if !httpMetricsPortFound {
+		t.Fatal("Service vault-dynamic-configuration-operator-metrics doesn't have http-metrics port 8383")
+	}
+	if !crMetricsPortFound {
+		t.Fatal("Service vault-dynamic-configuration-operator-metrics doesn't have cr-metrics port 8686")
+	}
+
+	framework.Global.Scheme.AddKnownTypes(monitoringv1.SchemeGroupVersion, &monitoringv1.ServiceMonitor{})
+	serviceMonitor := &monitoringv1.ServiceMonitor{}
+	err = wait.Poll(time.Second*2, time.Second*60, func() (done bool, err error) {
+		err = framework.Global.Client.Client.Get(context.TODO(), dynclient.ObjectKey{Namespace: "vault", Name: "vault-dynamic-configuration-operator-metrics"}, serviceMonitor)
+		if err != nil {
+			return false, err
+		}
+		return true, nil
+	})
+	if err != nil {
+		t.Fatal("Could not find metrics ServiceMonitor")
+	}
+	httpMetricsEndpointFound := false
+	crMetricsEndpointFound := false
+	for _, e := range serviceMonitor.Spec.Endpoints {
+		if e.Port == "http-metrics" {
+			httpMetricsEndpointFound = true
+			continue
+		}
+		if e.Port == "cr-metrics" {
+			crMetricsEndpointFound = true
+			continue
+		}
+	}
+	if !httpMetricsEndpointFound {
+		t.Error("ServiceMonitor vault-dynamic-configuration-operator-metrics doesn't have endpoint http-metrics")
+	}
+	if !crMetricsEndpointFound {
+		t.Error("ServiceMonitor vault-dynamic-configuration-operator-metrics doesn't have endpoint cr-metrics")
+	}
+}
+
 func TestSingleNamespaceServiceAccountAnnotation(t *testing.T) {
 	ctx := framework.NewTestCtx(t)
 	defer ctx.Cleanup()