diff --git a/.circleci/config.yml b/.circleci/config.yml index 2fc9724..4789b81 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -66,10 +66,12 @@ jobs: k3d create --image rancher/k3s:v1.0.0 -t=0 export KUBECONFIG="$(k3d get-kubeconfig)" k3d import-images patoarvizu/vault-dynamic-configuration-operator:latest + kubectl apply -f https://raw.githubusercontent.com/patoarvizu/common-manifests/master/prometheus-operator/crds.yaml kubectl apply -f https://raw.githubusercontent.com/patoarvizu/common-manifests/master/vault/vault-operator.yaml kubectl apply -f https://raw.githubusercontent.com/patoarvizu/common-manifests/master/vault/vault-cluster-kubernetes-and-db.yaml kubectl apply -f test/manifests/namespaces/test.yaml kubectl apply -f deploy/ + operator-sdk test local ./test/e2e/ --namespace "vault" --go-test-flags '-v -run TestMonitoringObjectsCreated' operator-sdk test local ./test/e2e/ --namespace "vault" --go-test-flags '-v -run TestSingleNamespace' kubectl -n vault patch deployment vault-dynamic-configuration-operator --type='json' -p='[{"op":"add", "path":"/spec/template/spec/containers/0/command/-", "value":"--bound-roles-to-all-namespaces"}]' operator-sdk test local ./test/e2e/ --namespace "vault" --go-test-flags '-v -run TestAllNamespaces' diff --git a/README.md b/README.md index b5a8baa..12bd5a8 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ - [Operator command-line flags](#operator-command-line-flags) - [Operator permissions](#operator-permissions) - [Vault agent sidecar auto-inject mutating webhook](#vault-agent-sidecar-auto-inject-mutating-webhook) + - [Monitoring](#monitoring) - [For security nerds](#for-security-nerds) - [Docker images are signed and published to Docker Hub's Notary server](#docker-images-are-signed-and-published-to-docker-hubs-notary-server) - [Docker images are labeled with Git and GPG metadata](#docker-images-are-labeled-with-git-and-gpg-metadata) @@ -69,6 +70,10 @@ Since the operator is **not** operating on the Vault cluster directly, it doesn' The [`vault-agent-auto-inject-webhook`](https://github.com/patoarvizu/vault-agent-auto-inject-webhook) is a companion component to this operator. It can be deployed as a Kubernetes [Mutating Webhook](https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/) to automatically inject a Vault agent sidecar into `Pod`s, using annotations. Read the component's documentation on how to deploy it. +## Monitoring + +If your Kubernetes cluster is running the Prometheus [operator](https://github.com/coreos/prometheus-operator), this operator will automatically create an additional `Service` called `vault-dynamic-configuration-operator-metrics` and a corresponding `ServiceMonitor` of the same name. This monitor will scrape the operator for metrics on two different ports. Port 8383 will post general metrics about the running process, while port 8686 will post metrics about the custom resources managed by the operator. More information can be found on the Operator SDK [website](https://sdk.operatorframework.io/docs/golang/monitoring/prometheus/). + ## For security nerds ### Docker images are signed and published to Docker Hub's Notary server diff --git a/cmd/manager/main.go b/cmd/manager/main.go index d09f55f..9948801 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -126,7 +126,11 @@ func main() { } services := []*v1.Service{service} - _, err = metrics.CreateServiceMonitors(cfg, namespace, services) + operatorNs, err := k8sutil.GetOperatorNamespace() + if err != nil { + log.Info("Could not get operator namespace", "error", err.Error()) + } + _, err = metrics.CreateServiceMonitors(cfg, operatorNs, services) if err != nil { log.Info("Could not create ServiceMonitor object", "error", err.Error()) if err == metrics.ErrServiceMonitorNotPresent { diff --git a/deploy/cluster_role.yaml b/deploy/cluster_role.yaml index 2cff299..3e8336d 100644 --- a/deploy/cluster_role.yaml +++ b/deploy/cluster_role.yaml @@ -35,4 +35,41 @@ rules: - get - list - watch - - update \ No newline at end of file + - update +- apiGroups: + - monitoring.coreos.com + resources: + - servicemonitors + verbs: + - get + - create +- apiGroups: + - apps + resourceNames: + - vault-dynamic-configuration-operator + resources: + - deployments/finalizers + verbs: + - update +- apiGroups: + - apps + resources: + - replicasets + - deployments + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - services + - services/finalizers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch \ No newline at end of file diff --git a/go.sum b/go.sum index 2790bc2..a690231 100644 --- a/go.sum +++ b/go.sum @@ -1000,6 +1000,7 @@ google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610 h1:Ygq9/SRJX9+dU0WCIICM8RkWvDw03lvB77hrhJnpxfU= google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20191028173616-919d9bdd9fe6 h1:UXl+Zk3jqqcbEVV7ace5lrt4YdA4tXiz3f/KbmD29Vo= google.golang.org/genproto v0.0.0-20191028173616-919d9bdd9fe6/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= @@ -1010,6 +1011,7 @@ google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.24.0 h1:vb/1TCsVn3DcJlQ0Gs1yB1pKI6Do2/QNwxdKqmc/b0s= google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= diff --git a/test/e2e/operator_test.go b/test/e2e/operator_test.go index 57f52ec..5b654d7 100644 --- a/test/e2e/operator_test.go +++ b/test/e2e/operator_test.go @@ -1,11 +1,83 @@ package e2e import ( + "context" "testing" + "time" + monitoringv1 "github.com/coreos/prometheus-operator/pkg/apis/monitoring/v1" framework "github.com/operator-framework/operator-sdk/pkg/test" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/wait" + dynclient "sigs.k8s.io/controller-runtime/pkg/client" ) +func TestMonitoringObjectsCreated(t *testing.T) { + ctx := framework.NewTestCtx(t) + defer ctx.Cleanup() + setup(t, ctx) + metricsService := &v1.Service{} + err := wait.Poll(time.Second*2, time.Second*60, func() (done bool, err error) { + err = framework.Global.Client.Get(context.TODO(), dynclient.ObjectKey{Namespace: "vault", Name: "vault-dynamic-configuration-operator-metrics"}, metricsService) + if err != nil { + return false, err + } + return true, nil + }) + if err != nil { + t.Fatal("Could not get metrics Service") + } + httpMetricsPortFound := false + crMetricsPortFound := false + for _, p := range metricsService.Spec.Ports { + if p.Name == "http-metrics" && p.Port == 8383 { + httpMetricsPortFound = true + continue + } + if p.Name == "cr-metrics" && p.Port == 8686 { + crMetricsPortFound = true + continue + } + } + if !httpMetricsPortFound { + t.Fatal("Service vault-dynamic-configuration-operator-metrics doesn't have http-metrics port 8383") + } + if !crMetricsPortFound { + t.Fatal("Service vault-dynamic-configuration-operator-metrics doesn't have cr-metrics port 8686") + } + + framework.Global.Scheme.AddKnownTypes(monitoringv1.SchemeGroupVersion, &monitoringv1.ServiceMonitor{}) + serviceMonitor := &monitoringv1.ServiceMonitor{} + err = wait.Poll(time.Second*2, time.Second*60, func() (done bool, err error) { + err = framework.Global.Client.Client.Get(context.TODO(), dynclient.ObjectKey{Namespace: "vault", Name: "vault-dynamic-configuration-operator-metrics"}, serviceMonitor) + if err != nil { + return false, err + } + return true, nil + }) + if err != nil { + t.Fatal("Could not find metrics ServiceMonitor") + } + httpMetricsEndpointFound := false + crMetricsEndpointFound := false + for _, e := range serviceMonitor.Spec.Endpoints { + if e.Port == "http-metrics" { + httpMetricsEndpointFound = true + continue + } + if e.Port == "cr-metrics" { + crMetricsEndpointFound = true + continue + } + } + if !httpMetricsEndpointFound { + t.Error("ServiceMonitor vault-dynamic-configuration-operator-metrics doesn't have endpoint http-metrics") + } + if !crMetricsEndpointFound { + t.Error("ServiceMonitor vault-dynamic-configuration-operator-metrics doesn't have endpoint cr-metrics") + } +} + func TestSingleNamespaceServiceAccountAnnotation(t *testing.T) { ctx := framework.NewTestCtx(t) defer ctx.Cleanup()