From 584cb5f3ce871efb1e90bd5dcebdb9b6c2fe67c6 Mon Sep 17 00:00:00 2001 From: Matheus Moraes Date: Thu, 10 Aug 2023 17:24:21 -0300 Subject: [PATCH] feat: agent mode support (#227) * remove kubeconfig flags from plugins cmd * add default-container annotation on plugin pods * set KUBECONFIG env only if the Cluster has kubeconfigKeyRef * add view permissions to zora-plugins ClusterRole * remove kubeconfigRef from Cluster sample * feat: add agent mode flag in helm chart * fix: operator should have the same permissions as the plugins * chore: remove `agent` parameter from helm chart * chore: update NOTES.txt --- charts/zora/Chart.yaml | 4 +- charts/zora/README.md | 6 +- charts/zora/templates/NOTES.txt | 19 +++-- charts/zora/templates/_helpers.tpl | 8 ++ charts/zora/templates/cluster/cluster.yaml | 33 ++++++++ charts/zora/templates/plugins/marvin.yaml | 2 +- charts/zora/templates/plugins/popeye.yaml | 1 - charts/zora/templates/plugins/rbac.yaml | 80 +++++++++++++------ charts/zora/values.yaml | 7 ++ config/rbac/zora_plugins_role.yaml | 73 ++++++++++++----- config/samples/zora_v1alpha1_cluster.yaml | 4 +- .../samples/zora_v1alpha1_plugin_marvin.yaml | 2 +- .../samples/zora_v1alpha1_plugin_popeye.yaml | 1 - .../zora_v1alpha1_plugin_popeye_all.yaml | 1 - .../controller/zora/clusterscan_controller.go | 20 +++-- pkg/plugins/cronjob.go | 40 ++++++---- 16 files changed, 217 insertions(+), 84 deletions(-) create mode 100644 charts/zora/templates/cluster/cluster.yaml diff --git a/charts/zora/Chart.yaml b/charts/zora/Chart.yaml index b3fc90e5..93420537 100644 --- a/charts/zora/Chart.yaml +++ b/charts/zora/Chart.yaml @@ -17,7 +17,7 @@ name: zora description: Zora scans multiple Kubernetes clusters and reports potential issues. icon: https://zora-docs.undistro.io/assets/logo.png type: application -version: 0.6.2 -appVersion: "v0.6.2" +version: 0.7.0-rc2 +appVersion: "v0.7.0-rc2" sources: - https://github.com/undistro/zora diff --git a/charts/zora/README.md b/charts/zora/README.md index 89e58b60..fa752f5b 100644 --- a/charts/zora/README.md +++ b/charts/zora/README.md @@ -1,6 +1,6 @@ # Zora Helm Chart -![Version: 0.6.2](https://img.shields.io/badge/Version-0.6.2-informational?style=flat-square&color=3CA9DD) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square&color=3CA9DD) ![AppVersion: v0.6.2](https://img.shields.io/badge/AppVersion-v0.6.2-informational?style=flat-square&color=3CA9DD) +![Version: 0.7.0-rc2](https://img.shields.io/badge/Version-0.7.0--rc2-informational?style=flat-square&color=3CA9DD) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square&color=3CA9DD) ![AppVersion: v0.7.0-rc2](https://img.shields.io/badge/AppVersion-v0.7.0--rc2-informational?style=flat-square&color=3CA9DD) Zora scans multiple Kubernetes clusters and reports potential issues. @@ -12,7 +12,7 @@ To install the chart with the release name `zora`: helm repo add undistro https://charts.undistro.io --force-update helm upgrade --install zora undistro/zora \ -n zora-system \ - --version 0.6.2 \ + --version 0.7.0-rc2 \ --create-namespace --wait ``` @@ -54,6 +54,8 @@ The following table lists the configurable parameters of the Zora chart and thei |-----|------|---------|-------------| | nameOverride | string | `""` | String to partially override fullname template with a string (will prepend the release name) | | fullnameOverride | string | `""` | String to fully override fullname template with a string | +| clusterName | string | `""` | Cluster name. Should be set by `kubectl config current-context`. | +| scanSchedule | string | Cron expression for every hour at the current minute + 5 minutes | Cluster scan schedule in Cron format | | saas.workspaceID | string | `""` | Your SaaS workspace ID | | saas.server | string | `"https://zora-dashboard.undistro.io"` | SaaS server URL | | saas.hooks.image.repository | string | `"curlimages/curl"` | SaaS hooks image repository | diff --git a/charts/zora/templates/NOTES.txt b/charts/zora/templates/NOTES.txt index 427a5fe7..8e8fcbec 100644 --- a/charts/zora/templates/NOTES.txt +++ b/charts/zora/templates/NOTES.txt @@ -1,9 +1,16 @@ -1. Connect clusters with `kubectl`: +Thank you for installing {{ .Chart.Name | title }} version {{ .Chart.Version }}. - For in-depth information about how to connect a cluster, visit - https://zora-docs.undistro.io/ +{{ if .Values.clusterName -}} +Cluster `{{ .Values.clusterName }}` is scheduled to be scanned. Check it by running: + kubectl get cluster,clusterscan -o wide -n {{ .Release.Namespace }} -{{- if .Values.saas.workspaceID }} -2. Now you can see your clusters and issues in the SaaS: - {{ .Values.saas.server }} +Once a cluster is successfully scanned, you can check issues by running: + kubectl get clusterissues -n {{ .Release.Namespace }} + +{{ end -}} + +Visit our documentation for in-depth information: https://zora-docs.undistro.io + +{{ if .Values.saas.workspaceID -}} +You can see your clusters and issues in SaaS: {{ .Values.saas.server }} {{- end }} diff --git a/charts/zora/templates/_helpers.tpl b/charts/zora/templates/_helpers.tpl index 7a868f4d..af7792e9 100644 --- a/charts/zora/templates/_helpers.tpl +++ b/charts/zora/templates/_helpers.tpl @@ -82,3 +82,11 @@ Create the name of the service account to use in Operator {{- printf "{\"auths\":{\"%s\":{\"auth\":\"%s\"}}}" .registry (printf "%s:%s" .username .password | b64enc) | b64enc }} {{- end }} {{- end }} + +{{- define "clusterName" }} +{{- regexReplaceAll "\\W+" (required "clusterName is required" .Values.clusterName) "-" }} +{{- end }} + +{{- define "scanSchedule"}} +{{- default (printf "%d * * * *" (add 5 (now | date "04"))) .Values.scanSchedule }} +{{- end }} diff --git a/charts/zora/templates/cluster/cluster.yaml b/charts/zora/templates/cluster/cluster.yaml new file mode 100644 index 00000000..6a24cb8d --- /dev/null +++ b/charts/zora/templates/cluster/cluster.yaml @@ -0,0 +1,33 @@ +# Copyright 2023 Undistro Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +{{ if .Values.clusterName }} +apiVersion: zora.undistro.io/v1alpha1 +kind: Cluster +metadata: + labels: + {{- include "zora.labels" . | nindent 4 }} + name: {{ include "clusterName" . }} +spec: {} +--- +apiVersion: zora.undistro.io/v1alpha1 +kind: ClusterScan +metadata: + labels: + {{- include "zora.labels" . | nindent 4 }} + name: {{ include "clusterName" . }} +spec: + clusterRef: + name: {{ include "clusterName" . }} + schedule: {{ include "scanSchedule" . | quote }} +{{- end }} diff --git a/charts/zora/templates/plugins/marvin.yaml b/charts/zora/templates/plugins/marvin.yaml index c50dff63..157204ea 100644 --- a/charts/zora/templates/plugins/marvin.yaml +++ b/charts/zora/templates/plugins/marvin.yaml @@ -34,7 +34,7 @@ spec: mkdir -p $(CUSTOM_CHECKS_PATH) ls -lh $(CUSTOM_CHECKS_PATH) echo Scanning... - /marvin scan --disable-annotation-skip -f $(CUSTOM_CHECKS_PATH) -o json -v 2 --kubeconfig $(KUBECONFIG) > $(DONE_DIR)/results.json + /marvin scan --disable-annotation-skip -f $(CUSTOM_CHECKS_PATH) -o json -v 2 > $(DONE_DIR)/results.json exitcode=$(echo $?) if [ $exitcode -ne 0 ]; then echo "ERROR" > $(DONE_DIR)/error diff --git a/charts/zora/templates/plugins/popeye.yaml b/charts/zora/templates/plugins/popeye.yaml index 5f583a25..61e77fda 100644 --- a/charts/zora/templates/plugins/popeye.yaml +++ b/charts/zora/templates/plugins/popeye.yaml @@ -45,7 +45,6 @@ spec: POPEYE_REPORT_DIR=$(DONE_DIR) \ /bin/popeye \ -o json \ - --kubeconfig $(KUBECONFIG) \ {{- if .Values.scan.plugins.popeye.skipInternalResources }} -f /tmp/spinach.yml \ {{- end }} diff --git a/charts/zora/templates/plugins/rbac.yaml b/charts/zora/templates/plugins/rbac.yaml index ab068870..375a9666 100644 --- a/charts/zora/templates/plugins/rbac.yaml +++ b/charts/zora/templates/plugins/rbac.yaml @@ -17,31 +17,66 @@ kind: ClusterRole metadata: name: zora-plugins rules: - - apiGroups: - - zora.undistro.io + - apiGroups: [ "zora.undistro.io" ] resources: - clusterissues - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - - apiGroups: - - zora.undistro.io + verbs: [ "create", "delete", "get", "list", "patch", "update", "watch" ] + - apiGroups: [ "zora.undistro.io" ] resources: - clusterissues/status - verbs: - - get - - apiGroups: - - "" + verbs: [ "get" ] + - apiGroups: [ "" ] resources: - configmaps - verbs: - - get - - list + - endpoints + - limitranges + - namespaces + - nodes + - persistentvolumes + - persistentvolumeclaims + - pods + - secrets + - serviceaccounts + - services + verbs: [ "get", "list" ] + - apiGroups: [ "apps" ] + resources: + - daemonsets + - deployments + - statefulsets + - replicasets + verbs: [ "get", "list" ] + - apiGroups: [ "autoscaling" ] + resources: + - horizontalpodautoscalers + verbs: [ "get", "list" ] + - apiGroups: [ "networking.k8s.io" ] + resources: + - ingresses + - networkpolicies + verbs: [ "get", "list" ] + - apiGroups: [ "policy" ] + resources: + - poddisruptionbudgets + - podsecuritypolicies + verbs: [ "get", "list" ] + - apiGroups: [ "rbac.authorization.k8s.io" ] + resources: + - clusterroles + - clusterrolebindings + - roles + - rolebindings + verbs: [ "get", "list" ] + - apiGroups: [ "metrics.k8s.io" ] + resources: + - pods + - nodes + verbs: [ "get", "list" ] + - apiGroups: [ batch ] + resources: + - jobs + - cronjobs + verbs: [ "get", "list" ] --- {{ $crb := (lookup "rbac.authorization.k8s.io/v1" "ClusterRoleBinding" "" "zora-plugins-rolebinding") }} apiVersion: rbac.authorization.k8s.io/v1 @@ -52,13 +87,12 @@ roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: zora-plugins -{{- if $crb }} subjects: + - kind: ServiceAccount + name: {{ include "zora.operatorServiceAccountName" . }} + namespace: {{ .Release.Namespace }} {{- range $s := $crb.subjects }} - kind: {{ $s.kind }} name: {{ $s.name }} namespace: {{ $s.namespace }} {{- end }} -{{- else }} -subjects: [] -{{- end }} diff --git a/charts/zora/values.yaml b/charts/zora/values.yaml index 93e41e8e..c806ce45 100644 --- a/charts/zora/values.yaml +++ b/charts/zora/values.yaml @@ -17,6 +17,13 @@ nameOverride: "" # -- String to fully override fullname template with a string fullnameOverride: "" +# -- Cluster name. Should be set by `kubectl config current-context`. +clusterName: "" + +# -- Cluster scan schedule in Cron format +# @default -- Cron expression for every hour at the current minute + 5 minutes +scanSchedule: "" + saas: # -- Your SaaS workspace ID workspaceID: "" diff --git a/config/rbac/zora_plugins_role.yaml b/config/rbac/zora_plugins_role.yaml index b53e37d3..819e159f 100644 --- a/config/rbac/zora_plugins_role.yaml +++ b/config/rbac/zora_plugins_role.yaml @@ -24,28 +24,63 @@ metadata: app.kubernetes.io/managed-by: kustomize name: zora-plugins rules: - - apiGroups: - - zora.undistro.io + - apiGroups: [ "zora.undistro.io" ] resources: - clusterissues - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - - apiGroups: - - zora.undistro.io + verbs: [ "create", "delete", "get", "list", "patch", "update", "watch" ] + - apiGroups: [ "zora.undistro.io" ] resources: - clusterissues/status - verbs: - - get - - apiGroups: - - "" + verbs: [ "get" ] + - apiGroups: [ "" ] resources: - configmaps - verbs: - - get - - list + - endpoints + - limitranges + - namespaces + - nodes + - persistentvolumes + - persistentvolumeclaims + - pods + - secrets + - serviceaccounts + - services + verbs: [ "get", "list" ] + - apiGroups: [ "apps" ] + resources: + - daemonsets + - deployments + - statefulsets + - replicasets + verbs: [ "get", "list" ] + - apiGroups: [ "autoscaling" ] + resources: + - horizontalpodautoscalers + verbs: [ "get", "list" ] + - apiGroups: [ "networking.k8s.io" ] + resources: + - ingresses + - networkpolicies + verbs: [ "get", "list" ] + - apiGroups: [ "policy" ] + resources: + - poddisruptionbudgets + - podsecuritypolicies + verbs: [ "get", "list" ] + - apiGroups: [ "rbac.authorization.k8s.io" ] + resources: + - clusterroles + - clusterrolebindings + - roles + - rolebindings + verbs: [ "get", "list" ] + - apiGroups: [ "metrics.k8s.io" ] + resources: + - pods + - nodes + verbs: [ "get", "list" ] + - apiGroups: [ batch ] + resources: + - jobs + - cronjobs + verbs: [ "get", "list" ] diff --git a/config/samples/zora_v1alpha1_cluster.yaml b/config/samples/zora_v1alpha1_cluster.yaml index c7eb6b81..79a05141 100644 --- a/config/samples/zora_v1alpha1_cluster.yaml +++ b/config/samples/zora_v1alpha1_cluster.yaml @@ -8,6 +8,4 @@ metadata: app.kubernetes.io/managed-by: kustomize app.kubernetes.io/created-by: zora name: mycluster -spec: - kubeconfigRef: - name: mycluster-kubeconfig +spec: {} diff --git a/config/samples/zora_v1alpha1_plugin_marvin.yaml b/config/samples/zora_v1alpha1_plugin_marvin.yaml index 5ad538b1..6f5256e7 100644 --- a/config/samples/zora_v1alpha1_plugin_marvin.yaml +++ b/config/samples/zora_v1alpha1_plugin_marvin.yaml @@ -23,7 +23,7 @@ spec: mkdir -p $(CUSTOM_CHECKS_PATH) ls -lh $(CUSTOM_CHECKS_PATH) echo Scanning... - /marvin scan --disable-annotation-skip -f $(CUSTOM_CHECKS_PATH) -o json -v 2 --kubeconfig $(KUBECONFIG) > $(DONE_DIR)/results.json + /marvin scan --disable-annotation-skip -f $(CUSTOM_CHECKS_PATH) -o json -v 2 > $(DONE_DIR)/results.json exitcode=$(echo $?) if [ $exitcode -ne 0 ]; then echo "ERROR" > $(DONE_DIR)/error diff --git a/config/samples/zora_v1alpha1_plugin_popeye.yaml b/config/samples/zora_v1alpha1_plugin_popeye.yaml index ab498858..53f9d757 100644 --- a/config/samples/zora_v1alpha1_plugin_popeye.yaml +++ b/config/samples/zora_v1alpha1_plugin_popeye.yaml @@ -81,7 +81,6 @@ spec: POPEYE_REPORT_DIR=$(DONE_DIR) \ /bin/popeye \ -o json \ - --kubeconfig $(KUBECONFIG) \ --all-namespaces \ --force-exit-zero \ -f /tmp/spinach.yml \ diff --git a/config/samples/zora_v1alpha1_plugin_popeye_all.yaml b/config/samples/zora_v1alpha1_plugin_popeye_all.yaml index 8e246d2c..b6533867 100644 --- a/config/samples/zora_v1alpha1_plugin_popeye_all.yaml +++ b/config/samples/zora_v1alpha1_plugin_popeye_all.yaml @@ -23,7 +23,6 @@ spec: POPEYE_REPORT_DIR=$(DONE_DIR) \ /bin/popeye \ -o json \ - --kubeconfig $(KUBECONFIG) \ --all-namespaces \ --force-exit-zero \ --save \ diff --git a/internal/controller/zora/clusterscan_controller.go b/internal/controller/zora/clusterscan_controller.go index 512df6f0..161d98b5 100644 --- a/internal/controller/zora/clusterscan_controller.go +++ b/internal/controller/zora/clusterscan_controller.go @@ -162,12 +162,16 @@ func (r *ClusterScanReconciler) reconcile(ctx context.Context, clusterscan *v1al log.Error(notReadyErr, "Cluster is not ready") clusterscan.SetReadyStatus(false, "ClusterNotReady", notReadyErr.Error()) } - kubeconfigKey := cluster.KubeconfigRefKey() - kubeconfigSecret, err := kubeconfig.SecretFromRef(ctx, r.Client, *kubeconfigKey) - if err != nil { - log.Error(err, fmt.Sprintf("failed to get kubeconfig secret %s", kubeconfigKey.String())) - clusterscan.SetReadyStatus(false, "ClusterKubeconfigError", err.Error()) - return err + var kubeconfigSecret *corev1.Secret + if cluster.Spec.KubeconfigRef != nil { + key := cluster.KubeconfigRefKey() + sec, err := kubeconfig.SecretFromRef(ctx, r.Client, *key) + if err != nil { + log.Error(err, fmt.Sprintf("failed to get kubeconfig secret %s", key.String())) + clusterscan.SetReadyStatus(false, "ClusterKubeconfigError", err.Error()) + return err + } + kubeconfigSecret = sec } if err := r.setControllerReference(ctx, clusterscan, cluster); err != nil { @@ -193,7 +197,7 @@ func (r *ClusterScanReconciler) reconcile(ctx context.Context, clusterscan *v1al clusterscan.SetReadyStatus(false, "PluginFetchError", err.Error()) return err } - cronJob := plugins.NewCronJob(fmt.Sprintf("%s-%s", clusterscan.Name, plugin.Name), kubeconfigSecret.Namespace) + cronJob := plugins.NewCronJob(fmt.Sprintf("%s-%s", clusterscan.Name, plugin.Name), clusterscan.Namespace) cronJobMutator := &plugins.CronJobMutator{ Scheme: r.Scheme, Existing: cronJob, @@ -441,7 +445,7 @@ func (r *ClusterScanReconciler) defaultPlugins() []v1alpha1.PluginReference { return p } -// applyRBAC Create or Update a ServiceAccount (with ClusterScan as Owner) and append it to ClusterRoleBinding +// applyRBAC Create or Update a ServiceAccount in the ClusterScan namespace (with ClusterScan as Owner) and append it to ClusterRoleBinding func (r *ClusterScanReconciler) applyRBAC(ctx context.Context, clusterscan *v1alpha1.ClusterScan) error { log := ctrllog.FromContext(ctx) diff --git a/pkg/plugins/cronjob.go b/pkg/plugins/cronjob.go index cac76fbb..03160f35 100644 --- a/pkg/plugins/cronjob.go +++ b/pkg/plugins/cronjob.go @@ -65,11 +65,11 @@ var ( }, } // pluginVolumes represents the volume mounts to be used in plugin container - pluginVolumes = append(commonVolumeMounts, corev1.VolumeMount{ + kubeconfigVolumeMount = corev1.VolumeMount{ Name: kubeconfigVolumeName, ReadOnly: true, MountPath: kubeconfigMountPath, - }) + } // customChecksVolume represents the volume mount to be used in the init container customChecksVolume = corev1.VolumeMount{Name: checksVolumeName, MountPath: checksPath} @@ -116,8 +116,19 @@ func (r *CronJobMutator) Mutate() error { r.Existing.Spec.JobTemplate.Spec.Template.Spec.RestartPolicy = corev1.RestartPolicyNever r.Existing.Spec.JobTemplate.Spec.BackoffLimit = pointer.Int32(0) r.Existing.Spec.JobTemplate.Spec.Template.Spec.ServiceAccountName = r.ServiceAccountName + r.Existing.Spec.JobTemplate.Spec.Template.Annotations = map[string]string{"kubectl.kubernetes.io/default-container": r.Plugin.Name} r.Existing.Spec.JobTemplate.Spec.Template.Spec.Volumes = []corev1.Volume{ { + Name: resultsVolumeName, + VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, + }, + { + Name: checksVolumeName, + VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, + }, + } + if r.KubeconfigSecret != nil { + r.Existing.Spec.JobTemplate.Spec.Template.Spec.Volumes = append(r.Existing.Spec.JobTemplate.Spec.Template.Spec.Volumes, corev1.Volume{ Name: kubeconfigVolumeName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ @@ -126,15 +137,7 @@ func (r *CronJobMutator) Mutate() error { Items: []corev1.KeyToPath{{Key: kubeconfig.SecretField, Path: kubeconfigFile}}, }, }, - }, - { - Name: resultsVolumeName, - VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, - }, - { - Name: checksVolumeName, - VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, - }, + }) } r.Existing.Spec.JobTemplate.Spec.Template.Spec.SecurityContext = &corev1.PodSecurityContext{ RunAsNonRoot: pointer.Bool(true), @@ -211,7 +214,10 @@ func (r *CronJobMutator) pluginContainer() corev1.Container { Resources: r.Plugin.Spec.Resources, ImagePullPolicy: r.Plugin.Spec.GetImagePullPolicy(), SecurityContext: r.Plugin.Spec.SecurityContext, - VolumeMounts: pluginVolumes, + VolumeMounts: commonVolumeMounts, + } + if r.KubeconfigSecret != nil { + c.VolumeMounts = append(c.VolumeMounts, kubeconfigVolumeMount) } if pointer.BoolDeref(r.Plugin.Spec.MountCustomChecksVolume, false) { c.VolumeMounts = append(c.VolumeMounts, customChecksVolume) @@ -240,10 +246,6 @@ func (r *CronJobMutator) pluginEnv() []corev1.EnvVar { p := append(r.Plugin.Spec.Env, r.PluginRef.Env...) p = append(p, commonEnv...) p = append(p, - corev1.EnvVar{ - Name: "KUBECONFIG", - Value: filepath.Join(kubeconfigMountPath, kubeconfigFile), - }, corev1.EnvVar{ Name: "CRONJOB_NAMESPACE", Value: r.Existing.ObjectMeta.Namespace, @@ -253,6 +255,12 @@ func (r *CronJobMutator) pluginEnv() []corev1.EnvVar { Value: r.Existing.ObjectMeta.Name, }, ) + if r.KubeconfigSecret != nil { + p = append(p, corev1.EnvVar{ + Name: "KUBECONFIG", + Value: filepath.Join(kubeconfigMountPath, kubeconfigFile), + }) + } if pointer.BoolDeref(r.Plugin.Spec.MountCustomChecksVolume, false) { p = append(p, corev1.EnvVar{Name: "CUSTOM_CHECKS_PATH", Value: checksVolumeName}) }