[UD-1580] Remove kube-rbac-proxy (#324)

* use filters.WithAuthenticationAndAuthorization * update kustomize files * remove kube-rbac-proxy from helm chart * go mod tidy
undistro · Dec 10, 2024 · d0581a2 · d0581a2
1 parent 7d367b6
commit d0581a2
Show file tree

Hide file tree

Showing 46 changed files with 270 additions and 346 deletions.
diff --git a/charts/zora/templates/operator/deployment.yaml b/charts/zora/templates/operator/deployment.yaml
@@ -65,22 +65,6 @@ spec:
       {{- end }}
       {{- end }}
       containers:
-        - name: kube-rbac-proxy
-          args:
-            - --secure-listen-address=0.0.0.0:8443
-            - --upstream=http://127.0.0.1:8080/
-            - --logtostderr=true
-            - --v=0
-          image: "{{ .Values.operator.rbacProxy.image.repository }}:{{ .Values.operator.rbacProxy.image.tag }}"
-          imagePullPolicy: {{ .Values.operator.rbacProxy.image.pullPolicy }}
-          ports:
-            - containerPort: 8443
-              name: https
-              protocol: TCP
-          resources:
-            {{- toYaml .Values.operator.rbacProxy.resources | nindent 12 }}
-          securityContext:
-            {{- toYaml .Values.operator.rbacProxy.securityContext | nindent 12 }}
         - name: manager
           {{- if .Values.httpsProxy }}
           env:
@@ -93,7 +77,7 @@ spec:
             - /manager
           args:
             - --health-probe-bind-address=:8081
-            - --metrics-bind-address=127.0.0.1:8080
+            - --metrics-bind-address=:8443
             - --leader-elect
             - --zap-encoder={{ .Values.operator.log.encoding }}
             - --zap-log-level={{ .Values.operator.log.level }}
@@ -106,9 +90,9 @@ spec:
             - --cronjob-serviceaccount-name=zora-plugins
             - --trivy-db-pvc={{- if .Values.scan.plugins.trivy.persistence.enabled }}trivy-dbs-volume{{- end }}
             - --trivy-fs-group={{ .Values.scan.plugins.trivy.fsGroup }}
-{{- if .Values.scan.plugins.annotations}}
+            {{- if .Values.scan.plugins.annotations }}
             - --cronjob-serviceaccount-annotations={{ $first := true }}{{- range $key, $value := .Values.scan.plugins.annotations }}{{if not $first}},{{else}}{{$first = false}}{{end}}{{ $key }}={{$value}}{{- end }}
-{{- end }}
+            {{- end }}
             - --saas-workspace-id={{ .Values.saas.workspaceID }}
             - --saas-server={{ .Values.saas.server }}
             - --version={{ .Chart.Version }}
@@ -127,7 +111,7 @@ spec:
             - containerPort: 8081
               protocol: TCP
               name: probe
-            - containerPort: 8080
+            - containerPort: 8443
               protocol: TCP
               name: metrics
           {{- if .Values.operator.webhook.enabled }}

diff --git a/charts/zora/templates/operator/metrics-service.yaml b/charts/zora/templates/operator/metrics-service.yaml
@@ -24,6 +24,6 @@ spec:
     - name: https
       port: {{ .Values.operator.metricsService.port }}
       protocol: TCP
-      targetPort: https
+      targetPort: metrics
   selector:
     {{- include "zora.operatorSelectorLabels" . | nindent 4 }}
diff --git a/charts/zora/values.yaml b/charts/zora/values.yaml
@@ -102,29 +102,6 @@ operator:
     requests:
       cpu: 10m
       memory: 64Mi
-  rbacProxy:
-    image:
-      # -- `kube-rbac-proxy` image repository
-      repository: gcr.io/kubebuilder/kube-rbac-proxy
-      # -- `kube-rbac-proxy` image tag
-      tag: v0.15.0
-      # -- Image pull policy
-      pullPolicy: IfNotPresent
-    # -- [Security Context](https://kubernetes.io/docs/tasks/configure-pod-container/security-context) to add to `kube-rbac-proxy` container
-    securityContext:
-      allowPrivilegeEscalation: false
-      readOnlyRootFilesystem: true
-      capabilities:
-        drop:
-          - ALL
-    # -- [Resources](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers) to add to `kube-rbac-proxy` container
-    resources:
-      limits:
-        cpu: 500m
-        memory: 128Mi
-      requests:
-        cpu: 5m
-        memory: 64Mi
   # -- [Node selection](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) to constrain a Pod to only be able to run on particular Node(s)
   nodeSelector: {}
   # -- [Tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration) for pod assignment

diff --git a/cmd/main.go b/cmd/main.go
@@ -23,12 +23,9 @@ import (
 	"strings"
 	"time"
 
-	"github.com/undistro/zora/pkg/crds"
-
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
-	ctrllog "sigs.k8s.io/controller-runtime/pkg/log"
 
 	"go.uber.org/zap/zapcore"
 	apiextensionsv1client "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1"
@@ -38,14 +35,17 @@ import (
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
+	ctrllog "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 
 	zorav1alpha1 "github.com/undistro/zora/api/zora/v1alpha1"
 	zorav1alpha2 "github.com/undistro/zora/api/zora/v1alpha2"
 	zoracontroller "github.com/undistro/zora/internal/controller/zora"
 	"github.com/undistro/zora/internal/saas"
+	"github.com/undistro/zora/pkg/crds"
 	//+kubebuilder:scaffold:imports
 )
 
@@ -91,13 +91,14 @@ func main() {
 	var webhookServicePath string
 	var tokenPath string
 
-	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
+	flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+
+		"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
-	flag.BoolVar(&secureMetrics, "metrics-secure", false,
-		"If set the metrics endpoint is served securely")
+	flag.BoolVar(&secureMetrics, "metrics-secure", true,
+		"If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
 	flag.BoolVar(&enableHTTP2, "enable-http2", false,
 		"If set, HTTP/2 will be enabled for the metrics and webhook servers")
 	flag.StringVar(&defaultPluginsNamespace, "default-plugins-namespace", "zora-system", "The namespace of default plugins")
@@ -153,22 +154,33 @@ func main() {
 		c.NextProtos = []string{"http/1.1"}
 	}
 
-	tlsOpts := []func(*tls.Config){}
+	var tlsOpts []func(*tls.Config)
 	if !enableHTTP2 {
 		tlsOpts = append(tlsOpts, disableHTTP2)
 	}
 	webhookServer := webhook.NewServer(webhook.Options{
 		TLSOpts: tlsOpts,
 	})
 
+	metricsServerOptions := metricsserver.Options{
+		BindAddress:   metricsAddr,
+		SecureServing: secureMetrics,
+		TLSOpts:       tlsOpts,
+	}
+	if secureMetrics {
+		// FilterProvider is used to protect the metrics endpoint with authn/authz.
+		// These configurations ensure that only authorized users and service accounts
+		// can access the metrics endpoint.
+		metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization
+
+		// If CertDir, CertName, and KeyName are not specified, controller-runtime will automatically
+		// generate self-signed certificates for the metrics server.
+	}
+
 	restConfig := ctrl.GetConfigOrDie()
 	mgr, err := ctrl.NewManager(restConfig, ctrl.Options{
-		Scheme: scheme,
-		Metrics: metricsserver.Options{
-			BindAddress:   metricsAddr,
-			SecureServing: secureMetrics,
-			TLSOpts:       tlsOpts,
-		},
+		Scheme:                 scheme,
+		Metrics:                metricsServerOptions,
 		HealthProbeBindAddress: probeAddr,
 		LeaderElection:         enableLeaderElection,
 		LeaderElectionID:       "e0f4eef4.zora.undistro.io",

diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml
@@ -39,6 +39,5 @@ patches:
 
 # [WEBHOOK] To enable webhook, uncomment the following section
 # the following config is for teaching kustomize how to do kustomization for CRDs.
-
 configurations:
 - kustomizeconfig.yaml