feat(kuma-cp) localhost is not an admin on kubernetes (#3003)

Signed-off-by: Jakub Dyszkiewicz <jakub.dyszkiewicz@gmail.com>
kumahq · Oct 26, 2021 · 9aeea34 · 9aeea34
1 parent e046fe0
commit 9aeea34
Show file tree

Hide file tree

Showing 12 changed files with 80 additions and 16 deletions.
diff --git a/app/kumactl/cmd/install/testdata/install-control-plane.cni-enabled.golden.yaml b/app/kumactl/cmd/install/testdata/install-control-plane.cni-enabled.golden.yaml
@@ -1033,6 +1033,8 @@ spec:
           image: "docker.io/kumahq/kuma-cp:0.0.1"
           imagePullPolicy: IfNotPresent
           env:
+            - name: KUMA_API_SERVER_AUTHN_LOCALHOST_IS_ADMIN
+              value: "false"
             - name: KUMA_API_SERVER_READ_ONLY
               value: "true"
             - name: KUMA_DEFAULTS_SKIP_MESH_CREATION

diff --git a/app/kumactl/cmd/install/testdata/install-control-plane.defaults.golden.yaml b/app/kumactl/cmd/install/testdata/install-control-plane.defaults.golden.yaml
@@ -858,6 +858,8 @@ spec:
           image: "docker.io/kumahq/kuma-cp:0.0.1"
           imagePullPolicy: IfNotPresent
           env:
+            - name: KUMA_API_SERVER_AUTHN_LOCALHOST_IS_ADMIN
+              value: "false"
             - name: KUMA_API_SERVER_READ_ONLY
               value: "true"
             - name: KUMA_DEFAULTS_SKIP_MESH_CREATION

diff --git a/app/kumactl/cmd/install/testdata/install-control-plane.global.golden.yaml b/app/kumactl/cmd/install/testdata/install-control-plane.global.golden.yaml
@@ -867,6 +867,8 @@ spec:
           image: "docker.io/kumahq/kuma-cp:0.0.1"
           imagePullPolicy: IfNotPresent
           env:
+            - name: KUMA_API_SERVER_AUTHN_LOCALHOST_IS_ADMIN
+              value: "false"
             - name: KUMA_API_SERVER_READ_ONLY
               value: "true"
             - name: KUMA_DEFAULTS_SKIP_MESH_CREATION

diff --git a/app/kumactl/cmd/install/testdata/install-control-plane.override-env-vars.golden.yaml b/app/kumactl/cmd/install/testdata/install-control-plane.override-env-vars.golden.yaml
@@ -858,6 +858,8 @@ spec:
           image: "docker.io/kumahq/kuma-cp:0.0.1"
           imagePullPolicy: IfNotPresent
           env:
+            - name: KUMA_API_SERVER_AUTHN_LOCALHOST_IS_ADMIN
+              value: "false"
             - name: KUMA_API_SERVER_READ_ONLY
               value: "true"
             - name: KUMA_DEFAULTS_SKIP_MESH_CREATION

diff --git a/app/kumactl/cmd/install/testdata/install-control-plane.overrides.golden.yaml b/app/kumactl/cmd/install/testdata/install-control-plane.overrides.golden.yaml
@@ -862,6 +862,8 @@ spec:
           image: "kuma-ci/kuma-cp:greatest"
           imagePullPolicy: Never
           env:
+            - name: KUMA_API_SERVER_AUTHN_LOCALHOST_IS_ADMIN
+              value: "false"
             - name: KUMA_API_SERVER_AUTH_CLIENT_CERTS_DIR
               value: "/var/run/secrets/kuma.io/api-server-client-certs/"
             - name: KUMA_API_SERVER_HTTPS_TLS_CERT_FILE

diff --git a/app/kumactl/cmd/install/testdata/install-control-plane.with-ingress.golden.yaml b/app/kumactl/cmd/install/testdata/install-control-plane.with-ingress.golden.yaml
@@ -887,6 +887,8 @@ spec:
           image: "docker.io/kumahq/kuma-cp:0.0.1"
           imagePullPolicy: IfNotPresent
           env:
+            - name: KUMA_API_SERVER_AUTHN_LOCALHOST_IS_ADMIN
+              value: "false"
             - name: KUMA_API_SERVER_READ_ONLY
               value: "true"
             - name: KUMA_DEFAULTS_SKIP_MESH_CREATION

diff --git a/app/kumactl/cmd/install/testdata/install-control-plane.zone.golden.yaml b/app/kumactl/cmd/install/testdata/install-control-plane.zone.golden.yaml
@@ -862,6 +862,8 @@ spec:
           image: "docker.io/kumahq/kuma-cp:0.0.1"
           imagePullPolicy: IfNotPresent
           env:
+            - name: KUMA_API_SERVER_AUTHN_LOCALHOST_IS_ADMIN
+              value: "false"
             - name: KUMA_API_SERVER_READ_ONLY
               value: "true"
             - name: KUMA_DEFAULTS_SKIP_MESH_CREATION

diff --git a/deployments/charts/kuma/templates/_helpers.tpl b/deployments/charts/kuma/templates/_helpers.tpl
@@ -158,6 +158,8 @@ env:
 - name: KUMA_MULTIZONE_ZONE_KDS_ROOT_CA_FILE
   value: /var/run/secrets/kuma.io/kds-client-tls-cert/ca.crt
 {{- end }}
+- name: KUMA_API_SERVER_AUTHN_LOCALHOST_IS_ADMIN
+  value: "false"
 {{- end }}
 
 {{/*

diff --git a/test/e2e/compatibility/cp_compatibility_kubernetes_multizone.go b/test/e2e/compatibility/cp_compatibility_kubernetes_multizone.go
@@ -61,6 +61,7 @@ func CpCompatibilityMultizoneKubernetes() {
 		)
 
 		globalDeployOptsFuncs = append(globalDeployOptsFuncs,
+			WithEnv("KUMA_API_SERVER_AUTH_ALLOW_FROM_LOCALHOST", "true"),
 			WithInstallationMode(HelmInstallationMode),
 			WithHelmChartPath(HelmRepo),
 			WithHelmReleaseName(globalReleaseName),
@@ -90,6 +91,7 @@ func CpCompatibilityMultizoneKubernetes() {
 		)
 
 		zoneDeployOptsFuncs = append(zoneDeployOptsFuncs,
+			WithEnv("KUMA_API_SERVER_AUTH_ALLOW_FROM_LOCALHOST", "true"),
 			WithInstallationMode(HelmInstallationMode),
 			WithHelmChartPath(HelmRepo),
 			WithHelmReleaseName(zoneReleaseName),

diff --git a/test/e2e/helm/kuma_helm_upgrade.go b/test/e2e/helm/kuma_helm_upgrade.go
@@ -55,6 +55,7 @@ func UpgradingWithHelmChart() {
 			)
 
 			deployOptsFuncs = append(deployOptsFuncs,
+				WithEnv("KUMA_API_SERVER_AUTH_ALLOW_FROM_LOCALHOST", "true"),
 				WithInstallationMode(HelmInstallationMode),
 				WithHelmChartPath(HelmRepo),
 				WithHelmReleaseName(releaseName),

diff --git a/test/framework/k8s_cluster.go b/test/framework/k8s_cluster.go
@@ -495,7 +495,9 @@ func (c *K8sCluster) DeployKuma(mode core.CpMode, opt ...KumaDeploymentOption) e
 		replicas = opts.cpReplicas
 	}
 
-	c.controlplane = NewK8sControlPlane(c.t, mode, c.name, c.kubeconfig, c, c.loPort, c.hiPort, c.verbose, replicas)
+	// backwards compatibility, check for 1.3.x localhost is admin env variable.
+	localhostIsAdmin := opts.env["KUMA_API_SERVER_AUTH_ALLOW_FROM_LOCALHOST"] == "true"
+	c.controlplane = NewK8sControlPlane(c.t, mode, c.name, c.kubeconfig, c, c.loPort, c.hiPort, c.verbose, replicas, localhostIsAdmin)
 
 	switch mode {
 	case core.Zone:

diff --git a/test/framework/k8s_controlplane.go b/test/framework/k8s_controlplane.go
@@ -11,6 +11,7 @@ import (
 
 	http_helper "github.com/gruntwork-io/terratest/modules/http-helper"
 	"github.com/gruntwork-io/terratest/modules/k8s"
+	"github.com/gruntwork-io/terratest/modules/retry"
 	"github.com/gruntwork-io/terratest/modules/testing"
 	"github.com/pkg/errors"
 	v1 "k8s.io/api/core/v1"
@@ -32,15 +33,16 @@ type PortFwd struct {
 }
 
 type K8sControlPlane struct {
-	t          testing.TestingT
-	mode       core.CpMode
-	name       string
-	kubeconfig string
-	kumactl    *KumactlOptions
-	cluster    *K8sCluster
-	portFwd    PortFwd
-	verbose    bool
-	replicas   int
+	t                testing.TestingT
+	mode             core.CpMode
+	name             string
+	kubeconfig       string
+	kumactl          *KumactlOptions
+	cluster          *K8sCluster
+	portFwd          PortFwd
+	verbose          bool
+	replicas         int
+	localhostIsAdmin bool
 }
 
 func NewK8sControlPlane(
@@ -53,6 +55,7 @@ func NewK8sControlPlane(
 	hiPort uint32,
 	verbose bool,
 	replicas int,
+	localhostIsAdmin bool,
 ) *K8sControlPlane {
 	name := clusterName + "-" + mode
 	kumactl, _ := NewKumactlOptions(t, name, verbose)
@@ -66,8 +69,9 @@ func NewK8sControlPlane(
 		portFwd: PortFwd{
 			localAPIPort: loPort,
 		},
-		verbose:  verbose,
-		replicas: replicas,
+		verbose:          verbose,
+		replicas:         replicas,
+		localhostIsAdmin: localhostIsAdmin,
 	}
 }
 
@@ -194,8 +198,25 @@ func (c *K8sControlPlane) FinalizeAdd() error {
 	if err := c.PortForwardKumaCP(); err != nil {
 		return err
 	}
-	// token is not important since we are accessing it from localhost anyways so we are admin
-	return c.kumactl.KumactlConfigControlPlanesAdd(c.name, c.GetAPIServerAddress(), "")
+	var token string
+	if !c.localhostIsAdmin {
+		t, err := c.retrieveAdminToken()
+		if err != nil {
+			return err
+		}
+		token = t
+	}
+	return c.kumactl.KumactlConfigControlPlanesAdd(c.name, c.GetAPIServerAddress(), token)
+}
+
+func (c *K8sControlPlane) retrieveAdminToken() (string, error) {
+	return retry.DoWithRetryE(c.t, "generating DP token", DefaultRetries, DefaultTimeout, func() (string, error) {
+		sec, err := k8s.GetSecretE(c.t, c.GetKubectlOptions(KumaNamespace), "admin-user-token")
+		if err != nil {
+			return "", err
+		}
+		return string(sec.Data["value"]), nil
+	})
 }
 
 func (c *K8sControlPlane) InstallCP(args ...string) (string, error) {
@@ -261,6 +282,14 @@ func (c *K8sControlPlane) GetGlobaStatusAPI() string {
 }
 
 func (c *K8sControlPlane) GenerateDpToken(mesh, service string) (string, error) {
+	var token string
+	if !c.localhostIsAdmin {
+		t, err := c.retrieveAdminToken()
+		if err != nil {
+			return "", err
+		}
+		token = t
+	}
 	dpType := ""
 	if service == "ingress" {
 		dpType = "ingress"
@@ -270,7 +299,10 @@ func (c *K8sControlPlane) GenerateDpToken(mesh, service string) (string, error)
 		"POST",
 		fmt.Sprintf("http://localhost:%d/tokens", c.portFwd.localAPIPort),
 		[]byte(fmt.Sprintf(`{"mesh": "%s", "type": "%s", "tags": {"kuma.io/service": ["%s"]}}`, mesh, dpType, service)),
-		map[string]string{"content-type": "application/json"},
+		map[string]string{
+			"content-type":  "application/json",
+			"authorization": "Bearer " + token,
+		},
 		200,
 		DefaultRetries,
 		DefaultTimeout,
@@ -279,12 +311,23 @@ func (c *K8sControlPlane) GenerateDpToken(mesh, service string) (string, error)
 }
 
 func (c *K8sControlPlane) GenerateZoneIngressToken(zone string) (string, error) {
+	var token string
+	if !c.localhostIsAdmin {
+		t, err := c.retrieveAdminToken()
+		if err != nil {
+			return "", err
+		}
+		token = t
+	}
 	return http_helper.HTTPDoWithRetryE(
 		c.t,
 		"POST",
 		fmt.Sprintf("http://localhost:%d/tokens/zone-ingress", c.portFwd.localAPIPort),
 		[]byte(fmt.Sprintf(`{"zone": "%s"}`, zone)),
-		map[string]string{"content-type": "application/json"},
+		map[string]string{
+			"content-type":  "application/json",
+			"authorization": "Bearer " + token,
+		},
 		200,
 		DefaultRetries,
 		DefaultTimeout,