Merge pull request #638 from makhov/fix-637

Do not use default 6443 port for kube-apiserver in the container

internal/controller/k0smotron.io/jointokenrequest_controller.go

@@ -23,7 +23,6 @@ import (
 	"encoding/base64"
 	"fmt"
 	"io"
-	"net/url"
 	"strings"
 	"time"
 
@@ -34,7 +33,6 @@ import (
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
-	"k8s.io/client-go/tools/clientcmd/api"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
@@ -115,18 +113,12 @@ func (r *JoinTokenRequestReconciler) Reconcile(ctx context.Context, req ctrl.Req
 		return ctrl.Result{Requeue: true, RequeueAfter: time.Minute}, err
 	}
 
-	newToken, newKubeconfig, err := ReplaceTokenPort(token, cluster)
-	if err != nil {
-		r.updateStatus(ctx, jtr, "Failed update token URL")
-		return ctrl.Result{Requeue: true, RequeueAfter: time.Minute}, err
-	}
-
-	if err := r.reconcileSecret(ctx, jtr, newToken); err != nil {
+	if err := r.reconcileSecret(ctx, jtr, token); err != nil {
 		r.updateStatus(ctx, jtr, "Failed creating secret")
 		return ctrl.Result{Requeue: true, RequeueAfter: time.Minute}, err
 	}
 
-	tokenID, err := getTokenID(newKubeconfig, jtr.Spec.Role)
+	tokenID, err := getTokenID(token, jtr.Spec.Role)
 	if err != nil {
 		r.updateStatus(ctx, jtr, "Failed getting token id")
 		return ctrl.Result{Requeue: true, RequeueAfter: time.Minute}, err
@@ -199,46 +191,17 @@ func (r *JoinTokenRequestReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		Complete(r)
 }
 
-func replaceKubeconfigPort(in string, cluster km.Cluster) (string, *api.Config, error) {
-	cfg, err := clientcmd.Load([]byte(in))
-	if err != nil {
-		return "", nil, err
-	}
-
-	u, err := url.Parse(cfg.Clusters["k0s"].Server)
-	if err != nil {
-		return "", nil, err
-	}
-	parts := strings.Split(u.Host, ":")
-	u.Host = fmt.Sprintf("%s:%d", parts[0], cluster.Spec.Service.APIPort)
-
-	cfg.Clusters["k0s"].Server = u.String()
-
-	b, err := clientcmd.Write(*cfg)
-	if err != nil {
-		return "", nil, err
-	}
-
-	return string(b), cfg, nil
-}
-
-func ReplaceTokenPort(token string, cluster km.Cluster) (string, *api.Config, error) {
+func getTokenID(token, role string) (string, error) {
 	b, err := tokenDecode(token)
 	if err != nil {
-		return "", nil, err
+		return "", err
 	}
 
-	updatedKubeconfig, cfg, err := replaceKubeconfigPort(string(b), cluster)
+	cfg, err := clientcmd.Load(b)
 	if err != nil {
-		return "", nil, err
+		return "", err
 	}
 
-	newToken, err := tokenEncode([]byte(updatedKubeconfig))
-
-	return newToken, cfg, err
-}
-
-func getTokenID(cfg *api.Config, role string) (string, error) {
 	var userName string
 	switch role {
 	case "controller":
@@ -270,24 +233,3 @@ func tokenDecode(token string) ([]byte, error) {
 
 	return output, err
 }
-
-func tokenEncode(token []byte) (string, error) {
-	in := bytes.NewReader(token)
-
-	var outBuf bytes.Buffer
-	gz, err := gzip.NewWriterLevel(&outBuf, gzip.BestCompression)
-	if err != nil {
-		return "", err
-	}
-
-	_, err = io.Copy(gz, in)
-	gzErr := gz.Close()
-	if err != nil {
-		return "", err
-	}
-	if gzErr != nil {
-		return "", gzErr
-	}
-
-	return base64.StdEncoding.EncodeToString(outBuf.Bytes()), nil
-}

internal/controller/k0smotron.io/k0smotroncluster_configmap.go

@@ -214,7 +214,7 @@ func getV1Beta1Spec(kmc *km.Cluster, sans []string) map[string]interface{} {
 	v1beta1Spec := map[string]interface{}{
 		"api": map[string]interface{}{
 			"externalAddress": kmc.Spec.ExternalAddress,
-			"port":            defaultKubeAPIPort,
+			"port":            kmc.Spec.Service.APIPort,
 			"sans":            sans,
 		},
 		"konnectivity": map[string]interface{}{

internal/controller/k0smotron.io/k0smotroncluster_controller.go

@@ -36,8 +36,6 @@ import (
 	km "github.com/k0sproject/k0smotron/api/k0smotron.io/v1beta1"
 )
 
-const defaultKubeAPIPort = 6443
-
 var patchOpts []client.PatchOption = []client.PatchOption{
 	client.FieldOwner("k0smotron-operator"),
 	client.ForceOwnership,

internal/controller/k0smotron.io/k0smotroncluster_entrypoint.go

@@ -38,9 +38,10 @@ func init() {
 
 func (r *ClusterReconciler) generateEntrypointCM(kmc *km.Cluster) (v1.ConfigMap, error) {
 	var entrypointBuf bytes.Buffer
-	err := entrypointTmpl.Execute(&entrypointBuf, map[string]string{
+	err := entrypointTmpl.Execute(&entrypointBuf, map[string]interface{}{
 		"KineDataSourceURLPlaceholder": kineDataSourceURLPlaceholder,
 		"K0sControllerArgs":            r.getControllerFlags(kmc),
+		"PrivilegedPortIsUsed":         kmc.Spec.Service.APIPort <= 1024,
 	})
 	if err != nil {
 		return v1.ConfigMap{}, err
@@ -110,6 +111,11 @@ mkdir /etc/k0s && echo "$K0SMOTRON_K0S_YAML" > /etc/k0s/k0s.yaml
 # Substitute the kine datasource URL from the env var
 sed -i "s {{ .KineDataSourceURLPlaceholder }} ${K0SMOTRON_KINE_DATASOURCE_URL} g" /etc/k0s/k0s.yaml
 
+{{if .PrivilegedPortIsUsed}}
+apk add --no-cache libcap
+{ while ! setcap 'cap_net_bind_service=+ep' /var/lib/k0s/bin/kube-apiserver; do sleep 1 ; done ; } &
+{{end}}
+
 # Run the k0s controller
 k0s controller {{ .K0sControllerArgs }}
 `

internal/controller/k0smotron.io/k0smotroncluster_kubeconfig.go

@@ -42,11 +42,6 @@ func (r *ClusterReconciler) reconcileKubeConfigSecret(ctx context.Context, kmc k
 		return err
 	}
 
-	output, _, err = replaceKubeconfigPort(output, kmc)
-	if err != nil {
-		return err
-	}
-
 	logger.Info("Kubeconfig generated, creating the secret")
 
 	secret := v1.Secret{

internal/controller/k0smotron.io/k0smotroncluster_service.go

@@ -40,8 +40,8 @@ func (r *ClusterReconciler) generateService(kmc *km.Cluster) v1.Service {
 		name = kmc.GetNodePortServiceName()
 		ports = append(ports,
 			v1.ServicePort{
-				Port:       int32(defaultKubeAPIPort),
-				TargetPort: intstr.FromInt(defaultKubeAPIPort),
+				Port:       int32(kmc.Spec.Service.APIPort),
+				TargetPort: intstr.FromInt(kmc.Spec.Service.APIPort),
 				Name:       "api",
 				NodePort:   int32(kmc.Spec.Service.APIPort),
 			},
@@ -57,7 +57,7 @@ func (r *ClusterReconciler) generateService(kmc *km.Cluster) v1.Service {
 		ports = append(ports,
 			v1.ServicePort{
 				Port:       int32(kmc.Spec.Service.APIPort),
-				TargetPort: intstr.FromInt(defaultKubeAPIPort),
+				TargetPort: intstr.FromInt(kmc.Spec.Service.APIPort),
 				Name:       "api",
 			},
 			v1.ServicePort{
@@ -76,7 +76,7 @@ func (r *ClusterReconciler) generateService(kmc *km.Cluster) v1.Service {
 		ports = append(ports,
 			v1.ServicePort{
 				Port:       int32(kmc.Spec.Service.APIPort),
-				TargetPort: intstr.FromInt(defaultKubeAPIPort),
+				TargetPort: intstr.FromInt(kmc.Spec.Service.APIPort),
 				Name:       "api",
 			},
 			v1.ServicePort{

internal/controller/k0smotron.io/k0smotroncluster_statefulset.go

@@ -120,7 +120,7 @@ func (r *ClusterReconciler) generateStatefulSet(kmc *km.Cluster) (apps.StatefulS
 							{
 								Name:          "api",
 								Protocol:      v1.ProtocolTCP,
-								ContainerPort: int32(defaultKubeAPIPort),
+								ContainerPort: int32(kmc.Spec.Service.APIPort),
 							},
 							{
 								Name:          "konnectivity",

inttest/basic/basic_test.go

@@ -91,14 +91,14 @@ func (s *BasicSuite) TestK0sGetsUp() {
 	s.checkClusterStatus(s.Context(), rc)
 
 	s.T().Log("Generating k0smotron join token")
-	token, err := util.GetJoinToken(kc, rc, "kmc-kmc-test-0", "kmc-test", 30443)
+	token, err := util.GetJoinToken(kc, rc, "kmc-kmc-test-0", "kmc-test")
 	s.Require().NoError(err)
 
 	s.T().Log("joining worker to k0smotron cluster")
 	s.Require().NoError(s.RunWithToken(s.K0smotronNode(0), token))
 
 	s.T().Log("Starting portforward")
-	fw, err := util.GetPortForwarder(rc, "kmc-kmc-test-0", "kmc-test", 6443)
+	fw, err := util.GetPortForwarder(rc, "kmc-kmc-test-0", "kmc-test", 30443)
 	s.Require().NoError(err)
 
 	go fw.Start(s.Require().NoError)

inttest/capi-docker-machinedeployment/capi_docker_test.go

@@ -90,7 +90,7 @@ func (s *CAPIDockerSuite) TestCAPIDocker() {
 	s.Require().NoError(common.WaitForStatefulSet(s.ctx, s.client, "kmc-docker-md-test", "default"))
 
 	s.T().Log("Starting portforward")
-	fw, err := util.GetPortForwarder(s.restConfig, "kmc-docker-md-test-0", "default", 6443)
+	fw, err := util.GetPortForwarder(s.restConfig, "kmc-docker-md-test-0", "default", 30443)
 	s.Require().NoError(err)
 
 	go fw.Start(s.Require().NoError)

inttest/capi-docker/capi_docker_test.go

@@ -103,7 +103,7 @@ func (s *CAPIDockerSuite) TestCAPIDocker() {
 	s.checkControlPlaneStatus(s.ctx, s.restConfig)
 
 	s.T().Log("Starting portforward")
-	fw, err := util.GetPortForwarder(s.restConfig, "kmc-docker-test-0", "default", 6443)
+	fw, err := util.GetPortForwarder(s.restConfig, "kmc-docker-test-0", "default", 30443)
 	s.Require().NoError(err)
 
 	go fw.Start(s.Require().NoError)

inttest/capi-remote-machine-job-provision/capi_remote_machine_job_provision_test.go

@@ -133,7 +133,7 @@ func (s *RemoteMachineSuite) TestCAPIRemoteMachine() {
 	s.Require().NoError(common.WaitForStatefulSet(ctx, s.client, "kmc-remote-test", "default"))
 
 	s.T().Log("Starting portforward")
-	fw, err := util.GetPortForwarder(s.restConfig, "kmc-remote-test-0", "default", 6443)
+	fw, err := util.GetPortForwarder(s.restConfig, "kmc-remote-test-0", "default", 30443)
 	s.Require().NoError(err)
 
 	go fw.Start(s.Require().NoError)

inttest/capi-remote-machine/capi_remote_machine_test.go

@@ -134,7 +134,7 @@ func (s *RemoteMachineSuite) TestCAPIRemoteMachine() {
 	s.Require().NoError(common.WaitForStatefulSet(ctx, s.client, "kmc-remote-test", "default"))
 
 	s.T().Log("Starting portforward")
-	fw, err := util.GetPortForwarder(s.restConfig, "kmc-remote-test-0", "default", 6443)
+	fw, err := util.GetPortForwarder(s.restConfig, "kmc-remote-test-0", "default", 30443)
 	s.Require().NoError(err)
 
 	go fw.Start(s.Require().NoError)

inttest/config-update-hcp/config_update_test.go

@@ -67,7 +67,7 @@ func (s *ConfigUpdateSuite) TestK0sGetsUp() {
 	s.checkClusterStatus(s.Context(), rc)
 
 	s.T().Log("Starting portforward")
-	fw, err := util.GetPortForwarder(rc, "kmc-kmc-test-0", "kmc-test", 6443)
+	fw, err := util.GetPortForwarder(rc, "kmc-kmc-test-0", "kmc-test", 30443)
 	s.Require().NoError(err)
 
 	go fw.Start(s.Require().NoError)

inttest/ha-controller-etcd/ha_controller_etcd_test.go

@@ -56,7 +56,7 @@ func (s *HAControllerEtcdSuite) TestK0sGetsUp() {
 	s.Require().NoError(common.WaitForStatefulSet(s.Context(), kc, "kmc-kmc-test", "kmc-test"))
 
 	s.T().Log("Generating k0smotron join token")
-	token, err := util.GetJoinToken(kc, rc, "kmc-kmc-test-0", "kmc-test", 30443)
+	token, err := util.GetJoinToken(kc, rc, "kmc-kmc-test-0", "kmc-test")
 	s.Require().NoError(err)
 
 	s.T().Log("joining worker to k0smotron cluster")
@@ -65,18 +65,15 @@ func (s *HAControllerEtcdSuite) TestK0sGetsUp() {
 	s.T().Log("Starting portforward")
 	pod := s.getPod(s.Context(), kc)
 
-	fw, err := util.GetPortForwarder(rc, pod.Name, pod.Namespace, 6443)
+	fw, err := util.GetPortForwarder(rc, pod.Name, pod.Namespace, 30443)
 	s.Require().NoError(err)
 	go fw.Start(s.Require().NoError)
 	defer fw.Close()
 
 	<-fw.ReadyChan
 
-	localPort, err := fw.LocalPort()
-	s.Require().NoError(err)
-
 	s.T().Log("waiting for node to be ready")
-	kmcKC, err := util.GetKMCClientSet(s.Context(), kc, "kmc-test", "kmc-test", localPort)
+	kmcKC, err := util.GetKMCClientSet(s.Context(), kc, "kmc-test", "kmc-test", 30443)
 	s.Require().NoError(err)
 	s.Require().NoError(s.WaitForNodeReady(s.K0smotronNode(0), kmcKC))
 

inttest/ha-controller-secret/ha_controller_secret_test.go

@@ -64,7 +64,7 @@ func (s *HAControllerSecretSuite) TestK0sGetsUp() {
 	s.Require().NoError(common.WaitForStatefulSet(s.Context(), kc, "kmc-kmc-test-secret", "kmc-test"))
 
 	s.T().Log("Generating k0smotron join token")
-	token, err := util.GetJoinToken(kc, rc, "kmc-kmc-test-secret-0", "kmc-test", 30443)
+	token, err := util.GetJoinToken(kc, rc, "kmc-kmc-test-secret-0", "kmc-test")
 	s.Require().NoError(err)
 
 	s.T().Log("joining worker to k0smotron cluster")
@@ -73,18 +73,15 @@ func (s *HAControllerSecretSuite) TestK0sGetsUp() {
 	s.T().Log("Starting portforward")
 	pod := s.getPod(s.Context(), kc)
 
-	fw, err := util.GetPortForwarder(rc, pod.Name, pod.Namespace, 6443)
+	fw, err := util.GetPortForwarder(rc, pod.Name, pod.Namespace, 30443)
 	s.Require().NoError(err)
 	go fw.Start(s.Require().NoError)
 	defer fw.Close()
 
 	<-fw.ReadyChan
 	s.T().Log("portforward ready")
-	localPort, err := fw.LocalPort()
-	s.Require().NoError(err)
-
 	s.T().Log("getting child clientset")
-	kmcKC, err := util.GetKMCClientSet(s.Context(), kc, "kmc-test-secret", "kmc-test", localPort)
+	kmcKC, err := util.GetKMCClientSet(s.Context(), kc, "kmc-test-secret", "kmc-test", 30443)
 	s.Require().NoError(err)
 	s.T().Log("waiting for node to be ready")
 	s.Require().NoError(s.WaitForNodeReady(s.K0smotronNode(0), kmcKC))

inttest/ha-controller/ha_controller_test.go

@@ -60,7 +60,7 @@ func (s *HAControllerSuite) TestK0sGetsUp() {
 	s.Require().NoError(common.WaitForStatefulSet(s.Context(), kc, "kmc-kmc-test", "kmc-test"))
 
 	s.T().Log("Generating k0smotron join token")
-	token, err := util.GetJoinToken(kc, rc, "kmc-kmc-test-0", "kmc-test", 30443)
+	token, err := util.GetJoinToken(kc, rc, "kmc-kmc-test-0", "kmc-test")
 	s.Require().NoError(err)
 
 	s.T().Log("joining worker to k0smotron cluster")
@@ -69,18 +69,15 @@ func (s *HAControllerSuite) TestK0sGetsUp() {
 	s.T().Log("Starting portforward")
 	pod := s.getPod(s.Context(), kc)
 
-	fw, err := util.GetPortForwarder(rc, pod.Name, pod.Namespace, 6443)
+	fw, err := util.GetPortForwarder(rc, pod.Name, pod.Namespace, 30443)
 	s.Require().NoError(err)
 	go fw.Start(s.Require().NoError)
 	defer fw.Close()
 
 	<-fw.ReadyChan
 
-	localPort, err := fw.LocalPort()
-	s.Require().NoError(err)
-
 	s.T().Log("waiting for node to be ready")
-	kmcKC, err := util.GetKMCClientSet(s.Context(), kc, "kmc-test", "kmc-test", localPort)
+	kmcKC, err := util.GetKMCClientSet(s.Context(), kc, "kmc-test", "kmc-test", 30443)
 	s.Require().NoError(err)
 	s.Require().NoError(s.WaitForNodeReady(s.K0smotronNode(0), kmcKC))
 

inttest/hostpath/hostpath_test.go

@@ -63,25 +63,22 @@ func (s *HostPathSuite) TestK0sGetsUp() {
 	s.Require().NoError(common.WaitForStatefulSet(s.Context(), kc, "kmc-kmc-test", "kmc-test"))
 
 	s.T().Log("Generating k0smotron join token")
-	token, err := util.GetJoinToken(kc, rc, "kmc-kmc-test-0", "kmc-test", 30443)
+	token, err := util.GetJoinToken(kc, rc, "kmc-kmc-test-0", "kmc-test")
 	s.Require().NoError(err)
 
 	s.T().Log("joining worker to k0smotron cluster")
 	s.Require().NoError(s.RunWithToken(s.K0smotronNode(0), token))
 
 	s.T().Log("Starting portforward")
-	fw, err := util.GetPortForwarder(rc, "kmc-kmc-test-0", "kmc-test", 6443)
+	fw, err := util.GetPortForwarder(rc, "kmc-kmc-test-0", "kmc-test", 30443)
 	s.Require().NoError(err)
 	go fw.Start(s.Require().NoError)
 	defer fw.Close()
 
 	<-fw.ReadyChan
 
-	localPort, err := fw.LocalPort()
-	s.Require().NoError(err)
-
 	s.T().Log("waiting for node to be ready")
-	kmcKC, err := util.GetKMCClientSet(s.Context(), kc, "kmc-test", "kmc-test", localPort)
+	kmcKC, err := util.GetKMCClientSet(s.Context(), kc, "kmc-test", "kmc-test", 30443)
 	s.Require().NoError(err)
 	s.Require().NoError(s.WaitForNodeReady(s.K0smotronNode(0), kmcKC))