From 0707ed0627265887e14441e102c9a14e8da20d68 Mon Sep 17 00:00:00 2001 From: Yusuke KUOKA Date: Sun, 30 Sep 2018 22:16:16 +0900 Subject: [PATCH] bump kiam to 2.8 Just because it is the latest release and I've used it for testing in combination with the initial amazon-vpc-cni-k8s support. --- core/controlplane/config/amazon_vpc.go | 58 +++++++++++++++++++ core/controlplane/config/config.go | 6 +- .../config/templates/cloud-config-controller | 31 ++++++---- .../config/templates/cloud-config-worker | 20 +++++-- core/root/config/templates/cluster.yaml | 2 +- node/uploaded_file.go | 18 +++--- test/integration/maincluster_test.go | 6 +- test/integration/plugin_test.go | 2 +- 8 files changed, 108 insertions(+), 35 deletions(-) create mode 100644 core/controlplane/config/amazon_vpc.go diff --git a/core/controlplane/config/amazon_vpc.go b/core/controlplane/config/amazon_vpc.go new file mode 100644 index 000000000..bcf0da486 --- /dev/null +++ b/core/controlplane/config/amazon_vpc.go @@ -0,0 +1,58 @@ +package config + +import ( + "fmt" + "github.com/aws/amazon-vpc-cni-k8s/pkg/awsutils" + "github.com/kubernetes-incubator/kube-aws/node" +) + +type AmazonVPC struct { + Enabled bool `yaml:"enabled"` +} + +func (a AmazonVPC) MaxPodsScript() node.UploadedFileContent { + script := `#!/usr/bin/env bash + +set -e + +declare -A instance_eni_available +` + + for it, num := range awsutils.InstanceENIsAvailable { + script = script + fmt.Sprintf(`instance_eni_available["%s"]=%d +`, it, num) + } + + script = script + ` +declare -A instance_ip_available +` + for it, num := range awsutils.InstanceIPsAvailable { + script = script + fmt.Sprintf(`instance_ip_available["%s"]=%d +`, it, num) + } + + script = script + ` + +instance_type=$(curl http://169.254.169.254/latest/meta-data/instance-type) + +enis=${instance_eni_available["$instance_type"]} + +if [ "" == "$enis" ]; then + echo "unsupported instance type: no enis_per_eni defined: $instance_type" 1>&2 + exit 1 +fi + +# According to https://github.com/aws/amazon-vpc-cni-k8s#eni-allocation +ips_per_eni=${instance_ip_available["$instance_type"]} + +if [ "" == "$ips_per_eni" ]; then + echo "unsupported instance type: no ips_per_eni defined: $instance_type" 1>&2 + exit 1 +fi + +max_pods=$(( (enis * (ips_per_eni - 1)) + 2 )) + +printf $max_pods +` + return node.NewUploadedFileContent([]byte(script)) +} diff --git a/core/controlplane/config/config.go b/core/controlplane/config/config.go index 77df1a8f1..cce4d7e71 100644 --- a/core/controlplane/config/config.go +++ b/core/controlplane/config/config.go @@ -123,7 +123,7 @@ func NewDefaultCluster() *Cluster { }, KIAMSupport: KIAMSupport{ Enabled: false, - Image: model.Image{Repo: "quay.io/uswitch/kiam", Tag: "v2.7", RktPullDocker: false}, + Image: model.Image{Repo: "quay.io/uswitch/kiam", Tag: "v2.8", RktPullDocker: false}, SessionDuration: "15m", ServerAddresses: KIAMServerAddresses{ServerAddress: "localhost:443", AgentAddress: "kiam-server:443"}, }, @@ -794,10 +794,6 @@ type Networking struct { SelfHosting SelfHosting `yaml:"selfHosting"` } -type AmazonVPC struct { - Enabled bool `yaml:"enabled"` -} - type SelfHosting struct { Type string `yaml:"type"` Typha bool `yaml:"typha"` diff --git a/core/controlplane/config/templates/cloud-config-controller b/core/controlplane/config/templates/cloud-config-controller index d0f3af9af..d29ee6fe2 100644 --- a/core/controlplane/config/templates/cloud-config-controller +++ b/core/controlplane/config/templates/cloud-config-controller @@ -346,6 +346,7 @@ coreos: {{- end }} {{- if .Kubernetes.Networking.AmazonVPC.Enabled }} --node-ip=$$(curl http://169.254.169.254/latest/meta-data/local-ipv4) \ + --max-pods=$$(/opt/bin/aws-k8s-cni-max-pods) \ {{- end }} $KUBELET_OPTS \ " @@ -4848,7 +4849,13 @@ write_files: namespace: kube-system {{end}} -{{if .Kubernetes.Networking.AmazonVPC.Enabled }} +{{if .Kubernetes.Networking.AmazonVPC.Enabled}} + - path: /opt/bin/aws-k8s-cni-max-pods + owner: root:root + permissions: 0755 + encoding: gzip+base64 + content: {{.Kubernetes.Networking.AmazonVPC.MaxPodsScript.ToGzip.ToBase64}} + - path: /srv/kubernetes/manifests/aws-k8s-cni.yaml content: | --- @@ -4922,16 +4929,16 @@ write_files: - operator: Exists # required to avoid pod creation errors like the below: # NetworkPlugin cni failed to set up pod "heapster-5ccb7ff4b-cdq24_kube-system" network: failed to find plugin "loopback" in path [/opt/cni/bin] - initContainers: - - name: hyperkube - image: {{ .HyperkubeImage.RepoWithTag }} - command: - - /bin/sh - - -c - - "cp /opt/cni/bin/* /host/opt/cni/bin/" - volumeMounts: - - mountPath: /host/opt/cni/bin - name: cni-bin-dir + initContainers: + - name: hyperkube + image: {{ .HyperkubeImage.RepoWithTag }} + command: + - /bin/sh + - -c + - "cp /opt/cni/bin/* /host/opt/cni/bin/" + volumeMounts: + - mountPath: /host/opt/cni/bin + name: cni-bin-dir containers: - image: 602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:1.2.0 imagePullPolicy: Always @@ -5201,7 +5208,6 @@ write_files: - --prometheus-listen-addr=0.0.0.0:9620 - --prometheus-sync-interval=5s - --session-duration={{.Experimental.KIAMSupport.SessionDuration}} - {{if .Kubernetes.Networking.AmazonVPC.Enabled }}- --host-interface=!eni0{{ end }} volumeMounts: - mountPath: /etc/ssl/certs name: ssl-certs @@ -5362,6 +5368,7 @@ write_files: - --server-address={{.Experimental.KIAMSupport.ServerAddresses.AgentAddress}} - --prometheus-listen-addr=0.0.0.0:9620 - --prometheus-sync-interval=5s + {{if .Kubernetes.Networking.AmazonVPC.Enabled }}- --host-interface=!eni0{{ end }} env: - name: HOST_IP valueFrom: diff --git a/core/nodepool/config/templates/cloud-config-worker b/core/nodepool/config/templates/cloud-config-worker index ebae66c4d..052897d8c 100644 --- a/core/nodepool/config/templates/cloud-config-worker +++ b/core/nodepool/config/templates/cloud-config-worker @@ -342,13 +342,13 @@ coreos: ExecStartPre=/usr/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/usr/bin/mkdir -p /var/run/calico ExecStartPre=/usr/bin/mkdir -p /var/lib/calico - ExecStart=/usr/lib/coreos/kubelet-wrapper \ + ExecStart=/bin/sh -c "exec /usr/lib/coreos/kubelet-wrapper \ --cni-conf-dir=/etc/kubernetes/cni/net.d \ {{/* Work-around until https://github.com/kubernetes/kubernetes/issues/43967 is fixed via https://github.com/kubernetes/kubernetes/pull/43995 */ -}} --cni-bin-dir=/opt/cni/bin \ --network-plugin={{.K8sNetworkPlugin}} \ --container-runtime={{.ContainerRuntime}} \ - --node-labels=kubernetes.io/role=node,node-role.kubernetes.io/node="",node-role.kubernetes.io/{{ toLabel .NodePoolName }}=""{{if .NodeLabels.Enabled}},{{.NodeLabels.String}}{{end}} \ + --node-labels=kubernetes.io/role=node,node-role.kubernetes.io/node=\"\",node-role.kubernetes.io/{{ toLabel .NodePoolName }}=\"\"{{if .NodeLabels.Enabled}},{{.NodeLabels.String}}{{end}} \ --register-node=true \ {{if .Taints}}--register-with-taints={{.Taints.String}}\ {{end}}--allow-privileged=true \ @@ -370,7 +370,7 @@ coreos: {{- end }} --kubeconfig=/etc/kubernetes/kubeconfig/worker.yaml \ {{- if .FeatureGates.Enabled }} - --feature-gates="{{.FeatureGates.String}}" \ + --feature-gates=\"{{.FeatureGates.String}}\" \ {{- end }} {{- if .Kubelet.SystemReservedResources }} --system-reserved={{ .Kubelet.SystemReservedResources }} \ @@ -378,10 +378,14 @@ coreos: {{- if .Kubelet.KubeReservedResources }} --kube-reserved={{ .Kubelet.KubeReservedResources }} \ {{- end }} + {{- if .Kubernetes.Networking.AmazonVPC.Enabled }} + --node-ip=$$(curl http://169.254.169.254/latest/meta-data/local-ipv4) \ + --max-pods=$$(/opt/bin/aws-k8s-cni-max-pods) \ + {{- end }} {{if checkVersion "<1.10" .K8sVer -}} --require-kubeconfig \ {{end -}} - $KUBELET_OPTS + $KUBELET_OPTS" Restart=always RestartSec=10 [Install] @@ -1066,6 +1070,14 @@ write_files: content: {{.AssetsConfig.TLSBootstrapToken}} {{ end }} +{{if .Kubernetes.Networking.AmazonVPC.Enabled}} + - path: /opt/bin/aws-k8s-cni-max-pods + owner: root:root + permissions: 0755 + encoding: gzip+base64 + content: {{.Kubernetes.Networking.AmazonVPC.MaxPodsScript.ToGzip.ToBase64}} +{{end}} + {{ if .Gpu.Nvidia.IsEnabledOn .InstanceType }} - path: /opt/nvidia-build/README owner: root:root diff --git a/core/root/config/templates/cluster.yaml b/core/root/config/templates/cluster.yaml index bd8e53002..14c23bb3b 100644 --- a/core/root/config/templates/cluster.yaml +++ b/core/root/config/templates/cluster.yaml @@ -1433,7 +1433,7 @@ experimental: enabled: false image: repo: quay.io/uswitch/kiam - tag: v2.7 + tag: v2.8 rktPullDocker: false sessionDuration: 15m serverAddresses: diff --git a/node/uploaded_file.go b/node/uploaded_file.go index a1a888d25..998ef4ba9 100644 --- a/node/uploaded_file.go +++ b/node/uploaded_file.go @@ -9,36 +9,36 @@ import ( type UploadedFile struct { Path string - Content uploadedFileContent + Content UploadedFileContent } -type uploadedFileContent struct { +type UploadedFileContent struct { bytes []byte } -func NewUploadedFileContent(bytes []byte) uploadedFileContent { - return uploadedFileContent{ +func NewUploadedFileContent(bytes []byte) UploadedFileContent { + return UploadedFileContent{ bytes: bytes, } } -func (c uploadedFileContent) ToBase64() uploadedFileContent { +func (c UploadedFileContent) ToBase64() UploadedFileContent { bytes := []byte(base64.StdEncoding.EncodeToString(c.bytes)) - return uploadedFileContent{ + return UploadedFileContent{ bytes: bytes, } } -func (c uploadedFileContent) ToGzip() uploadedFileContent { +func (c UploadedFileContent) ToGzip() UploadedFileContent { bytes, err := gzipcompressor.BytesToBytes(c.bytes) if err != nil { panic(fmt.Errorf("Unexpected error in ToGzip: %v", err)) } - return uploadedFileContent{ + return UploadedFileContent{ bytes: bytes, } } -func (c uploadedFileContent) String() string { +func (c UploadedFileContent) String() string { return string(c.bytes) } diff --git a/test/integration/maincluster_test.go b/test/integration/maincluster_test.go index be3e5a78d..d5ff40c32 100644 --- a/test/integration/maincluster_test.go +++ b/test/integration/maincluster_test.go @@ -142,7 +142,7 @@ func TestMainClusterConfig(t *testing.T) { }, KIAMSupport: controlplane_config.KIAMSupport{ Enabled: false, - Image: model.Image{Repo: "quay.io/uswitch/kiam", Tag: "v2.7", RktPullDocker: false}, + Image: model.Image{Repo: "quay.io/uswitch/kiam", Tag: "v2.8", RktPullDocker: false}, SessionDuration: "15m", ServerAddresses: controlplane_config.KIAMServerAddresses{ServerAddress: "localhost:443", AgentAddress: "kiam-server:443"}, }, @@ -1442,7 +1442,7 @@ worker: }, KIAMSupport: controlplane_config.KIAMSupport{ Enabled: false, - Image: model.Image{Repo: "quay.io/uswitch/kiam", Tag: "v2.7", RktPullDocker: false}, + Image: model.Image{Repo: "quay.io/uswitch/kiam", Tag: "v2.8", RktPullDocker: false}, SessionDuration: "15m", ServerAddresses: controlplane_config.KIAMServerAddresses{ServerAddress: "localhost:443", AgentAddress: "kiam-server:443"}, }, @@ -1686,7 +1686,7 @@ worker: expected := controlplane_config.Experimental{ KIAMSupport: controlplane_config.KIAMSupport{ Enabled: true, - Image: model.Image{Repo: "quay.io/uswitch/kiam", Tag: "v2.7", RktPullDocker: false}, + Image: model.Image{Repo: "quay.io/uswitch/kiam", Tag: "v2.8", RktPullDocker: false}, SessionDuration: "15m", ServerAddresses: controlplane_config.KIAMServerAddresses{ServerAddress: "localhost:443", AgentAddress: "kiam-server:443"}, }, diff --git a/test/integration/plugin_test.go b/test/integration/plugin_test.go index e8059429b..580036856 100644 --- a/test/integration/plugin_test.go +++ b/test/integration/plugin_test.go @@ -458,7 +458,7 @@ spec: } // A kube-aws plugin can activate feature gates - if !strings.Contains(workerUserdataS3Part, `--feature-gates="Accelerators=true"`) { + if !strings.Contains(workerUserdataS3Part, `--feature-gates=\"Accelerators=true\"`) { t.Error("missing worker feature gate: Accelerators=true") }