WIP: Bump to Kubernetes v1.6.1 (#492)

* Bump to Kubernetes v1.6.1 This change was just the result of running the following commands: ``` $ contrib/bump-version v1.6.1_coreos.0 Updating contrib/bump-version Updating core/controlplane/config/config.go Updating core/controlplane/config/templates/cluster.yaml Updating e2e/kubernetes/Dockerfile Updating e2e/kubernetes/Makefile Updating vendor/github.com/aws/aws-sdk-go/CHANGELOG.md $ git checkout -p -- vendor ``` As etcd3 support is already introduced via #417, after this change is introduced, it was ideally a matter of running E2E against a newly created kube-aws cluster with k8s 1.6.1, which turned out not to be true, hence the subsequent changes. * Use etcd3 by default etcd2 support will be dropped soon, as the etcd3 storage driver is already the default since k8s v.1.6.0. * Bump to calico-cni v1.6.2, which is an even newer release than the one included in the latest calico v2.1.2, to deal with kubernetes/kubernetes#43488 * Set up /etc/kubernetes/cni/net.d not using calico-cni but by our own to deal with kubernetes/kubernetes#43014 * Set up /opt/cni/bin using docker rather than a k8s static pod to prevent temporary "failed to find plugin * in path" errors from cni They were emitted when pods are scheduled but /opt/cni/bin is not yet populated ``` Error syncing pod, skipping: failed to "CreatePodSandbox" for "kube-dns-3816048056-cwx62_kube-system(12c3204f-1a54-11e7-bfb0-06751e989ae7)" with CreatePodSandboxError: "CreatePodSandbox for pod \"kube-dns-3816048056-cwx62_kube-system(12c3204f-1a54-11e7-bfb0-06751e989ae7)\" failed: rpc error: code = 2 desc = NetworkPlugin cni failed to set up pod \"kube-dns-3816048056-cwx62_kube-system\" network: failed to find plugin \"loopback\" in path [/opt/loopback/bin /opt/cni/bin]" ``` * Fix a bug that resulted etcd-member.service to use the default version number 3.0.x regardless of what is specified via `etcd.version` in cluster.yaml. The bug was reported in #497 (comment) * Simplify EtcdVersion func According to the review comment #492 (review) * Fix permanent errors like "failed to find plugin * in path" from cni which was breaking cni + flannel/calico in k8s 1.6, by specifying the `--cni-bin-dir=/opt/cni/bin` flag for kubelets The default dir had been accidentally changed at least in k8s 1.6.0 and 1.6.1. Resolves #494 Resolves #495 E2E against a cluster with flannel passed after this change: ``` $ ETCD_VERSION=3 ETCD_SNAPSHOT_AUTOMATED=1 ETCD_DISASTER_RECOVERY_AUTOMATED=1 ETCD_COUNT=3 KUBE_AWS_CLUSTER_NAME=kubeaws2 ./run all *snip* Ran 151 of 588 Specs in 3492.050 seconds SUCCESS! -- 151 Passed | 0 Failed | 0 Pending | 437 Skipped PASS Ginkgo ran 1 suite in 58m12.359210255s Test Suite Passed 2017/04/04 09:35:29 util.go:127: Step './hack/ginkgo-e2e.sh --ginkgo.focus=\[Conformance\]' finished in 58m12.683100213s 2017/04/04 09:35:29 e2e.go:80: Done ``` Also passed against a cluster with calico: ``` Ran 151 of 588 Specs in 3381.108 seconds SUCCESS! -- 151 Passed | 0 Failed | 0 Pending | 437 Skipped PASS Ginkgo ran 1 suite in 56m21.415087252s Test Suite Passed 2017/04/06 03:58:20 util.go:131: Step './hack/ginkgo-e2e.sh --ginkgo.focus=\[Conformance\]' finished in 56m21.76726736s 2017/04/06 03:58:20 e2e.go:80: Done ```
kubernetes-retired · Apr 6, 2017 · b0614a8 · b0614a8
1 parent 16168b7
commit b0614a8
Show file tree

Hide file tree

Showing 12 changed files with 116 additions and 50 deletions.
diff --git a/contrib/bump-version b/contrib/bump-version
@@ -6,11 +6,11 @@
 
 if [ $# -ne 1 ] || [ `expr $1 : ".*_.*"` == 0 ]; then
     echo "USAGE: $0 <target-version>"
-    echo "  example: $0 'v1.5.5_coreos.0'"
+    echo "  example: $0 'v1.6.1_coreos.0'"
     exit 1
 fi
 
-CURRENT_VERSION=${CURRENT_VERSION:-"v1.5.5_coreos.0"}
+CURRENT_VERSION=${CURRENT_VERSION:-"v1.6.1_coreos.0"}
 TARGET_VERSION=${1}
 
 CURRENT_VERSION_BASE=${CURRENT_VERSION%%_*}

diff --git a/core/controlplane/config/config.go b/core/controlplane/config/config.go
@@ -27,7 +27,7 @@ import (
 )
 
 const (
-	k8sVer = "v1.5.5_coreos.0"
+	k8sVer = "v1.6.1_coreos.0"
 
 	credentialsDir = "credentials"
 	userDataDir    = "userdata"
@@ -105,7 +105,7 @@ func NewDefaultCluster() *Cluster {
 			HyperkubeImage:              model.Image{Repo: "quay.io/coreos/hyperkube", Tag: k8sVer, RktPullDocker: false},
 			AWSCliImage:                 model.Image{Repo: "quay.io/coreos/awscli", Tag: "master", RktPullDocker: false},
 			CalicoNodeImage:             model.Image{Repo: "quay.io/calico/node", Tag: "v1.1.0", RktPullDocker: false},
-			CalicoCniImage:              model.Image{Repo: "quay.io/calico/cni", Tag: "v1.6.1", RktPullDocker: false},
+			CalicoCniImage:              model.Image{Repo: "quay.io/calico/cni", Tag: "v1.6.2", RktPullDocker: false},
 			CalicoPolicyControllerImage: model.Image{Repo: "quay.io/calico/kube-policy-controller", Tag: "v0.5.4", RktPullDocker: false},
 			ClusterAutoscalerImage:      model.Image{Repo: "gcr.io/google_containers/cluster-proportional-autoscaler-amd64", Tag: "1.0.0", RktPullDocker: false},
 			KubeDnsImage:                model.Image{Repo: "gcr.io/google_containers/kubedns-amd64", Tag: "1.9", RktPullDocker: false},

diff --git a/core/controlplane/config/templates/cloud-config-controller b/core/controlplane/config/templates/cloud-config-controller
@@ -165,9 +165,14 @@ coreos:
                        cluster-health
 
         ExecStartPre=/bin/sh -ec "find /etc/kubernetes/manifests /srv/kubernetes/manifests  -maxdepth 1 -type f | xargs --no-run-if-empty sed -i 's|#ETCD_ENDPOINTS#|${ETCD_ENDPOINTS}|'"
+        {{if .UseCalico -}}
+        ExecStartPre=/usr/bin/docker run --rm -e SLEEP=false -v /opt/cni/bin:/host/opt/cni/bin {{ .CalicoCniImage.RepoWithTag }} /install-cni.sh
+        {{end -}}
         ExecStart=/usr/lib/coreos/kubelet-wrapper \
         --api-servers=http://localhost:8080 \
         --cni-conf-dir=/etc/kubernetes/cni/net.d \
+        {{/* Work-around until https://github.com/kubernetes/kubernetes/issues/43967 is fixed via https://github.com/kubernetes/kubernetes/pull/43995 */ -}}
+        --cni-bin-dir=/opt/cni/bin \
         --network-plugin={{.K8sNetworkPlugin}} \
         --container-runtime={{.ContainerRuntime}} \
         --rkt-path=/usr/bin/rkt \
@@ -628,10 +633,14 @@ write_files:
               k8s-app: calico-node
             annotations:
               scheduler.alpha.kubernetes.io/critical-pod: ''
-              scheduler.alpha.kubernetes.io/tolerations: |
-                [{"key": "node.alpha.kubernetes.io/role", "value": "master", "effect": "NoSchedule" },
-                {"key":"CriticalAddonsOnly", "operator":"Exists"}]
           spec:
+            tolerations:
+            - key: "node.alpha.kubernetes.io/role"
+              operator: "Equal"
+              value: "master"
+              effect: "NoSchedule"
+            - key: "CriticalAddonsOnly"
+              operator: "Exists"
             hostNetwork: true
             containers:
               - name: calico-node
@@ -677,30 +686,6 @@ write_files:
                   - mountPath: /etc/resolv.conf
                     name: dns
                     readOnly: true
-              - name: install-cni
-                image: {{ .CalicoCniImage.RepoWithTag }}
-                imagePullPolicy: Always
-                command: ["/install-cni.sh"]
-                env:
-                  - name: ETCD_ENDPOINTS
-                    valueFrom:
-                      configMapKeyRef:
-                        name: calico-config
-                        key: etcd_endpoints
-                  - name: CNI_NETWORK_CONFIG
-                    valueFrom:
-                      configMapKeyRef:
-                        name: calico-config
-                        key: cni_network_config
-                  - name: CNI_NET_DIR
-                    value: "/etc/kubernetes/cni/net.d"
-                volumeMounts:
-                  - mountPath: /host/opt/cni/bin
-                    name: cni-bin-dir
-                  - mountPath: /host/etc/cni/net.d
-                    name: cni-net-dir
-                  - mountPath: /calico-secrets
-                    name: etcd-certs
             volumes:
               - name: lib-modules
                 hostPath:
@@ -732,9 +717,6 @@ write_files:
           k8s-app: calico-policy
         annotations:
           scheduler.alpha.kubernetes.io/critical-pod: ''
-          scheduler.alpha.kubernetes.io/tolerations: |
-            [{"key": "node.alpha.kubernetes.io/role", "value": "master", "effect": "NoSchedule" },
-            {"key":"CriticalAddonsOnly", "operator":"Exists"}]
 
       spec:
         replicas: 1
@@ -745,6 +727,13 @@ write_files:
             labels:
               k8s-app: calico-policy
           spec:
+            tolerations:
+            - key: "node.alpha.kubernetes.io/role"
+              operator: "Equal"
+              value: "master"
+              effect: "NoSchedule"
+            - key: "CriticalAddonsOnly"
+              operator: "Exists"
             hostNetwork: true
             containers:
               - name: calico-policy-controller
@@ -1262,8 +1251,10 @@ write_files:
               k8s-app: kube-rescheduler
             annotations:
               scheduler.alpha.kubernetes.io/critical-pod: ''
-              scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]'
           spec:
+            tolerations:
+            - key: "CriticalAddonsOnly"
+              operator: "Exists"
             hostNetwork: true
             containers:
             - name: kube-rescheduler
@@ -1291,8 +1282,10 @@ write_files:
                 k8s-app: kube-dns-autoscaler
               annotations:
                 scheduler.alpha.kubernetes.io/critical-pod: ''
-                scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]'
             spec:
+              tolerations:
+              - key: "CriticalAddonsOnly"
+                operator: "Exists"
               containers:
               - name: autoscaler
                 image: {{ .ClusterAutoscalerImage.RepoWithTag }}
@@ -1338,8 +1331,10 @@ write_files:
                 k8s-app: kube-dns
               annotations:
                 scheduler.alpha.kubernetes.io/critical-pod: ''
-                scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]'
             spec:
+              tolerations:
+              - key: "CriticalAddonsOnly"
+                operator: "Exists"
               containers:
               - name: kubedns
                 image: {{ .KubeDnsImage.RepoWithTag }}
@@ -1501,8 +1496,10 @@ write_files:
                 version: v1.3.0
               annotations:
                 scheduler.alpha.kubernetes.io/critical-pod: ''
-                scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]'
             spec:
+              tolerations:
+              - key: "CriticalAddonsOnly"
+                operator: "Exists"
               containers:
                 - image: {{ .HeapsterImage.RepoWithTag }}
                   name: heapster
@@ -1594,8 +1591,10 @@ write_files:
                 kubernetes.io/cluster-service: "true"
               annotations:
                 scheduler.alpha.kubernetes.io/critical-pod: ''
-                scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]'
             spec:
+              tolerations:
+              - key: "CriticalAddonsOnly"
+                operator: "Exists"
               containers:
               - name: kubernetes-dashboard
                 image: {{ .KubeDashboardImage.RepoWithTag }}
@@ -1680,6 +1679,25 @@ write_files:
 
 {{ else }}
 
+  - path: /etc/kubernetes/cni/net.d/10-calico.conf
+    content: |
+      {
+        "name": "calico",
+        "type": "flannel",
+        "delegate": {
+          "type": "calico",
+          "etcd_endpoints": "#ETCD_ENDPOINTS#",
+          "etcd_key_file": "/etc/kubernetes/ssl/etcd-client-key.pem",
+          "etcd_cert_file": "/etc/kubernetes/ssl/etcd-client.pem",
+          "etcd_ca_cert_file": "/etc/kubernetes/ssl/ca.pem",
+          "log_level": "info",
+          "policy": {
+            "type": "k8s",
+            "k8s_api_root": "http://127.0.0.1:8080/api/v1/"
+          }
+        }
+      }
+
   # http://docs.projectcalico.org/v2.0/usage/configuration/
   - path: /etc/modules-load.d/nf.conf
     content: |

diff --git a/core/controlplane/config/templates/cloud-config-etcd b/core/controlplane/config/templates/cloud-config-etcd
@@ -164,6 +164,12 @@ coreos:
             ExecStartPre=/usr/bin/systemctl is-active decrypt-assets.service
             {{- end}}
             ExecStartPre=/usr/bin/chown -R etcd:etcd /var/lib/etcd2
+        {{if .Etcd.Version.Is3 }}
+        - name: 40-version.conf
+          content: |
+            [Service]
+            Environment="ETCD_IMAGE_TAG=v{{.Etcd.Version}}"
+        {{end}}
       enable: true
       command: start
 

diff --git a/core/controlplane/config/templates/cloud-config-worker b/core/controlplane/config/templates/cloud-config-worker
@@ -160,9 +160,14 @@ coreos:
                        --cert-file /etc/kubernetes/ssl/etcd-client.pem \
                        --endpoints "${ETCD_ENDPOINTS}" \
                        cluster-health
+        {{if .UseCalico -}}
+        ExecStartPre=/usr/bin/docker run --rm -e SLEEP=false -v /opt/cni/bin:/host/opt/cni/bin {{ .CalicoCniImage.RepoWithTag }} /install-cni.sh
+        {{end -}}
         ExecStart=/usr/lib/coreos/kubelet-wrapper \
         --api-servers={{.APIServerEndpoint}} \
-        --network-plugin-dir=/etc/kubernetes/cni/net.d \
+        --cni-conf-dir=/etc/kubernetes/cni/net.d \
+        {{/* Work-around until https://github.com/kubernetes/kubernetes/issues/43967 is fixed via https://github.com/kubernetes/kubernetes/pull/43995 */ -}}
+        --cni-bin-dir=/opt/cni/bin \
         --network-plugin={{.K8sNetworkPlugin}} \
         --container-runtime={{.ContainerRuntime}} \
         --rkt-path=/usr/bin/rkt \
@@ -811,6 +816,28 @@ write_files:
 
 {{ else }}
 
+  - path: /etc/kubernetes/cni/net.d/10-calico.conf
+    content: |
+      {
+        "name": "calico",
+        "type": "flannel",
+        "delegate": {
+          "type": "calico",
+          "etcd_endpoints": "#ETCD_ENDPOINTS#",
+          "etcd_key_file": "/etc/kubernetes/ssl/etcd-client-key.pem",
+          "etcd_cert_file": "/etc/kubernetes/ssl/etcd-client.pem",
+          "etcd_ca_cert_file": "/etc/kubernetes/ssl/ca.pem",
+          "log_level": "info",
+          "policy": {
+            "type": "k8s",
+            "k8s_api_root": "https://{{.APIEndpoint.DNSName}}/api/v1/",
+            "k8s_client_key": "/etc/kubernetes/ssl/worker-key.pem",
+            "k8s_client_certificate": "/etc/kubernetes/ssl/worker.pem",
+            "k8s_certificate_authority": "/etc/kubernetes/ssl/ca.pem"
+          }
+        }
+      }
+
   # http://docs.projectcalico.org/v2.0/usage/configuration/
   - path: /etc/modules-load.d/nf.conf
     content: |

diff --git a/core/controlplane/config/templates/cluster.yaml b/core/controlplane/config/templates/cluster.yaml
@@ -801,7 +801,7 @@ worker:
 # and pulling from quay or dockerhub is slow and you get many timeouts.
 
 # Version of hyperkube image to use. This is the tag for the hyperkube image repository.
-# kubernetesVersion: v1.5.5_coreos.0
+# kubernetesVersion: v1.6.1_coreos.0
 
 # Hyperkube image repository to use.
 # hyperkubeImage:

diff --git a/core/controlplane/config/templates/stack-template.json b/core/controlplane/config/templates/stack-template.json
@@ -884,6 +884,23 @@
       },
       "Type": "AWS::EC2::SecurityGroup"
     },
+    {{if $.UseCalico -}}
+    {{/* Required by calico-policy-controller when calico is enabled. See https://github.com/kubernetes-incubator/kube-aws/issues/494#issuecomment-291687137 */}}
+    "SecurityGroupControllerIngressFromControllerToController": {
+      "Properties": {
+        "FromPort": 443,
+        "GroupId": {
+          "Ref": "SecurityGroupController"
+        },
+        "IpProtocol": "tcp",
+        "SourceSecurityGroupId": {
+          "Ref": "SecurityGroupController"
+        },
+        "ToPort": 443
+      },
+      "Type": "AWS::EC2::SecurityGroupIngress"
+    },
+    {{end -}}
     "SecurityGroupControllerIngressFromControllerToKubelet": {
       "Properties": {
         "FromPort": 10250,

diff --git a/core/nodepool/config/deployment.go b/core/nodepool/config/deployment.go
@@ -95,6 +95,7 @@ func (c DeploymentSettings) WithDefaultsFrom(main cfg.DeploymentSettings) Deploy
 	c.HyperkubeImage.Tag = c.K8sVer
 	c.AWSCliImage.MergeIfEmpty(main.AWSCliImage)
 	c.CalicoCtlImage.MergeIfEmpty(main.CalicoCtlImage)
+	c.CalicoCniImage.MergeIfEmpty(main.CalicoCniImage)
 	c.PauseImage.MergeIfEmpty(main.PauseImage)
 	c.FlannelImage.MergeIfEmpty(main.FlannelImage)
 

diff --git a/e2e/kubernetes/Dockerfile b/e2e/kubernetes/Dockerfile
@@ -1,6 +1,6 @@
 FROM golang:1.7.1
 
-ARG KUBERNETES_VERSION=${KUBERNETES_VERSION:-v1.5.5+coreos.0}
+ARG KUBERNETES_VERSION=${KUBERNETES_VERSION:-v1.6.1+coreos.0}
 
 RUN apt-get update && \
     apt-get install -y rsync && \

diff --git a/e2e/kubernetes/Makefile b/e2e/kubernetes/Makefile
@@ -1,4 +1,4 @@
-KUBERNETES_VERSION ?= v1.5.5+coreos.0
+KUBERNETES_VERSION ?= v1.6.1+coreos.0
 DOCKER_REPO ?=
 DOCKER_TAG ?= $(DOCKER_REPO)kube-e2e:$(KUBERNETES_VERSION)
 DOCKER_TAG_SANITIZED ?= $(shell echo $(DOCKER_TAG) | sed -e 's/+/_/')

diff --git a/e2e/run b/e2e/run
@@ -9,7 +9,7 @@ SRC_DIR=$(cd $(dirname $0); cd ..; pwd)
 KUBECONFIG=${WORK_DIR}/kubeconfig
 ETCD_COUNT=${ETCD_COUNT:-3}
 CONTROLLER_COUNT=${CONTROLLER_COUNT:-2}
-ETCD_VERSION=${ETCD_VERSION:-3}
+ETCD_VERSION=${ETCD_VERSION:-}
 
 export KUBECONFIG
 
@@ -234,7 +234,7 @@ etcd:
   count: $ETCD_COUNT" >> cluster.yaml
 
   if [ "${ETCD_VERSION}" != "" ]; then
-    echo -e "  version: 3" >> cluster.yaml
+    echo -e "  version: ${ETCD_VERSION}" >> cluster.yaml
   fi
 
   if [ "${ETCD_DISASTER_RECOVERY_AUTOMATED}" != "" ]; then

diff --git a/model/etcd.go b/model/etcd.go
@@ -129,15 +129,12 @@ func (e Etcd) SystemdUnitName() string {
 	return "etcd2.service"
 }
 
-// Version returns the version of etcd (e.g. `2`, `3`, `3.1.3`) to be used for this etcd cluster
+// Version returns the version of etcd (e.g. `3.1.5`) to be used for this etcd cluster
 func (e Etcd) Version() EtcdVersion {
 	if e.Cluster.Version != "" {
 		return e.Cluster.Version
 	}
-	if e.Cluster.Version == "3" {
-		return "3.1.3"
-	}
-	return "2"
+	return "3.1.5"
 }
 
 func (v EtcdVersion) Is3() bool {