diff --git a/.circleci/config.yml b/.circleci/config.yml index 4c3d43b865..c1479f737a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,5 +1,9 @@ # https://circleci.com/docs/2.0/language-go/ version: 2.1 + +orbs: + aws-cli: circleci/aws-cli@0.1.19 + jobs: build: docker: @@ -16,9 +20,33 @@ jobs: - run: make lint - run: make vet - run: make unit-test - + integration_test: + docker: + - image: circleci/golang:1.13-stretch + working_directory: /go/src/github.com/{{ORG_NAME}}/{{REPO_NAME}} + environment: + GO111MODULE: "on" + steps: + - checkout + - setup_remote_docker + - aws-cli/setup: + profile-name: awstester + - run: + name: Run the integration tests + command: ./scripts/run-integration-tests.sh + no_output_timeout: 15m + - store_artifacts: + path: /tmp/cni-test + workflows: version: 2 check: jobs: - build + - hold: + type: approval + requires: + - build + - integration_test: + requires: + - hold diff --git a/.gitignore b/.gitignore index 0a846fa990..7932a705df 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,4 @@ verify-network .DS_Store cni-metrics-helper/cni-metrics-helper portmap -grpc_health_probe +grpc-health-probe diff --git a/config/v1.6/aws-k8s-cni.yaml b/config/v1.6/aws-k8s-cni.yaml new file mode 100644 index 0000000000..6b698087c7 --- /dev/null +++ b/config/v1.6/aws-k8s-cni.yaml @@ -0,0 +1,163 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: aws-node +rules: + - apiGroups: + - crd.k8s.amazonaws.com + resources: + - "*" + - namespaces + verbs: + - "*" + - apiGroups: [""] + resources: + - pods + - nodes + - namespaces + verbs: ["list", "watch", "get"] + - apiGroups: ["extensions"] + resources: + - daemonsets + verbs: ["list", "watch"] + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: aws-node + namespace: kube-system + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: aws-node +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: aws-node +subjects: + - kind: ServiceAccount + name: aws-node + namespace: kube-system + +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: aws-node + namespace: kube-system + labels: + k8s-app: aws-node +spec: + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: "10%" + selector: + matchLabels: + k8s-app: aws-node + template: + metadata: + labels: + k8s-app: aws-node + spec: + priorityClassName: system-node-critical + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "beta.kubernetes.io/os" + operator: In + values: + - linux + - key: "beta.kubernetes.io/arch" + operator: In + values: + - amd64 + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + serviceAccountName: aws-node + hostNetwork: true + tolerations: + - operator: Exists + containers: + - image: 602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:v1.6.0-rc5 + imagePullPolicy: Always + ports: + - containerPort: 61678 + name: metrics + name: aws-node + readinessProbe: + exec: + command: ["/app/grpc-health-probe", "-addr=:50051"] + initialDelaySeconds: 35 + livenessProbe: + exec: + command: ["/app/grpc-health-probe", "-addr=:50051"] + initialDelaySeconds: 35 + env: + - name: AWS_VPC_K8S_CNI_LOGLEVEL + value: DEBUG + - name: AWS_VPC_K8S_CNI_VETHPREFIX + value: eni + - name: AWS_VPC_ENI_MTU + value: "9001" + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 10m + securityContext: + privileged: true + volumeMounts: + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + - mountPath: /host/etc/cni/net.d + name: cni-net-dir + - mountPath: /host/var/log + name: log-dir + - mountPath: /var/run/docker.sock + name: dockersock + - mountPath: /var/run/dockershim.sock + name: dockershim + volumes: + - name: cni-bin-dir + hostPath: + path: /opt/cni/bin + - name: cni-net-dir + hostPath: + path: /etc/cni/net.d + - name: log-dir + hostPath: + path: /var/log + - name: dockersock + hostPath: + path: /var/run/docker.sock + - name: dockershim + hostPath: + path: /var/run/dockershim.sock + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: eniconfigs.crd.k8s.amazonaws.com +spec: + scope: Cluster + group: crd.k8s.amazonaws.com + versions: + - name: v1alpha1 + served: true + storage: true + names: + plural: eniconfigs + singular: eniconfig + kind: ENIConfig diff --git a/config/v1.6/calico.yaml b/config/v1.6/calico.yaml new file mode 100644 index 0000000000..18ccde3655 --- /dev/null +++ b/config/v1.6/calico.yaml @@ -0,0 +1,726 @@ +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: calico-node + namespace: kube-system + labels: + k8s-app: calico-node +spec: + selector: + matchLabels: + k8s-app: calico-node + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + template: + metadata: + labels: + k8s-app: calico-node + annotations: + # This, along with the CriticalAddonsOnly toleration below, + # marks the pod as a critical add-on, ensuring it gets + # priority scheduling and that its resources are reserved + # if it ever gets evicted. + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + nodeSelector: + beta.kubernetes.io/os: linux + hostNetwork: true + serviceAccountName: calico-node + # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force + # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. + terminationGracePeriodSeconds: 0 + containers: + # Runs calico/node container on each Kubernetes node. This + # container programs network policy and routes on each + # host. + - name: calico-node + image: quay.io/calico/node:v3.8.1 + env: + # Use Kubernetes API as the backing datastore. + - name: DATASTORE_TYPE + value: "kubernetes" + # Use eni not cali for interface prefix + - name: FELIX_INTERFACEPREFIX + value: "eni" + # Enable felix info logging. + - name: FELIX_LOGSEVERITYSCREEN + value: "info" + # Don't enable BGP. + - name: CALICO_NETWORKING_BACKEND + value: "none" + # Cluster type to identify the deployment type + - name: CLUSTER_TYPE + value: "k8s,ecs" + # Disable file logging so `kubectl logs` works. + - name: CALICO_DISABLE_FILE_LOGGING + value: "true" + - name: FELIX_TYPHAK8SSERVICENAME + value: "calico-typha" + # Set Felix endpoint to host default action to ACCEPT. + - name: FELIX_DEFAULTENDPOINTTOHOSTACTION + value: "ACCEPT" + # This will make Felix honor AWS VPC CNI's mangle table + # rules. + - name: FELIX_IPTABLESMANGLEALLOWACTION + value: Return + # Disable IPV6 on Kubernetes. + - name: FELIX_IPV6SUPPORT + value: "false" + # Wait for the datastore. + - name: WAIT_FOR_DATASTORE + value: "true" + - name: FELIX_LOGSEVERITYSYS + value: "none" + - name: FELIX_PROMETHEUSMETRICSENABLED + value: "true" + - name: NO_DEFAULT_POOLS + value: "true" + # Set based on the k8s node name. + - name: NODENAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # No IP address needed. + - name: IP + value: "" + - name: FELIX_HEALTHENABLED + value: "true" + securityContext: + privileged: true + livenessProbe: + httpGet: + path: /liveness + port: 9099 + host: localhost + periodSeconds: 10 + initialDelaySeconds: 10 + failureThreshold: 6 + readinessProbe: + exec: + command: + - /bin/calico-node + - -felix-ready + periodSeconds: 10 + volumeMounts: + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false + - mountPath: /var/run/calico + name: var-run-calico + readOnly: false + - mountPath: /var/lib/calico + name: var-lib-calico + readOnly: false + volumes: + # Used to ensure proper kmods are installed. + - name: lib-modules + hostPath: + path: /lib/modules + - name: var-run-calico + hostPath: + path: /var/run/calico + - name: var-lib-calico + hostPath: + path: /var/lib/calico + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + tolerations: + # Make sure calico/node gets scheduled on all nodes. + - effect: NoSchedule + operator: Exists + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + +--- + +# Create all the CustomResourceDefinitions needed for +# Calico policy-only mode. + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: felixconfigurations.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + versions: + - name: v1 + served: true + storage: true + names: + kind: FelixConfiguration + plural: felixconfigurations + singular: felixconfiguration + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: ipamblocks.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + versions: + - name: v1 + served: true + storage: true + names: + kind: IPAMBlock + plural: ipamblocks + singular: ipamblock + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: blockaffinities.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + versions: + - name: v1 + served: true + storage: true + names: + kind: BlockAffinity + plural: blockaffinities + singular: blockaffinity + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: bgpconfigurations.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + versions: + - name: v1 + served: true + storage: true + names: + kind: BGPConfiguration + plural: bgpconfigurations + singular: bgpconfiguration + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: bgppeers.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + versions: + - name: v1 + served: true + storage: true + names: + kind: BGPPeer + plural: bgppeers + singular: bgppeer +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: ippools.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + versions: + - name: v1 + served: true + storage: true + names: + kind: IPPool + plural: ippools + singular: ippool + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: hostendpoints.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + versions: + - name: v1 + served: true + storage: true + names: + kind: HostEndpoint + plural: hostendpoints + singular: hostendpoint + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: clusterinformations.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + versions: + - name: v1 + served: true + storage: true + names: + kind: ClusterInformation + plural: clusterinformations + singular: clusterinformation + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: globalnetworkpolicies.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + versions: + - name: v1 + served: true + storage: true + names: + kind: GlobalNetworkPolicy + plural: globalnetworkpolicies + singular: globalnetworkpolicy + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: globalnetworksets.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + versions: + - name: v1 + served: true + storage: true + names: + kind: GlobalNetworkSet + plural: globalnetworksets + singular: globalnetworkset + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: networkpolicies.crd.projectcalico.org +spec: + scope: Namespaced + group: crd.projectcalico.org + versions: + - name: v1 + served: true + storage: true + names: + kind: NetworkPolicy + plural: networkpolicies + singular: networkpolicy + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: networksets.crd.projectcalico.org +spec: + scope: Namespaced + group: crd.projectcalico.org + versions: + - name: v1 + served: true + storage: true + names: + kind: NetworkSet + plural: networksets + singular: networkset + +--- + +# Create the ServiceAccount and roles necessary for Calico. + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-node + namespace: kube-system + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: calico-node +rules: + - apiGroups: [""] + resources: + - namespaces + - serviceaccounts + verbs: + - get + - list + - watch + - apiGroups: [""] + resources: + - pods/status + verbs: + - patch + - apiGroups: [""] + resources: + - nodes/status + verbs: + - patch + - update + - apiGroups: [""] + resources: + - pods + verbs: + - get + - list + - watch + - apiGroups: [""] + resources: + - services + verbs: + - get + - apiGroups: [""] + resources: + - endpoints + verbs: + - get + - apiGroups: [""] + resources: + - nodes + verbs: + - get + - list + - update + - watch + - apiGroups: ["extensions"] + resources: + - networkpolicies + verbs: + - get + - list + - watch + - apiGroups: ["networking.k8s.io"] + resources: + - networkpolicies + verbs: + - watch + - list + - apiGroups: ["crd.projectcalico.org"] + resources: + - globalfelixconfigs + - felixconfigurations + - bgppeers + - globalbgpconfigs + - bgpconfigurations + - ippools + - ipamblocks + - globalnetworkpolicies + - globalnetworksets + - networkpolicies + - networksets + - clusterinformations + - hostendpoints + verbs: + - create + - get + - list + - update + - watch + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + - ipamblocks + - ipamhandles + verbs: + - get + - list + - create + - update + - delete + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + verbs: + - watch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: calico-node +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-node +subjects: + - kind: ServiceAccount + name: calico-node + namespace: kube-system + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: calico-typha + namespace: kube-system + labels: + k8s-app: calico-typha +spec: + revisionHistoryLimit: 2 + selector: + matchLabels: + k8s-app: calico-typha + template: + metadata: + labels: + k8s-app: calico-typha + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + cluster-autoscaler.kubernetes.io/safe-to-evict: 'true' + spec: + nodeSelector: + beta.kubernetes.io/os: linux + tolerations: + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + hostNetwork: true + serviceAccountName: calico-node + containers: + - image: quay.io/calico/typha:v3.8.1 + name: calico-typha + ports: + - containerPort: 5473 + name: calico-typha + protocol: TCP + env: + # Use eni not cali for interface prefix + - name: FELIX_INTERFACEPREFIX + value: "eni" + - name: TYPHA_LOGFILEPATH + value: "none" + - name: TYPHA_LOGSEVERITYSYS + value: "none" + - name: TYPHA_LOGSEVERITYSCREEN + value: "info" + - name: TYPHA_PROMETHEUSMETRICSENABLED + value: "true" + - name: TYPHA_CONNECTIONREBALANCINGMODE + value: "kubernetes" + - name: TYPHA_PROMETHEUSMETRICSPORT + value: "9093" + - name: TYPHA_DATASTORETYPE + value: "kubernetes" + - name: TYPHA_MAXCONNECTIONSLOWERLIMIT + value: "1" + - name: TYPHA_HEALTHENABLED + value: "true" + # This will make Felix honor AWS VPC CNI's mangle table + # rules. + - name: FELIX_IPTABLESMANGLEALLOWACTION + value: Return + livenessProbe: + httpGet: + path: /liveness + port: 9098 + host: localhost + periodSeconds: 30 + initialDelaySeconds: 30 + readinessProbe: + httpGet: + path: /readiness + port: 9098 + host: localhost + periodSeconds: 10 + +--- + +# This manifest creates a Pod Disruption Budget for Typha to allow K8s Cluster Autoscaler to evict +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: calico-typha + namespace: kube-system + labels: + k8s-app: calico-typha +spec: + maxUnavailable: 1 + selector: + matchLabels: + k8s-app: calico-typha + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: typha-cpha +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: typha-cpha +subjects: + - kind: ServiceAccount + name: typha-cpha + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: typha-cpha +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["list"] + +--- + +kind: ConfigMap +apiVersion: v1 +metadata: + name: calico-typha-horizontal-autoscaler + namespace: kube-system +data: + ladder: |- + { + "coresToReplicas": [], + "nodesToReplicas": + [ + [1, 1], + [10, 2], + [100, 3], + [250, 4], + [500, 5], + [1000, 6], + [1500, 7], + [2000, 8] + ] + } + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: calico-typha-horizontal-autoscaler + namespace: kube-system + labels: + k8s-app: calico-typha-autoscaler +spec: + selector: + matchLabels: + k8s-app: calico-typha-autoscaler + replicas: 1 + template: + metadata: + labels: + k8s-app: calico-typha-autoscaler + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + nodeSelector: + beta.kubernetes.io/os: linux + containers: + - image: k8s.gcr.io/cluster-proportional-autoscaler-amd64:1.1.2 + name: autoscaler + command: + - /cluster-proportional-autoscaler + - --namespace=kube-system + - --configmap=calico-typha-horizontal-autoscaler + - --target=deployment/calico-typha + - --logtostderr=true + - --v=2 + resources: + requests: + cpu: 10m + limits: + cpu: 10m + serviceAccountName: typha-cpha + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: typha-cpha + namespace: kube-system +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get"] + - apiGroups: ["extensions"] + resources: ["deployments/scale"] + verbs: ["get", "update"] + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: typha-cpha + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: typha-cpha + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: typha-cpha +subjects: + - kind: ServiceAccount + name: typha-cpha + namespace: kube-system + +--- + +apiVersion: v1 +kind: Service +metadata: + name: calico-typha + namespace: kube-system + labels: + k8s-app: calico-typha +spec: + ports: + - port: 5473 + protocol: TCP + targetPort: calico-typha + name: calico-typha + selector: + k8s-app: calico-typha diff --git a/config/v1.6/cni-metrics-helper.yaml b/config/v1.6/cni-metrics-helper.yaml new file mode 100644 index 0000000000..408fb71143 --- /dev/null +++ b/config/v1.6/cni-metrics-helper.yaml @@ -0,0 +1,85 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +# kubernetes versions before 1.8.0 should use rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: cni-metrics-helper +rules: + - apiGroups: [""] + resources: + - nodes + - pods + - pods/proxy + - services + - resourcequotas + - replicationcontrollers + - limitranges + - persistentvolumeclaims + - persistentvolumes + - namespaces + - endpoints + verbs: ["list", "watch", "get"] + - apiGroups: ["extensions"] + resources: + - daemonsets + - deployments + - replicasets + verbs: ["list", "watch"] + - apiGroups: ["apps"] + resources: + - statefulsets + verbs: ["list", "watch"] + - apiGroups: ["batch"] + resources: + - cronjobs + - jobs + verbs: ["list", "watch"] + - apiGroups: ["autoscaling"] + resources: + - horizontalpodautoscalers + verbs: ["list", "watch"] +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cni-metrics-helper + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +# kubernetes versions before 1.8.0 should use rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: cni-metrics-helper +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cni-metrics-helper +subjects: + - kind: ServiceAccount + name: cni-metrics-helper + namespace: kube-system +--- +kind: Deployment +apiVersion: extensions/v1beta1 +metadata: + name: cni-metrics-helper + namespace: kube-system + labels: + k8s-app: cni-metrics-helper +spec: + selector: + matchLabels: + k8s-app: cni-metrics-helper + template: + metadata: + labels: + k8s-app: cni-metrics-helper + spec: + serviceAccountName: cni-metrics-helper + containers: + - image: 602401143452.dkr.ecr.us-west-2.amazonaws.com/cni-metrics-helper:v1.6.0-rc5 + imagePullPolicy: Always + name: cni-metrics-helper + env: + - name: USE_CLOUDWATCH + value: "true" diff --git a/scripts/lib/aws.sh b/scripts/lib/aws.sh index ebd5d134bc..f932d89bee 100644 --- a/scripts/lib/aws.sh +++ b/scripts/lib/aws.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash check_aws_credentials() { - aws sts get-caller-identity --query "Account" >/dev/null 2>&1 || + aws sts get-caller-identity --query "Account" || ( echo "No AWS credentials found. Please run \`aws configure\` to set up the CLI for your credentials." && exit 1) } diff --git a/scripts/lib/cluster.sh b/scripts/lib/cluster.sh index 01a5fe5fbe..670b1fd0f9 100644 --- a/scripts/lib/cluster.sh +++ b/scripts/lib/cluster.sh @@ -1,15 +1,19 @@ #!/usr/bin/env bash function down-test-cluster() { - echo -n "Deleting cluster $CLUSTER_NAME (this may take ~10 mins) ... " - $TESTER_PATH eks delete cluster --path $CLUSTER_CONFIG >>$CLUSTER_MANAGE_LOG_PATH 2>&1 || - ( echo "failed. Check $CLUSTER_MANAGE_LOG_PATH." && exit 1 ) - echo "ok." + if [[ -n "${CIRCLE_JOB:-}" ]]; then + $TESTER_PATH eks delete cluster --path $CLUSTER_CONFIG || (echo "failed!" && exit 1) + else + echo -n "Deleting cluster $CLUSTER_NAME (this may take ~10 mins) ... " + $TESTER_PATH eks delete cluster --path $CLUSTER_CONFIG >>$CLUSTER_MANAGE_LOG_PATH 2>&1 || + (echo "failed. Check $CLUSTER_MANAGE_LOG_PATH." && exit 1) + echo "ok." + fi } function up-test-cluster() { + echo -n "Configuring cluster $CLUSTER_NAME" ssh-keygen -q -P cni-test -f $SSH_KEY_PATH - AWS_K8S_TESTER_EKS_CLUSTER_NAME=$CLUSTER_NAME \ AWS_K8S_TESTER_EKS_KUBECONFIG_PATH=$KUBECONFIG_PATH \ AWS_K8S_TESTER_EKS_KUBERNETES_VERSION=${K8S_VERSION%.*} \ @@ -28,9 +32,14 @@ function up-test-cluster() { AWS_K8S_TESTER_EKS_ADD_ON_NLB_HELLO_WORLD_ENABLE=true \ AWS_K8S_TESTER_EKS_ADD_ON_ALB_2048_ENABLE=true \ AWS_K8S_TESTER_EKS_KUBECTL_PATH=$KUBECTL_PATH \ - $TESTER_PATH eks create config --path $CLUSTER_CONFIG 1>&2 - echo -n "Creating cluster $CLUSTER_NAME (this may take ~20 mins. details: tail -f $CLUSTER_MANAGE_LOG_PATH)... " - $TESTER_PATH eks create cluster --path $CLUSTER_CONFIG 2>>$CLUSTER_MANAGE_LOG_PATH 1>&2 || - ( echo "failed. Check $CLUSTER_MANAGE_LOG_PATH." && exit 1 ) - echo "ok." + $TESTER_PATH eks create config --path $CLUSTER_CONFIG 1>&2 + + if [[ -n "${CIRCLE_JOB:-}" ]]; then + $TESTER_PATH eks create cluster --path $CLUSTER_CONFIG || (echo "failed!" && exit 1) + else + echo -n "Creating cluster $CLUSTER_NAME (this may take ~20 mins. details: tail -f $CLUSTER_MANAGE_LOG_PATH)... " + $TESTER_PATH eks create cluster --path $CLUSTER_CONFIG >>$CLUSTER_MANAGE_LOG_PATH 1>&2 || + (echo "failed. Check $CLUSTER_MANAGE_LOG_PATH." && exit 1) + echo "ok." + fi } diff --git a/scripts/run-integration-tests.sh b/scripts/run-integration-tests.sh index 4cd89a695b..e3f72e217c 100755 --- a/scripts/run-integration-tests.sh +++ b/scripts/run-integration-tests.sh @@ -57,11 +57,11 @@ KUBECTL_PATH=${KUBECTL_PATH:-$TESTER_DIR/kubectl} LOCAL_GIT_VERSION=$(git describe --tags --always --dirty) # The stable image version is the image tag used in the latest stable # aws-k8s-cni.yaml manifest -STABLE_IMAGE_VERSION=${STABLE_IMAGE_VERSION:-v1.5.3} +STABLE_IMAGE_VERSION=${STABLE_IMAGE_VERSION:-v1.6.0-rc5} TEST_IMAGE_VERSION=${IMAGE_VERSION:-$LOCAL_GIT_VERSION} # The CNI version we will start our k8s clusters with. We will then perform an # upgrade from this CNI to the CNI being tested (TEST_IMAGE_VERSION) -BASE_CNI_VERSION=${BASE_CNI_VERSION:-v1.5} +BASE_CNI_VERSION=${BASE_CNI_VERSION:-v1.6} BASE_CONFIG_PATH="$DIR/../config/$BASE_CNI_VERSION/aws-k8s-cni.yaml" TEST_CONFIG_PATH="$TEST_CONFIG_DIR/aws-k8s-cni.yaml" @@ -89,11 +89,11 @@ ensure_ecr_repo "$AWS_ACCOUNT_ID" "$AWS_ECR_REPO_NAME" # Check to see if the image already exists in the Docker repository, and if # not, check out the CNI source code for that image tag, build the CNI # image and push it to the Docker repository -if [[ $(docker images -q $IMAGE_NAME:$TEST_IMAGE_VERSION 2>/dev/null) ]]; then +if [[ $(docker images -q $IMAGE_NAME:$TEST_IMAGE_VERSION 2> /dev/null) ]]; then echo "CNI image $IMAGE_NAME:$TEST_IMAGE_VERSION already exists in repository. Skipping image build..." else echo "CNI image $IMAGE_NAME:$TEST_IMAGE_VERSION does not exist in repository." - if [[ $TEST_IMAGE_VERSION != $LOCAL_GIT_VERSION ]]; then + if [[ $TEST_IMAGE_VERSION != "$LOCAL_GIT_VERSION" ]]; then __cni_source_tmpdir="/tmp/cni-src-$IMAGE_VERSION" echo "Checking out CNI source code for $IMAGE_VERSION ..." @@ -103,7 +103,7 @@ else fi make docker IMAGE=$IMAGE_NAME VERSION=$TEST_IMAGE_VERSION docker push $IMAGE_NAME:$TEST_IMAGE_VERSION - if [[ $TEST_IMAGE_VERSION != $LOCAL_GIT_VERSION ]]; then + if [[ $TEST_IMAGE_VERSION != "$LOCAL_GIT_VERSION" ]]; then popd fi fi @@ -138,10 +138,26 @@ cp "$BASE_CONFIG_PATH" "$TEST_CONFIG_PATH" sed -i'.bak' "s,602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni,$IMAGE_NAME," "$TEST_CONFIG_PATH" sed -i'.bak' "s,$STABLE_IMAGE_VERSION,$TEST_IMAGE_VERSION," "$TEST_CONFIG_PATH" -echo "*******************************************************************************" -echo "Deploying CNI with image $IMAGE_NAME" export KUBECONFIG=$KUBECONFIG_PATH -kubectl apply -f "$TEST_CONFIG_PATH" +ADDONS_CNI_IMAGE=$($KUBECTL_PATH describe daemonset aws-node -n kube-system | grep Image | cut -d ":" -f 2-3 | tr -d '[:space:]') + +echo "*******************************************************************************" +echo "Running integration tests on default CNI version, $ADDONS_CNI_IMAGE" +echo "" +pushd ./test/integration +GO111MODULE=on go test -v -timeout 0 ./... --kubeconfig=$KUBECONFIG --ginkgo.focus="\[cni-integration\]" --ginkgo.skip="\[Disruptive\]" \ + --assets=./assets +TEST_PASS=$? +popd + +echo "*******************************************************************************" +echo "Updating CNI to image $IMAGE_NAME:$TEST_IMAGE_VERSION" +$KUBECTL_PATH apply -f "$TEST_CONFIG_PATH" + +# Delay based on 3 nodes, 30s grace period per CNI pod +echo "Sleeping for 110s" +echo "TODO: Poll and wait for updates to complete instead!" +sleep 110 echo "*******************************************************************************" echo "Running integration tests on current image:"