upgrade aws-k8s-tester to v1.1.5 (#2373)

* upgrade aws-k8s-tester * fixed mng name * clean eks cluster if the test fails
pingcap · May 8, 2020 · c13f708 · c13f708
1 parent ac2b137
commit c13f708
Show file tree

Hide file tree

Showing 4 changed files with 29 additions and 23 deletions.
diff --git a/ci/aws-clean-eks.sh b/ci/aws-clean-eks.sh
@@ -69,28 +69,13 @@ function fix_eks_mng_deletion_issues() {
 
 function clean_eks() {
     local CLUSTER="$1"
-    echo "info: searching mng stack"
-    local regex='^'$CLUSTER'-mng-[0-9]+$'
-    local mngStack=
-    for stackName in $(get_stacks); do
-        if [[ ! "$stackName" =~ $regex ]]; then
-            continue
-        fi
-        mngStack=$stackName
-        break
-    done
-    if [ -n "$mngStack" ]; then
-        echo "info: mng stack found '$mngStack'"
-    else
-        echo "info: mng stack not found"
-    fi
-
-    echo "info: deleting mng/cluster/cluster-role/mng-role/vpc stacks"
+    echo "info: deleting mng-sg/mng/cluster/cluster-role/mng-role/vpc stacks"
     local stacks=(
-        $mngStack
+        $CLUSTER-mng-mng-sg
+        $CLUSTER-mng
+        $CLUSTER-role-mng
         $CLUSTER-cluster
         $CLUSTER-role-cluster
-        $CLUSTER-role-mng
         $CLUSTER-vpc
     )
     for stack in ${stacks[@]}; do
@@ -99,7 +84,7 @@ function clean_eks() {
         aws cloudformation wait stack-delete-complete --stack-name $stack
         if [ $? -ne 0 ]; then
             echo "error: failed to delete stack '$stack'"
-            if [ "$stack" == "$mngStack" ]; then
+            if [ "$stack" == "$CLUSTER-mng" ]; then
                 echo "info: try to fix mng stack '$stack'"
                 for mngName in $(aws eks list-nodegroups --cluster-name "$CLUSTER" --query 'nodegroups[*]' --output text); do
                     fix_eks_mng_deletion_issues "$CLUSTER" $mngName
@@ -125,6 +110,9 @@ function clean_eks() {
             fi
         fi
     done
+    local keyPairName=$CLUSTER-key-nodes
+    echo "info: deleting key pair $keyPairName"
+    aws ec2 delete-key-pair --key-name $keyPairName
 }
 
 # https://github.com/aws/aws-cli#other-configurable-variables

diff --git a/ci/e2e_eks.groovy b/ci/e2e_eks.groovy
@@ -158,6 +158,13 @@ pipeline {
                 junit testResults: "*.xml", allowEmptyResults: true
             }
         }
+        failure {
+            sh """
+            export CLUSTER=${params.CLUSTER}
+            echo "info: cleaning eks cluster \$CLUSTER"
+            ./ci/aws-clean-eks.sh \$CLUSTER
+            """
+        }
     }
 }
 

diff --git a/hack/e2e.sh b/hack/e2e.sh
@@ -496,13 +496,24 @@ elif [ "$PROVIDER" == "eks" ]; then
     aws configure set default.region "$AWS_REGION"
     aws configure set aws_access_key_id "$AWS_ACCESS_KEY_ID"
     aws configure set aws_secret_access_key "$AWS_SECRET_ACCESS_KEY"
-    mngName=$CLUSTER-mng-$RANDOM
+    if [ -z "$SKIP_UP" ]; then
+        echo "info: make $HOME/.ssh/kube_aws_rsa writable if we're going to start the cluster"
+        # aws-k8s-tester tries to create or update ~/.ssh/kube_aws_rsa with new key
+        # pair every time, however previous created file is read-only and can't be
+        # updated
+        if test -f $HOME/.ssh/kube_aws_rsa; then
+            chmod 0600 $HOME/.ssh/kube_aws_rsa
+        fi
+    fi
+    echo "info: exporting AWS_K8S_TESTER config environments"
+    mngName=$CLUSTER-mng
     export AWS_K8S_TESTER_EKS_NAME=$CLUSTER
     export AWS_K8S_TESTER_EKS_CONFIG_PATH=/tmp/kubetest2.eks.$CLUSTER
     export AWS_K8S_TESTER_EKS_PARAMETERS_VERSION="1.15"
     export AWS_K8S_TESTER_EKS_PARAMETERS_ENCRYPTION_CMK_CREATE="false"
     export AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ENABLE="true"
     export AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_MNGS=$(printf '{"%s":{"name":"%s","ami-type":"AL2_x86_64","asg-min-size":%d,"asg-max-size":%d,"asg-desired-capacity":%d,"instance-types":["c5.xlarge"],"volume-size":40}}' "$mngName" "$mngName" "$KUBE_WORKERS" "$KUBE_WORKERS" "$KUBE_WORKERS")
+    env | grep ^AWS_K8S_TESTER --color=never
     # override KUBECONFIG
     KUBECONFIG=$AWS_K8S_TESTER_EKS_CONFIG_PATH.kubeconfig.yaml
 else

diff --git a/hack/lib.sh b/hack/lib.sh
@@ -36,9 +36,9 @@ HELM_VERSION=${HELM_VERSION:-2.9.1}
 KIND_VERSION=${KIND_VERSION:-0.8.1}
 DOCS_VERSION=${DOCS_VERSION:-0.2.1}
 KIND_BIN=$OUTPUT_BIN/kind
-KUBETEST2_VERSION=v0.0.8
+KUBETEST2_VERSION=v0.1.0
 KUBETSTS2_BIN=$OUTPUT_BIN/kubetest2
-AWS_K8S_TESTER_VERSION=v0.7.4
+AWS_K8S_TESTER_VERSION=v1.1.5
 AWS_K8S_TESTER_BIN=$OUTPUT_BIN/aws-k8s-tester
 
 test -d "$OUTPUT_BIN" || mkdir -p "$OUTPUT_BIN"