Run scale test for 130 pods.

aws · Jun 12, 2020 · 1ac7d35 · 1ac7d35
1 parent 9933a09
commit 1ac7d35
Show file tree

Hide file tree

Showing 4 changed files with 60 additions and 12 deletions.
diff --git a/deploy-20-pods.yaml → deploy-130-pods.yaml b/deploy-20-pods.yaml → deploy-130-pods.yaml
@@ -1,17 +1,17 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: deploy-20-pods
+  name: deploy-130-pods
 spec:
-  replicas: 20
+  replicas: 130
   selector:
       matchLabels:
-        app: deploy-20-pods
+        app: deploy-130-pods
   template:
     metadata:
-      name: test-pod-20
+      name: test-pod-130
       labels:
-        app: deploy-20-pods
+        app: deploy-130-pods
         tier: backend
         track: stable
     spec:
@@ -23,4 +23,4 @@ spec:
               containerPort: 80
           imagePullPolicy: IfNotPresent
       nodeSelector:
-        eks.amazonaws.com/nodegroup: two-nodes
+        eks.amazonaws.com/nodegroup: three-nodes
diff --git a/scripts/lib/cluster.sh b/scripts/lib/cluster.sh
@@ -25,7 +25,7 @@ function up-test-cluster() {
         AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ENABLE=true \
         AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ROLE_CREATE=$ROLE_CREATE \
         AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_ROLE_ARN=$ROLE_ARN \
-        AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_MNGS='{"GetRef.Name-mng-for-cni":{"name":"GetRef.Name-mng-for-cni","remote-access-user-name":"ec2-user","tags":{"group":"amazon-vpc-cni-k8s"},"release-version":"","ami-type":"AL2_x86_64","asg-min-size":3,"asg-max-size":3,"asg-desired-capacity":3,"instance-types":["c5.xlarge"],"volume-size":40}, "two-nodes":{"name":"two-nodes","remote-access-user-name":"ec2-user","tags":{"group":"amazon-vpc-cni-k8s"},"release-version":"","ami-type":"AL2_x86_64","asg-min-size":2,"asg-max-size":2,"asg-desired-capacity":2,"instance-types":["m5.xlarge"],"volume-size":40}}' \
+        AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_MNGS='{"GetRef.Name-mng-for-cni":{"name":"GetRef.Name-mng-for-cni","remote-access-user-name":"ec2-user","tags":{"group":"amazon-vpc-cni-k8s"},"release-version":"","ami-type":"AL2_x86_64","asg-min-size":3,"asg-max-size":3,"asg-desired-capacity":3,"instance-types":["c5.xlarge"],"volume-size":40}, "three-nodes":{"name":"three-nodes","remote-access-user-name":"ec2-user","tags":{"group":"amazon-vpc-cni-k8s"},"release-version":"","ami-type":"AL2_x86_64","asg-min-size":3,"asg-max-size":3,"asg-desired-capacity":3,"instance-types":["m5.xlarge"],"volume-size":40}}' \
         AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_FETCH_LOGS=true \
         AWS_K8S_TESTER_EKS_ADD_ON_NLB_HELLO_WORLD_ENABLE=true \
         AWS_K8S_TESTER_EKS_ADD_ON_ALB_2048_ENABLE=true \

diff --git a/scripts/lib/common.sh b/scripts/lib/common.sh
@@ -28,3 +28,50 @@ function display_timelines() {
     echo "TIMELINE: Conformance tests took $CONFORMANCE_DURATION seconds."
     echo "TIMELINE: Down processes took $DOWN_DURATION seconds."
 }
+
+function run_scale_test_130_pods() {
+    SCALE_UP_DURATION_ARRAY=()
+    SCALE_DOWN_DURATION_ARRAY=()
+    while [ ${#SCALE_UP_DURATION_ARRAY[@]} -lt 3 ]
+    do
+        ITERATION_START=$SECONDS
+        $KUBECTL_PATH scale -f deploy-130-pods.yaml --replicas=130
+        GET_DEPLOY_OUTPUT=""
+        while [[ ${#GET_DEPLOY_OUTPUT} -lt 1 && $((SECONDS - ITERATION_START)) -lt 200 ]]
+        do
+            sleep 1
+            echo "Scaling UP"
+            echo $($KUBECTL_PATH get deploy)
+            GET_DEPLOY_OUTPUT=$($KUBECTL_PATH get deploy | grep 130/130)
+        done
+
+        SCALE_UP_DURATION_ARRAY+=( $((SECONDS - ITERATION_START)) )
+        MIDPOINT_START=$SECONDS
+        $KUBECTL_PATH scale -f deploy-130-pods.yaml --replicas=0
+        GET_DELETE_OUTPUT="TEMPSTRING"
+        while [[ ${#GET_DELETE_OUTPUT} -gt 1 && $((SECONDS - MIDPOINT_START)) -lt 200 ]]
+        do
+            sleep 1
+            echo "Scaling DOWN"
+            echo $($KUBECTL_PATH get deploy)
+            GET_DELETE_OUTPUT=$($KUBECTL_PATH get pods)
+        done
+        SCALE_DOWN_DURATION_ARRAY+=($((SECONDS - MIDPOINT_START)))
+    done
+
+    echo "Times to scale up:"
+    INDEX=0
+    while [ $INDEX -lt ${#SCALE_UP_DURATION_ARRAY[@]} ]
+    do
+        echo ${SCALE_UP_DURATION_ARRAY[$INDEX]}
+        INDEX=$((INDEX + 1))
+    done
+    echo ""
+    echo "Times to scale down:"
+    INDEX=0
+    while [ $INDEX -lt ${#SCALE_DOWN_DURATION_ARRAY[@]} ]
+    do
+        echo "${SCALE_DOWN_DURATION_ARRAY[$INDEX]} seconds"
+        INDEX=$((INDEX + 1))
+    done
+}
diff --git a/scripts/run-integration-tests.sh b/scripts/run-integration-tests.sh
@@ -167,13 +167,14 @@ aws sts get-caller-identity
 eksctl get clusters
 
 echo "TESTING trying to put 20 pods on new nodegroup"
-$KUBECTL_PATH get pods
-$KUBECTL_PATH get deploy
+DEPLOY_START=$SECONDS
 
+$KUBECTL_PATH apply -f deploy-130-pods.yaml
+run_scale_test_130_pods
+$KUBECTL_PATH delete -f deploy-130-pods.yaml
 
-$KUBECTL_PATH apply -f deploy-20-pods.yaml
-sleep 60
-$KUBECTL_PATH get pods
+DEPLOY_DURATION=$((SECONDS - DEPLOY_START))
+echo "TIMELINE: Current image integration tests took $DEPLOY_DURATION seconds."
 
 echo "*******************************************************************************"
 echo "Running integration tests on default CNI version, $ADDONS_CNI_IMAGE"