diff --git a/.github/workflows/pr-manual-tests.yaml b/.github/workflows/pr-manual-tests.yaml index 7d789f9150..277ccb3c03 100644 --- a/.github/workflows/pr-manual-tests.yaml +++ b/.github/workflows/pr-manual-tests.yaml @@ -50,5 +50,6 @@ jobs: ROLE_CREATE: false ROLE_ARN: ${{ secrets.EKS_CLUSTER_ROLE_ARN }} RUN_CONFORMANCE: true + RUN_INTEGRATION_DEFAULT_CNI: false run: | ./scripts/run-integration-tests.sh diff --git a/.github/workflows/weekly-cron-tests.yaml b/.github/workflows/weekly-cron-tests.yaml index a567340633..c1bcd1ff56 100644 --- a/.github/workflows/weekly-cron-tests.yaml +++ b/.github/workflows/weekly-cron-tests.yaml @@ -32,7 +32,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v1 with: role-to-assume: ${{ secrets.OSS_TEST_ROLE_ARN }} - role-duration-seconds: 14400 # 4 hours + role-duration-seconds: 28800 # 8 hours aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - name: Run perf tests env: @@ -40,10 +40,11 @@ jobs: S3_BUCKET_CREATE: false S3_BUCKET_NAME: ${{ secrets.S3_BUCKET_NAME }} ROLE_CREATE: false - ROLE_ARN: ${{ secrets.ROLE_ARN }} + ROLE_ARN: ${{ secrets.EKS_CLUSTER_ROLE_ARN }} PERFORMANCE_TEST_S3_BUCKET_NAME: cni-performance-tests RUN_PERFORMANCE_TESTS: true RUN_TESTER_LB_ADDONS: true + RUN_INTEGRATION_DEFAULT_CNI: false run: | ./scripts/run-integration-tests.sh - name: Run kops tests @@ -52,9 +53,10 @@ jobs: S3_BUCKET_CREATE: false S3_BUCKET_NAME: ${{ secrets.S3_BUCKET_NAME }} ROLE_CREATE: false - ROLE_ARN: ${{ secrets.ROLE_ARN }} + ROLE_ARN: ${{ secrets.EKS_CLUSTER_ROLE_ARN }} RUN_KOPS_TEST: true RUN_TESTER_LB_ADDONS: true + RUN_INTEGRATION_DEFAULT_CNI: false run: | ./scripts/run-integration-tests.sh if: always() @@ -64,9 +66,10 @@ jobs: S3_BUCKET_CREATE: false S3_BUCKET_NAME: ${{ secrets.S3_BUCKET_NAME }} ROLE_CREATE: false - ROLE_ARN: ${{ secrets.ROLE_ARN }} + ROLE_ARN: ${{ secrets.EKS_CLUSTER_ROLE_ARN }} RUN_BOTTLEROCKET_TEST: true RUN_TESTER_LB_ADDONS: true + RUN_INTEGRATION_DEFAULT_CNI: false run: | ./scripts/run-integration-tests.sh if: always() @@ -76,10 +79,11 @@ jobs: S3_BUCKET_CREATE: false S3_BUCKET_NAME: ${{ secrets.S3_BUCKET_NAME }} ROLE_CREATE: false - ROLE_ARN: ${{ secrets.ROLE_ARN }} + ROLE_ARN: ${{ secrets.EKS_CLUSTER_ROLE_ARN }} RUN_CALICO_TEST: true RUN_LATEST_CALICO_VERSION: true RUN_TESTER_LB_ADDONS: true + RUN_INTEGRATION_DEFAULT_CNI: false run: | ./scripts/run-integration-tests.sh if: always() diff --git a/scripts/lib/common.sh b/scripts/lib/common.sh index 38788454da..af2ca53851 100644 --- a/scripts/lib/common.sh +++ b/scripts/lib/common.sh @@ -22,8 +22,9 @@ function display_timelines() { echo "Displaying all step durations." echo "TIMELINE: Docker build took $DOCKER_BUILD_DURATION seconds." echo "TIMELINE: Upping test cluster took $UP_CLUSTER_DURATION seconds." - echo "TIMELINE: Default CNI integration tests took $DEFAULT_INTEGRATION_DURATION seconds." - echo "TIMELINE: Updating CNI image took $CNI_IMAGE_UPDATE_DURATION seconds." + if [[ $RUN_INTEGRATION_DEFAULT_CNI == true ]]; then + echo "TIMELINE: Default CNI integration tests took $DEFAULT_INTEGRATION_DURATION seconds." + fi echo "TIMELINE: Current image integration tests took $CURRENT_IMAGE_INTEGRATION_DURATION seconds." if [[ "$RUN_CONFORMANCE" == true ]]; then echo "TIMELINE: Conformance tests took $CONFORMANCE_DURATION seconds." diff --git a/scripts/run-integration-tests.sh b/scripts/run-integration-tests.sh index d5e10d3fc0..45985b4df6 100755 --- a/scripts/run-integration-tests.sh +++ b/scripts/run-integration-tests.sh @@ -11,6 +11,7 @@ source "$DIR"/lib/common.sh source "$DIR"/lib/aws.sh source "$DIR"/lib/cluster.sh source "$DIR"/lib/integration.sh +source "$DIR"/lib/k8s.sh source "$DIR"/lib/performance_tests.sh # Variables used in /lib/aws.sh @@ -26,6 +27,7 @@ ARCH=$(go env GOARCH) : "${RUN_CONFORMANCE:=false}" : "${RUN_TESTER_LB_ADDONS:=false}" : "${RUN_KOPS_TEST:=false}" +: "${RUN_INTEGRATION_DEFAULT_CNI:=true}" : "${RUN_BOTTLEROCKET_TEST:=false}" : "${RUN_PERFORMANCE_TESTS:=false}" : "${RUNNING_PERFORMANCE:=false}" @@ -210,50 +212,34 @@ grep -r -q $TEST_IMAGE_VERSION $TEST_CONFIG_PATH sed -i'.bak' "s,602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni-init,$INIT_IMAGE_NAME," "$TEST_CONFIG_PATH" grep -r -q $INIT_IMAGE_NAME $TEST_CONFIG_PATH - if [[ $RUN_KOPS_TEST == true ]]; then export KUBECONFIG=~/.kube/config -fi - -if [[ $RUN_KOPS_TEST == true ]]; then run_kops_conformance fi -ADDONS_CNI_IMAGE=$($KUBECTL_PATH describe daemonset aws-node -n kube-system | grep Image | cut -d ":" -f 2-3 | tr -d '[:space:]') -echo "*******************************************************************************" -echo "Running integration tests on default CNI version, $ADDONS_CNI_IMAGE" -echo "" -START=$SECONDS +if [[ $RUN_INTEGRATION_DEFAULT_CNI == true ]]; then + ADDONS_CNI_IMAGE=$($KUBECTL_PATH describe daemonset aws-node -n kube-system | grep Image | cut -d ":" -f 2-3 | tr -d '[:space:]') -focus="CANARY" -echo "Running ginkgo tests with focus: $focus" -(cd "$INTEGRATION_TEST_DIR/cni" && CGO_ENABLED=0 ginkgo --focus="$focus" -v --timeout 20m --no-color --fail-on-pending -- --cluster-kubeconfig="$KUBECONFIG" --cluster-name="$CLUSTER_NAME" --aws-region="$AWS_DEFAULT_REGION" --aws-vpc-id="$VPC_ID" --ng-name-label-key="kubernetes.io/os" --ng-name-label-val="linux") -(cd "$INTEGRATION_TEST_DIR/ipamd" && CGO_ENABLED=0 ginkgo --focus="$focus" -v --timeout 10m --no-color --fail-on-pending -- --cluster-kubeconfig="$KUBECONFIG" --cluster-name="$CLUSTER_NAME" --aws-region="$AWS_DEFAULT_REGION" --aws-vpc-id="$VPC_ID" --ng-name-label-key="kubernetes.io/os" --ng-name-label-val="linux") -TEST_PASS=$? -DEFAULT_INTEGRATION_DURATION=$((SECONDS - START)) -echo "TIMELINE: Default CNI integration tests took $DEFAULT_INTEGRATION_DURATION seconds." + echo "*******************************************************************************" + echo "Running integration tests on default CNI version, $ADDONS_CNI_IMAGE" + echo "" + START=$SECONDS + + focus="CANARY" + echo "Running ginkgo tests with focus: $focus" + (cd "$INTEGRATION_TEST_DIR/cni" && CGO_ENABLED=0 ginkgo --focus="$focus" -v --timeout 60m --no-color --fail-on-pending -- --cluster-kubeconfig="$KUBECONFIG" --cluster-name="$CLUSTER_NAME" --aws-region="$AWS_DEFAULT_REGION" --aws-vpc-id="$VPC_ID" --ng-name-label-key="kubernetes.io/os" --ng-name-label-val="linux") + (cd "$INTEGRATION_TEST_DIR/ipamd" && CGO_ENABLED=0 ginkgo --focus="$focus" -v --timeout 60m --no-color --fail-on-pending -- --cluster-kubeconfig="$KUBECONFIG" --cluster-name="$CLUSTER_NAME" --aws-region="$AWS_DEFAULT_REGION" --aws-vpc-id="$VPC_ID" --ng-name-label-key="kubernetes.io/os" --ng-name-label-val="linux") + TEST_PASS=$? + DEFAULT_INTEGRATION_DURATION=$((SECONDS - START)) + echo "TIMELINE: Default CNI integration tests took $DEFAULT_INTEGRATION_DURATION seconds." +fi echo "*******************************************************************************" echo "Updating CNI to image $IMAGE_NAME:$TEST_IMAGE_VERSION" echo "Using init container $INIT_IMAGE_NAME:$TEST_IMAGE_VERSION" START=$SECONDS $KUBECTL_PATH apply -f "$TEST_CONFIG_PATH" -NODE_COUNT=$($KUBECTL_PATH get nodes --no-headers=true | wc -l) -echo "Number of nodes in the test cluster is $NODE_COUNT" -UPDATED_PODS_COUNT=0 -MAX_RETRIES=20 -RETRY_ATTEMPT=0 -sleep 5 -while [[ $UPDATED_PODS_COUNT -lt $NODE_COUNT && $RETRY_ATTEMPT -lt $MAX_RETRIES ]]; do - UPDATED_PODS_COUNT=$($KUBECTL_PATH get pods -A --field-selector=status.phase=Running -l k8s-app=aws-node --no-headers=true | wc -l) - let RETRY_ATTEMPT=RETRY_ATTEMPT+1 - sleep 5 - echo "Waiting for cni daemonset update. $UPDATED_PODS_COUNT are in ready state in retry attempt $RETRY_ATTEMPT" -done -echo "Updated!" - -CNI_IMAGE_UPDATE_DURATION=$((SECONDS - START)) -echo "TIMELINE: Updating CNI image took $CNI_IMAGE_UPDATE_DURATION seconds." +check_ds_rollout "aws-node" "kube-system" "10m" echo "*******************************************************************************" echo "Running integration tests on current image:"