Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix Weekly Cron Tests #2154

Merged
merged 1 commit into from
Dec 9, 2022
Merged

Fix Weekly Cron Tests #2154

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/pr-manual-tests.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,5 +50,6 @@ jobs:
ROLE_CREATE: false
ROLE_ARN: ${{ secrets.EKS_CLUSTER_ROLE_ARN }}
RUN_CONFORMANCE: true
RUN_INTEGRATION_DEFAULT_CNI: false
run: |
./scripts/run-integration-tests.sh
14 changes: 9 additions & 5 deletions .github/workflows/weekly-cron-tests.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,18 +32,19 @@ jobs:
uses: aws-actions/configure-aws-credentials@v1
with:
role-to-assume: ${{ secrets.OSS_TEST_ROLE_ARN }}
role-duration-seconds: 14400 # 4 hours
role-duration-seconds: 28800 # 8 hours
aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
- name: Run perf tests
env:
DISABLE_PROMPT: true
S3_BUCKET_CREATE: false
S3_BUCKET_NAME: ${{ secrets.S3_BUCKET_NAME }}
ROLE_CREATE: false
ROLE_ARN: ${{ secrets.ROLE_ARN }}
ROLE_ARN: ${{ secrets.EKS_CLUSTER_ROLE_ARN }}
PERFORMANCE_TEST_S3_BUCKET_NAME: cni-performance-tests
RUN_PERFORMANCE_TESTS: true
RUN_TESTER_LB_ADDONS: true
RUN_INTEGRATION_DEFAULT_CNI: false
run: |
./scripts/run-integration-tests.sh
- name: Run kops tests
Expand All @@ -52,9 +53,10 @@ jobs:
S3_BUCKET_CREATE: false
S3_BUCKET_NAME: ${{ secrets.S3_BUCKET_NAME }}
ROLE_CREATE: false
ROLE_ARN: ${{ secrets.ROLE_ARN }}
ROLE_ARN: ${{ secrets.EKS_CLUSTER_ROLE_ARN }}
RUN_KOPS_TEST: true
RUN_TESTER_LB_ADDONS: true
RUN_INTEGRATION_DEFAULT_CNI: false
run: |
./scripts/run-integration-tests.sh
if: always()
Expand All @@ -64,9 +66,10 @@ jobs:
S3_BUCKET_CREATE: false
S3_BUCKET_NAME: ${{ secrets.S3_BUCKET_NAME }}
ROLE_CREATE: false
ROLE_ARN: ${{ secrets.ROLE_ARN }}
ROLE_ARN: ${{ secrets.EKS_CLUSTER_ROLE_ARN }}
RUN_BOTTLEROCKET_TEST: true
RUN_TESTER_LB_ADDONS: true
RUN_INTEGRATION_DEFAULT_CNI: false
run: |
./scripts/run-integration-tests.sh
if: always()
Expand All @@ -76,10 +79,11 @@ jobs:
S3_BUCKET_CREATE: false
S3_BUCKET_NAME: ${{ secrets.S3_BUCKET_NAME }}
ROLE_CREATE: false
ROLE_ARN: ${{ secrets.ROLE_ARN }}
ROLE_ARN: ${{ secrets.EKS_CLUSTER_ROLE_ARN }}
RUN_CALICO_TEST: true
RUN_LATEST_CALICO_VERSION: true
RUN_TESTER_LB_ADDONS: true
RUN_INTEGRATION_DEFAULT_CNI: false
run: |
./scripts/run-integration-tests.sh
if: always()
5 changes: 3 additions & 2 deletions scripts/lib/common.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,9 @@ function display_timelines() {
echo "Displaying all step durations."
echo "TIMELINE: Docker build took $DOCKER_BUILD_DURATION seconds."
echo "TIMELINE: Upping test cluster took $UP_CLUSTER_DURATION seconds."
echo "TIMELINE: Default CNI integration tests took $DEFAULT_INTEGRATION_DURATION seconds."
echo "TIMELINE: Updating CNI image took $CNI_IMAGE_UPDATE_DURATION seconds."
if [[ $RUN_INTEGRATION_DEFAULT_CNI == true ]]; then
echo "TIMELINE: Default CNI integration tests took $DEFAULT_INTEGRATION_DURATION seconds."
fi
echo "TIMELINE: Current image integration tests took $CURRENT_IMAGE_INTEGRATION_DURATION seconds."
if [[ "$RUN_CONFORMANCE" == true ]]; then
echo "TIMELINE: Conformance tests took $CONFORMANCE_DURATION seconds."
Expand Down
50 changes: 18 additions & 32 deletions scripts/run-integration-tests.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ source "$DIR"/lib/common.sh
source "$DIR"/lib/aws.sh
source "$DIR"/lib/cluster.sh
source "$DIR"/lib/integration.sh
source "$DIR"/lib/k8s.sh
source "$DIR"/lib/performance_tests.sh

# Variables used in /lib/aws.sh
Expand All @@ -26,6 +27,7 @@ ARCH=$(go env GOARCH)
: "${RUN_CONFORMANCE:=false}"
: "${RUN_TESTER_LB_ADDONS:=false}"
: "${RUN_KOPS_TEST:=false}"
: "${RUN_INTEGRATION_DEFAULT_CNI:=true}"
: "${RUN_BOTTLEROCKET_TEST:=false}"
: "${RUN_PERFORMANCE_TESTS:=false}"
: "${RUNNING_PERFORMANCE:=false}"
Expand Down Expand Up @@ -210,50 +212,34 @@ grep -r -q $TEST_IMAGE_VERSION $TEST_CONFIG_PATH
sed -i'.bak' "s,602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni-init,$INIT_IMAGE_NAME," "$TEST_CONFIG_PATH"
grep -r -q $INIT_IMAGE_NAME $TEST_CONFIG_PATH


if [[ $RUN_KOPS_TEST == true ]]; then
export KUBECONFIG=~/.kube/config
fi

if [[ $RUN_KOPS_TEST == true ]]; then
run_kops_conformance
fi
ADDONS_CNI_IMAGE=$($KUBECTL_PATH describe daemonset aws-node -n kube-system | grep Image | cut -d ":" -f 2-3 | tr -d '[:space:]')

echo "*******************************************************************************"
echo "Running integration tests on default CNI version, $ADDONS_CNI_IMAGE"
jdn5126 marked this conversation as resolved.
Show resolved Hide resolved
echo ""
START=$SECONDS
if [[ $RUN_INTEGRATION_DEFAULT_CNI == true ]]; then
ADDONS_CNI_IMAGE=$($KUBECTL_PATH describe daemonset aws-node -n kube-system | grep Image | cut -d ":" -f 2-3 | tr -d '[:space:]')

focus="CANARY"
echo "Running ginkgo tests with focus: $focus"
(cd "$INTEGRATION_TEST_DIR/cni" && CGO_ENABLED=0 ginkgo --focus="$focus" -v --timeout 20m --no-color --fail-on-pending -- --cluster-kubeconfig="$KUBECONFIG" --cluster-name="$CLUSTER_NAME" --aws-region="$AWS_DEFAULT_REGION" --aws-vpc-id="$VPC_ID" --ng-name-label-key="kubernetes.io/os" --ng-name-label-val="linux")
(cd "$INTEGRATION_TEST_DIR/ipamd" && CGO_ENABLED=0 ginkgo --focus="$focus" -v --timeout 10m --no-color --fail-on-pending -- --cluster-kubeconfig="$KUBECONFIG" --cluster-name="$CLUSTER_NAME" --aws-region="$AWS_DEFAULT_REGION" --aws-vpc-id="$VPC_ID" --ng-name-label-key="kubernetes.io/os" --ng-name-label-val="linux")
TEST_PASS=$?
DEFAULT_INTEGRATION_DURATION=$((SECONDS - START))
echo "TIMELINE: Default CNI integration tests took $DEFAULT_INTEGRATION_DURATION seconds."
echo "*******************************************************************************"
echo "Running integration tests on default CNI version, $ADDONS_CNI_IMAGE"
echo ""
START=$SECONDS

focus="CANARY"
echo "Running ginkgo tests with focus: $focus"
(cd "$INTEGRATION_TEST_DIR/cni" && CGO_ENABLED=0 ginkgo --focus="$focus" -v --timeout 60m --no-color --fail-on-pending -- --cluster-kubeconfig="$KUBECONFIG" --cluster-name="$CLUSTER_NAME" --aws-region="$AWS_DEFAULT_REGION" --aws-vpc-id="$VPC_ID" --ng-name-label-key="kubernetes.io/os" --ng-name-label-val="linux")
(cd "$INTEGRATION_TEST_DIR/ipamd" && CGO_ENABLED=0 ginkgo --focus="$focus" -v --timeout 60m --no-color --fail-on-pending -- --cluster-kubeconfig="$KUBECONFIG" --cluster-name="$CLUSTER_NAME" --aws-region="$AWS_DEFAULT_REGION" --aws-vpc-id="$VPC_ID" --ng-name-label-key="kubernetes.io/os" --ng-name-label-val="linux")
TEST_PASS=$?
DEFAULT_INTEGRATION_DURATION=$((SECONDS - START))
echo "TIMELINE: Default CNI integration tests took $DEFAULT_INTEGRATION_DURATION seconds."
fi

echo "*******************************************************************************"
echo "Updating CNI to image $IMAGE_NAME:$TEST_IMAGE_VERSION"
echo "Using init container $INIT_IMAGE_NAME:$TEST_IMAGE_VERSION"
START=$SECONDS
$KUBECTL_PATH apply -f "$TEST_CONFIG_PATH"
NODE_COUNT=$($KUBECTL_PATH get nodes --no-headers=true | wc -l)
echo "Number of nodes in the test cluster is $NODE_COUNT"
UPDATED_PODS_COUNT=0
MAX_RETRIES=20
RETRY_ATTEMPT=0
sleep 5
while [[ $UPDATED_PODS_COUNT -lt $NODE_COUNT && $RETRY_ATTEMPT -lt $MAX_RETRIES ]]; do
UPDATED_PODS_COUNT=$($KUBECTL_PATH get pods -A --field-selector=status.phase=Running -l k8s-app=aws-node --no-headers=true | wc -l)
let RETRY_ATTEMPT=RETRY_ATTEMPT+1
sleep 5
echo "Waiting for cni daemonset update. $UPDATED_PODS_COUNT are in ready state in retry attempt $RETRY_ATTEMPT"
done
echo "Updated!"

CNI_IMAGE_UPDATE_DURATION=$((SECONDS - START))
echo "TIMELINE: Updating CNI image took $CNI_IMAGE_UPDATE_DURATION seconds."
check_ds_rollout "aws-node" "kube-system" "10m"

echo "*******************************************************************************"
echo "Running integration tests on current image:"
Expand Down