Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

integration tests running with updated aws-k8s-tester #766

Merged
merged 3 commits into from
Jan 20, 2020
Merged

integration tests running with updated aws-k8s-tester #766

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1c5147b
harden the run-integration-tests.sh script
jaypipes Dec 19, 2019
227e9fc
do pre-checks before creating EKS test cluster
jaypipes Dec 19, 2019
cf66674
update script to work with 0.5.4 aws-k8s-tester
jaypipes Jan 3, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions scripts/lib/aws.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
#!/usr/bin/env bash

check_aws_credentials() {
aws sts get-caller-identity --query "Account" >/dev/null 2>&1 ||
( echo "No AWS credentials found. Please run \`aws configure\` to set up the CLI for your credentials." && exit 1)
}

ensure_ecr_repo() {
local __registry_account_id="$1"
local __repo_name="$2"
if ! `aws ecr describe-repositories --registry-id "$__registry_account_id" --repository-names "$__repo_name" >/dev/null 2>&1`; then
echo "creating ECR repo with name $__repo_name in registry account $__registry_account_id"
aws ecr create-repository --repository-name "$__repo_name"
fi
}

ensure_aws_k8s_tester() {
TESTER_RELEASE=${TESTER_RELEASE:-v0.5.4}
TESTER_DOWNLOAD_URL=https://github.com/aws/aws-k8s-tester/releases/download/$TESTER_RELEASE/aws-k8s-tester-$TESTER_RELEASE-$OS-$ARCH

# Download aws-k8s-tester if not yet
if [[ ! -e $TESTER_PATH ]]; then
mkdir -p $TESTER_DIR
echo "Downloading aws-k8s-tester from $TESTER_DOWNLOAD_URL to $TESTER_PATH"
curl -s -L -X GET $TESTER_DOWNLOAD_URL -o $TESTER_PATH
chmod +x $TESTER_PATH
fi
}
37 changes: 23 additions & 14 deletions scripts/lib/cluster.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,27 +1,36 @@
#!/usr/bin/env bash

function down-test-cluster() {
echo "Deleting cluster $CLUSTER_NAME"
$TESTER_PATH eks delete cluster --path $CLUSTER_CONFIG
echo -n "Deleting cluster $CLUSTER_NAME (this may take ~10 mins) ... "
$TESTER_PATH eks delete cluster --path $CLUSTER_CONFIG >>$CLUSTER_MANAGE_LOG_PATH 2>&1 ||
( echo "failed. Check $CLUSTER_MANAGE_LOG_PATH." && exit 1 )
echo "ok."
}

function up-test-cluster() {
echo "Creating cluster $CLUSTER_NAME"
ssh-keygen -P cni-test -f $SSH_KEY_PATH

$TESTER_PATH eks create config --path $CLUSTER_CONFIG
ssh-keygen -q -P cni-test -f $SSH_KEY_PATH

AWS_K8S_TESTER_EKS_CLUSTER_NAME=$CLUSTER_NAME \
AWS_K8S_TESTER_EKS_KUBECONFIG_PATH=$KUBECONFIG_PATH \
AWS_K8S_TESTER_EKS_KUBERNETES_VERSION=${K8S_VERSION%.*} \
AWS_K8S_TESTER_EKS_ENABLE_WORKER_NODE_PRIVILEGED_PORT_ACCESS=true \
AWS_K8S_TESTER_EKS_WORKER_NODE_ASG_MIN=3 \
AWS_K8S_TESTER_EKS_WORKER_NODE_ASG_MAX=3 \
AWS_K8S_TESTER_EKS_WORKER_NODE_ASG_DESIRED_CAPACITY=3 \
AWS_K8S_TESTER_EKS_WORKER_NODE_PRIVATE_KEY_PATH=$SSH_KEY_PATH \
AWS_K8S_TESTER_EKS_WORKER_NODE_INSTANCE_TYPE=m3.xlarge \
AWS_K8S_TESTER_EKS_ENABLE_MANAGED_NODE_GROUP_PRIVILEGED_PORT_ACCESS=true \
AWS_K8S_TESTER_EKS_MANAGED_NODE_GROUP_ASG_MIN=3 \
AWS_K8S_TESTER_EKS_MANAGED_NODE_GROUP_ASG_MAX=3 \
AWS_K8S_TESTER_EKS_MANAGED_NODE_GROUP_ASG_DESIRED_CAPACITY=3 \
AWS_K8S_TESTER_EKS_MANAGED_NODE_GROUP_REMOTE_ACCESS_PRIVATE_KEY_PATH=$SSH_KEY_PATH \
AWS_K8S_TESTER_EKS_MANAGED_NODE_GROUP_INSTANCE_TYPES=m3.xlarge \
AWS_K8S_TESTER_EKS_AWS_K8S_TESTER_PATH=$TESTER_PATH \
AWS_K8S_TESTER_EKS_AWS_IAM_AUTHENTICATOR_PATH=$AUTHENTICATOR_PATH \
AWS_K8S_TESTER_EKS_ADD_ON_JOB_ECHO_ENABLE=true \
AWS_K8S_TESTER_EKS_ADD_ON_JOB_ECHO_PARALLELS=3 \
AWS_K8S_TESTER_EKS_ADD_ON_JOB_ECHO_COMPLETES=30 \
AWS_K8S_TESTER_EKS_ADD_ON_JOB_PERL_ENABLE=true \
AWS_K8S_TESTER_EKS_ADD_ON_NLB_HELLO_WORLD_ENABLE=true \
AWS_K8S_TESTER_EKS_ADD_ON_ALB_2048_ENABLE=true \
AWS_K8S_TESTER_EKS_KUBECTL_PATH=$KUBECTL_PATH \
$TESTER_PATH eks create cluster --path $CLUSTER_CONFIG 1>&2
}
$TESTER_PATH eks create config --path $CLUSTER_CONFIG 1>&2
echo -n "Creating cluster $CLUSTER_NAME (this may take ~20 mins. details: tail -f $CLUSTER_MANAGE_LOG_PATH)... "
$TESTER_PATH eks create cluster --path $CLUSTER_CONFIG 2>>$CLUSTER_MANAGE_LOG_PATH 1>&2 ||
( echo "failed. Check $CLUSTER_MANAGE_LOG_PATH." && exit 1 )
echo "ok."
}
18 changes: 18 additions & 0 deletions scripts/lib/common.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
#!/usr/bin/env bash

check_is_installed() {
local __name="$1"
if ! is_installed "$__name"; then
echo "Please install $__name before running this script."
exit 1
fi
}

is_installed() {
local __name="$1"
if $(which $__name >/dev/null 2>&1); then
return 0
else
return 1
fi
}
85 changes: 57 additions & 28 deletions scripts/run-integration-tests.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,18 +1,37 @@
#!/usr/bin/env bash

set -euo pipefail
set -Euo pipefail

trap 'on_error $LINENO' ERR

DIR=$(cd "$(dirname "$0")"; pwd)
source $DIR/lib/common.sh
source $DIR/lib/aws.sh
source $DIR/lib/cluster.sh

OS=$(go env GOOS)
ARCH=amd64
REGION=${AWS_REGION:-us-west-2}
ARCH=$(go env GOARCH)
AWS_REGION=${AWS_REGION:-us-west-2}
K8S_VERSION=${K8S_VERSION:-1.14.6}
PROVISION=${PROVISION:-true}
DEPROVISION=${DEPROVISION:-true}
BUILD=${BUILD:-true}

__cluster_created=0
__cluster_deprovisioned=0

on_error() {
# Make sure we destroy any cluster that was created if we hit run into an
jaypipes marked this conversation as resolved.
Show resolved Hide resolved
# error when attempting to run tests against the cluster
if [[ $__cluster_created -eq 1 && $__cluster_deprovisioned -eq 0 && "$DEPROVISION" = true ]]; then
# prevent double-deprovisioning with ctrl-c during deprovisioning...
__cluster_deprovisioned=1
echo "Cluster was provisioned already. Deprovisioning it..."
down-test-cluster
fi
exit 1
}

# test specific config, results location
TEST_ID=${TEST_ID:-$RANDOM}
TEST_DIR=/tmp/cni-test/$(date "+%Y%M%d%H%M%S")-$TEST_ID
Expand All @@ -23,55 +42,62 @@ REPORT_DIR=${TEST_DIR}/report
CLUSTER_ID=${CLUSTER_ID:-$RANDOM}
CLUSTER_NAME=cni-test-$CLUSTER_ID
TEST_CLUSTER_DIR=/tmp/cni-test/cluster-$CLUSTER_NAME
CLUSTER_MANAGE_LOG_PATH=$TEST_CLUSTER_DIR/cluster-manage.log
CLUSTER_CONFIG=${CLUSTER_CONFIG:-${TEST_CLUSTER_DIR}/${CLUSTER_NAME}.yaml}
SSH_KEY_PATH=${SSH_KEY_PATH:-${TEST_CLUSTER_DIR}/id_rsa}
KUBECONFIG_PATH=${KUBECONFIG_PATH:-${TEST_CLUSTER_DIR}/kubeconfig}

# shared binaries
TESTER_DOWNLOAD_URL=https://github.com/aws/aws-k8s-tester/releases/download/v0.4.3/aws-k8s-tester-v0.4.3-$OS-$ARCH
TESTER_DIR=${TESTER_DIR:-/tmp/aws-k8s-tester}
TESTER_PATH=${TESTER_PATH:-$TESTER_DIR/aws-k8s-tester}
AUTHENTICATOR_PATH=${AUTHENTICATOR_PATH:-/tmp/aws-k8s-tester/aws-iam-authenticator}
KUBECTL_PATH=${KUBECTL_PATH:-/tmp/aws-k8s-tester/kubectl}
AUTHENTICATOR_PATH=${AUTHENTICATOR_PATH:-$TESTER_DIR/aws-iam-authenticator}
KUBECTL_PATH=${KUBECTL_PATH:-$TESTER_DIR/kubectl}

# double-check all our preconditions and requirements have been met
check_is_installed docker
check_is_installed aws
check_aws_credentials
ensure_aws_k8s_tester

AWS_ACCOUNT_ID=${AWS_ACCOUNT_ID:-$(aws sts get-caller-identity --query Account --output text)}
AWS_ECR_REGISTRY=${AWS_ECR_REGISTRY:-"$AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com"}
AWS_ECR_REPO_NAME=${AWS_ECR_REPO_NAME:-"amazon-k8s-cni"}
IMAGE_NAME=${IMAGE_NAME:-"$AWS_ECR_REGISTRY/$AWS_ECR_REPO_NAME"}
IMAGE_VERSION=${IMAGE_VERSION:-$(git describe --tags --always --dirty)}

if [[ "$BUILD" = true ]]; then
# `aws ec2 get-login` returns a docker login string, which we eval here to
# login to the ECR registry
eval $(aws ecr get-login --region $AWS_REGION --no-include-email) >/dev/null 2>&1
ensure_ecr_repo "$AWS_ACCOUNT_ID" "$AWS_ECR_REPO_NAME"
make docker IMAGE=$IMAGE_NAME VERSION=$IMAGE_VERSION
Copy link
Contributor

@SaranBalaji90 SaranBalaji90 Jan 16, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This doesn't pull source code for a given tag(version) right?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No, it doesn't. It runs docker build -t aws-vpc-cni:$IMAGE_VERSION. If IMAGE_VERSION envvar has been overridden, we should probably set the BUILD envvar to false automatically.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Either that, or check that IMAGE_VERSION is overridden and if so, do a git checkout $IMAGE_VERSION here to switch the working tree to that git tag.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

using git-checkout makes sense. Can you help me understand how disabling BUILD will help? Do you mean to say, if CNI image is already available in my repo then disabling BUILD will help?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes, exactly. if you've already pushed, say, a v1.5.5 CNI image built from that git tag/branch, then setting BUILD=false will prevent the run-integration-tests.sh script from docker pushing an image and the eks-k8s-tester binary will simply pull the image that was in the ECR repo for v1.5.5

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah ok makes sense. Thanks for clarifying.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In short, I need to make some changes to this script to facilitate the not-building-from-local-checkout scenarios...

docker push $IMAGE_NAME:$IMAGE_VERSION
fi

# The version substituted in ./config/X/aws-k8s-cni.yaml
CNI_TEMPLATE_VERSION=${CNI_TEMPLATE_VERSION:-v1.5}

echo "Running $TEST_ID on $CLUSTER_NAME in $REGION"
echo "*******************************************************************************"
echo "Running $TEST_ID on $CLUSTER_NAME in $AWS_REGION"
echo "+ Cluster config dir: $TEST_CLUSTER_DIR"
echo "+ Result dir: $TEST_DIR"
echo "+ Tester: $TESTER_PATH"
echo "+ Kubeconfig: $KUBECONFIG_PATH"
echo "+ Node SSH key: $SSH_KEY_PATH"
echo "+ Cluster config: $CLUSTER_CONFIG"
echo ""
echo "+ AWS Account ID: $AWS_ACCOUNT_ID"
echo "+ CNI image to test: $IMAGE_NAME:$IMAGE_VERSION"

mkdir -p $TEST_DIR
mkdir -p $REPORT_DIR
mkdir -p $TEST_CLUSTER_DIR

# Download aws-k8s-tester if not yet
if [[ ! -e $TESTER_PATH ]]; then
mkdir -p $TESTER_DIR
echo "Downloading aws-k8s-tester from $TESTER_DOWNLOAD_URL to $TESTER_PATH"
curl -s -L -X GET $TESTER_DOWNLOAD_URL -o $TESTER_PATH
chmod +x $TESTER_PATH
fi

if [[ "$PROVISION" = true ]]; then
up-test-cluster
__cluster_created=1
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nice!

fi

if [[ "$BUILD" = true ]]; then
# Push test image
eval $(aws ecr get-login --region $REGION --no-include-email)
AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
IMAGE_NAME="$AWS_ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/amazon/amazon-k8s-cni"
IMAGE_VERSION=$(git describe --tags --always --dirty)

make docker IMAGE=$IMAGE_NAME VERSION=$IMAGE_VERSION
docker push $IMAGE_NAME:$IMAGE_VERSION

echo "Using ./config/$CNI_TEMPLATE_VERSION/aws-k8s-cni.yaml as a template"
if [[ ! -f "./config/$CNI_TEMPLATE_VERSION/aws-k8s-cni.yaml" ]]; then
echo "./config/$CNI_TEMPLATE_VERSION/aws-k8s-cni.yaml DOES NOT exist. Set \$CNI_TEMPLATE_VERSION to an existing directory in ./config/"
Expand All @@ -82,13 +108,16 @@ if [[ "$BUILD" = true ]]; then
sed -i'.bak' "s,v1.5.3,$IMAGE_VERSION," ./config/$CNI_TEMPLATE_VERSION/aws-k8s-cni.yaml
fi

echo "Deploying CNI"
echo "*******************************************************************************"
echo "Deploying CNI with image $IMAGE_NAME"
export KUBECONFIG=$KUBECONFIG_PATH
kubectl apply -f ./config/$CNI_TEMPLATE_VERSION/aws-k8s-cni.yaml

# Run the test
echo "*******************************************************************************"
echo "Running integration tests"
echo ""
pushd ./test/integration
go test -v -timeout 0 ./... --kubeconfig=$KUBECONFIG --ginkgo.focus="\[cni-integration\]" --ginkgo.skip="\[Disruptive\]" \
GO111MODULE=on go test -v -timeout 0 ./... --kubeconfig=$KUBECONFIG --ginkgo.focus="\[cni-integration\]" --ginkgo.skip="\[Disruptive\]" \
--assets=./assets
TEST_PASS=$?
popd
Expand Down