diff --git a/scripts/lib/aws.sh b/scripts/lib/aws.sh new file mode 100644 index 0000000000..ebd5d134bc --- /dev/null +++ b/scripts/lib/aws.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +check_aws_credentials() { + aws sts get-caller-identity --query "Account" >/dev/null 2>&1 || + ( echo "No AWS credentials found. Please run \`aws configure\` to set up the CLI for your credentials." && exit 1) +} + +ensure_ecr_repo() { + local __registry_account_id="$1" + local __repo_name="$2" + if ! `aws ecr describe-repositories --registry-id "$__registry_account_id" --repository-names "$__repo_name" >/dev/null 2>&1`; then + echo "creating ECR repo with name $__repo_name in registry account $__registry_account_id" + aws ecr create-repository --repository-name "$__repo_name" + fi +} + +ensure_aws_k8s_tester() { + TESTER_RELEASE=${TESTER_RELEASE:-v0.5.4} + TESTER_DOWNLOAD_URL=https://github.com/aws/aws-k8s-tester/releases/download/$TESTER_RELEASE/aws-k8s-tester-$TESTER_RELEASE-$OS-$ARCH + + # Download aws-k8s-tester if not yet + if [[ ! -e $TESTER_PATH ]]; then + mkdir -p $TESTER_DIR + echo "Downloading aws-k8s-tester from $TESTER_DOWNLOAD_URL to $TESTER_PATH" + curl -s -L -X GET $TESTER_DOWNLOAD_URL -o $TESTER_PATH + chmod +x $TESTER_PATH + fi +} diff --git a/scripts/lib/cluster.sh b/scripts/lib/cluster.sh index 54119ffa8d..01a5fe5fbe 100644 --- a/scripts/lib/cluster.sh +++ b/scripts/lib/cluster.sh @@ -1,27 +1,36 @@ #!/usr/bin/env bash function down-test-cluster() { - echo "Deleting cluster $CLUSTER_NAME" - $TESTER_PATH eks delete cluster --path $CLUSTER_CONFIG + echo -n "Deleting cluster $CLUSTER_NAME (this may take ~10 mins) ... " + $TESTER_PATH eks delete cluster --path $CLUSTER_CONFIG >>$CLUSTER_MANAGE_LOG_PATH 2>&1 || + ( echo "failed. Check $CLUSTER_MANAGE_LOG_PATH." && exit 1 ) + echo "ok." } function up-test-cluster() { - echo "Creating cluster $CLUSTER_NAME" - ssh-keygen -P cni-test -f $SSH_KEY_PATH - - $TESTER_PATH eks create config --path $CLUSTER_CONFIG + ssh-keygen -q -P cni-test -f $SSH_KEY_PATH AWS_K8S_TESTER_EKS_CLUSTER_NAME=$CLUSTER_NAME \ AWS_K8S_TESTER_EKS_KUBECONFIG_PATH=$KUBECONFIG_PATH \ AWS_K8S_TESTER_EKS_KUBERNETES_VERSION=${K8S_VERSION%.*} \ - AWS_K8S_TESTER_EKS_ENABLE_WORKER_NODE_PRIVILEGED_PORT_ACCESS=true \ - AWS_K8S_TESTER_EKS_WORKER_NODE_ASG_MIN=3 \ - AWS_K8S_TESTER_EKS_WORKER_NODE_ASG_MAX=3 \ - AWS_K8S_TESTER_EKS_WORKER_NODE_ASG_DESIRED_CAPACITY=3 \ - AWS_K8S_TESTER_EKS_WORKER_NODE_PRIVATE_KEY_PATH=$SSH_KEY_PATH \ - AWS_K8S_TESTER_EKS_WORKER_NODE_INSTANCE_TYPE=m3.xlarge \ + AWS_K8S_TESTER_EKS_ENABLE_MANAGED_NODE_GROUP_PRIVILEGED_PORT_ACCESS=true \ + AWS_K8S_TESTER_EKS_MANAGED_NODE_GROUP_ASG_MIN=3 \ + AWS_K8S_TESTER_EKS_MANAGED_NODE_GROUP_ASG_MAX=3 \ + AWS_K8S_TESTER_EKS_MANAGED_NODE_GROUP_ASG_DESIRED_CAPACITY=3 \ + AWS_K8S_TESTER_EKS_MANAGED_NODE_GROUP_REMOTE_ACCESS_PRIVATE_KEY_PATH=$SSH_KEY_PATH \ + AWS_K8S_TESTER_EKS_MANAGED_NODE_GROUP_INSTANCE_TYPES=m3.xlarge \ AWS_K8S_TESTER_EKS_AWS_K8S_TESTER_PATH=$TESTER_PATH \ AWS_K8S_TESTER_EKS_AWS_IAM_AUTHENTICATOR_PATH=$AUTHENTICATOR_PATH \ + AWS_K8S_TESTER_EKS_ADD_ON_JOB_ECHO_ENABLE=true \ + AWS_K8S_TESTER_EKS_ADD_ON_JOB_ECHO_PARALLELS=3 \ + AWS_K8S_TESTER_EKS_ADD_ON_JOB_ECHO_COMPLETES=30 \ + AWS_K8S_TESTER_EKS_ADD_ON_JOB_PERL_ENABLE=true \ + AWS_K8S_TESTER_EKS_ADD_ON_NLB_HELLO_WORLD_ENABLE=true \ + AWS_K8S_TESTER_EKS_ADD_ON_ALB_2048_ENABLE=true \ AWS_K8S_TESTER_EKS_KUBECTL_PATH=$KUBECTL_PATH \ - $TESTER_PATH eks create cluster --path $CLUSTER_CONFIG 1>&2 -} \ No newline at end of file + $TESTER_PATH eks create config --path $CLUSTER_CONFIG 1>&2 + echo -n "Creating cluster $CLUSTER_NAME (this may take ~20 mins. details: tail -f $CLUSTER_MANAGE_LOG_PATH)... " + $TESTER_PATH eks create cluster --path $CLUSTER_CONFIG 2>>$CLUSTER_MANAGE_LOG_PATH 1>&2 || + ( echo "failed. Check $CLUSTER_MANAGE_LOG_PATH." && exit 1 ) + echo "ok." +} diff --git a/scripts/lib/common.sh b/scripts/lib/common.sh new file mode 100644 index 0000000000..51578123ac --- /dev/null +++ b/scripts/lib/common.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +check_is_installed() { + local __name="$1" + if ! is_installed "$__name"; then + echo "Please install $__name before running this script." + exit 1 + fi +} + +is_installed() { + local __name="$1" + if $(which $__name >/dev/null 2>&1); then + return 0 + else + return 1 + fi +} diff --git a/scripts/run-integration-tests.sh b/scripts/run-integration-tests.sh index f9db551a19..30961cdaba 100755 --- a/scripts/run-integration-tests.sh +++ b/scripts/run-integration-tests.sh @@ -1,18 +1,37 @@ #!/usr/bin/env bash -set -euo pipefail +set -Euo pipefail + +trap 'on_error $LINENO' ERR DIR=$(cd "$(dirname "$0")"; pwd) +source $DIR/lib/common.sh +source $DIR/lib/aws.sh source $DIR/lib/cluster.sh OS=$(go env GOOS) -ARCH=amd64 -REGION=${AWS_REGION:-us-west-2} +ARCH=$(go env GOARCH) +AWS_REGION=${AWS_REGION:-us-west-2} K8S_VERSION=${K8S_VERSION:-1.14.6} PROVISION=${PROVISION:-true} DEPROVISION=${DEPROVISION:-true} BUILD=${BUILD:-true} +__cluster_created=0 +__cluster_deprovisioned=0 + +on_error() { + # Make sure we destroy any cluster that was created if we hit run into an + # error when attempting to run tests against the cluster + if [[ $__cluster_created -eq 1 && $__cluster_deprovisioned -eq 0 && "$DEPROVISION" = true ]]; then + # prevent double-deprovisioning with ctrl-c during deprovisioning... + __cluster_deprovisioned=1 + echo "Cluster was provisioned already. Deprovisioning it..." + down-test-cluster + fi + exit 1 +} + # test specific config, results location TEST_ID=${TEST_ID:-$RANDOM} TEST_DIR=/tmp/cni-test/$(date "+%Y%M%d%H%M%S")-$TEST_ID @@ -23,55 +42,62 @@ REPORT_DIR=${TEST_DIR}/report CLUSTER_ID=${CLUSTER_ID:-$RANDOM} CLUSTER_NAME=cni-test-$CLUSTER_ID TEST_CLUSTER_DIR=/tmp/cni-test/cluster-$CLUSTER_NAME +CLUSTER_MANAGE_LOG_PATH=$TEST_CLUSTER_DIR/cluster-manage.log CLUSTER_CONFIG=${CLUSTER_CONFIG:-${TEST_CLUSTER_DIR}/${CLUSTER_NAME}.yaml} SSH_KEY_PATH=${SSH_KEY_PATH:-${TEST_CLUSTER_DIR}/id_rsa} KUBECONFIG_PATH=${KUBECONFIG_PATH:-${TEST_CLUSTER_DIR}/kubeconfig} # shared binaries -TESTER_DOWNLOAD_URL=https://github.com/aws/aws-k8s-tester/releases/download/v0.4.3/aws-k8s-tester-v0.4.3-$OS-$ARCH TESTER_DIR=${TESTER_DIR:-/tmp/aws-k8s-tester} TESTER_PATH=${TESTER_PATH:-$TESTER_DIR/aws-k8s-tester} -AUTHENTICATOR_PATH=${AUTHENTICATOR_PATH:-/tmp/aws-k8s-tester/aws-iam-authenticator} -KUBECTL_PATH=${KUBECTL_PATH:-/tmp/aws-k8s-tester/kubectl} +AUTHENTICATOR_PATH=${AUTHENTICATOR_PATH:-$TESTER_DIR/aws-iam-authenticator} +KUBECTL_PATH=${KUBECTL_PATH:-$TESTER_DIR/kubectl} + +# double-check all our preconditions and requirements have been met +check_is_installed docker +check_is_installed aws +check_aws_credentials +ensure_aws_k8s_tester + +AWS_ACCOUNT_ID=${AWS_ACCOUNT_ID:-$(aws sts get-caller-identity --query Account --output text)} +AWS_ECR_REGISTRY=${AWS_ECR_REGISTRY:-"$AWS_ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com"} +AWS_ECR_REPO_NAME=${AWS_ECR_REPO_NAME:-"amazon-k8s-cni"} +IMAGE_NAME=${IMAGE_NAME:-"$AWS_ECR_REGISTRY/$AWS_ECR_REPO_NAME"} +IMAGE_VERSION=${IMAGE_VERSION:-$(git describe --tags --always --dirty)} + +if [[ "$BUILD" = true ]]; then + # `aws ec2 get-login` returns a docker login string, which we eval here to + # login to the ECR registry + eval $(aws ecr get-login --region $AWS_REGION --no-include-email) >/dev/null 2>&1 + ensure_ecr_repo "$AWS_ACCOUNT_ID" "$AWS_ECR_REPO_NAME" + make docker IMAGE=$IMAGE_NAME VERSION=$IMAGE_VERSION + docker push $IMAGE_NAME:$IMAGE_VERSION +fi # The version substituted in ./config/X/aws-k8s-cni.yaml CNI_TEMPLATE_VERSION=${CNI_TEMPLATE_VERSION:-v1.5} -echo "Running $TEST_ID on $CLUSTER_NAME in $REGION" +echo "*******************************************************************************" +echo "Running $TEST_ID on $CLUSTER_NAME in $AWS_REGION" echo "+ Cluster config dir: $TEST_CLUSTER_DIR" echo "+ Result dir: $TEST_DIR" echo "+ Tester: $TESTER_PATH" echo "+ Kubeconfig: $KUBECONFIG_PATH" echo "+ Node SSH key: $SSH_KEY_PATH" echo "+ Cluster config: $CLUSTER_CONFIG" -echo "" +echo "+ AWS Account ID: $AWS_ACCOUNT_ID" +echo "+ CNI image to test: $IMAGE_NAME:$IMAGE_VERSION" mkdir -p $TEST_DIR mkdir -p $REPORT_DIR mkdir -p $TEST_CLUSTER_DIR -# Download aws-k8s-tester if not yet -if [[ ! -e $TESTER_PATH ]]; then - mkdir -p $TESTER_DIR - echo "Downloading aws-k8s-tester from $TESTER_DOWNLOAD_URL to $TESTER_PATH" - curl -s -L -X GET $TESTER_DOWNLOAD_URL -o $TESTER_PATH - chmod +x $TESTER_PATH -fi - if [[ "$PROVISION" = true ]]; then up-test-cluster + __cluster_created=1 fi if [[ "$BUILD" = true ]]; then - # Push test image - eval $(aws ecr get-login --region $REGION --no-include-email) - AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) - IMAGE_NAME="$AWS_ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/amazon/amazon-k8s-cni" - IMAGE_VERSION=$(git describe --tags --always --dirty) - - make docker IMAGE=$IMAGE_NAME VERSION=$IMAGE_VERSION - docker push $IMAGE_NAME:$IMAGE_VERSION - echo "Using ./config/$CNI_TEMPLATE_VERSION/aws-k8s-cni.yaml as a template" if [[ ! -f "./config/$CNI_TEMPLATE_VERSION/aws-k8s-cni.yaml" ]]; then echo "./config/$CNI_TEMPLATE_VERSION/aws-k8s-cni.yaml DOES NOT exist. Set \$CNI_TEMPLATE_VERSION to an existing directory in ./config/" @@ -82,13 +108,16 @@ if [[ "$BUILD" = true ]]; then sed -i'.bak' "s,v1.5.3,$IMAGE_VERSION," ./config/$CNI_TEMPLATE_VERSION/aws-k8s-cni.yaml fi -echo "Deploying CNI" +echo "*******************************************************************************" +echo "Deploying CNI with image $IMAGE_NAME" export KUBECONFIG=$KUBECONFIG_PATH kubectl apply -f ./config/$CNI_TEMPLATE_VERSION/aws-k8s-cni.yaml -# Run the test +echo "*******************************************************************************" +echo "Running integration tests" +echo "" pushd ./test/integration -go test -v -timeout 0 ./... --kubeconfig=$KUBECONFIG --ginkgo.focus="\[cni-integration\]" --ginkgo.skip="\[Disruptive\]" \ +GO111MODULE=on go test -v -timeout 0 ./... --kubeconfig=$KUBECONFIG --ginkgo.focus="\[cni-integration\]" --ginkgo.skip="\[Disruptive\]" \ --assets=./assets TEST_PASS=$? popd