[WIP] Add run-integration-tests.sh script

- Runs integration tests
- Uses aws-k8s-tester
- Adapted from CSI test script, there may still be some stuff in here
that won't apply

go.mod

@@ -25,8 +25,8 @@ require (
 	github.com/json-iterator/go v1.1.5 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 // indirect
-	github.com/onsi/ginkgo v1.8.0 // indirect
-	github.com/onsi/gomega v1.5.0 // indirect
+	github.com/onsi/ginkgo v1.8.0
+	github.com/onsi/gomega v1.5.0
 	github.com/operator-framework/operator-sdk v0.0.7
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pkg/errors v0.8.0

scripts/down-test-cluster.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+function down-test-cluster() {
+    echo "Deleting cluster $CLUSTER_NAME"
+    $TESTER_PATH delete cluster --name $CLUSTER_NAME.k8s.local --state $KOPS_STATE_FILE --yes
+}

scripts/run-integration-tests.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+DIR=$(cd "$(dirname "$0")"; pwd)
+source $DIR/up-test-cluster.sh
+source $DIR/down-test-cluster.sh
+
+OS=$(go env GOOS)
+ARCH=amd64
+TEST_ID=$RANDOM
+CLUSTER_NAME=test-cluster-$TEST_ID
+BASE_DIR=$(dirname $0)
+TEST_DIR=/tmp/cni-test
+REPORT_DIR=${TEST_DIR}/report
+REGION=${AWS_REGION:-us-west-2}
+K8S_VERSION=${K8S_VERSION:-1.14.1}
+PROVISION=${PROVISION:-true}
+DEPROVISION=${DEPROVISION:-true}
+BUILD=${BUILD:-true}
+
+echo "Testing in region: $REGION"
+
+TESTER_DOWNLOAD_URL=https://github.com/aws/aws-k8s-tester/releases/download/v0.4.3/aws-k8s-tester-v0.4.3-$OS-$ARCH
+TESTER_PATH=$TEST_DIR/aws-k8s-tester
+
+mkdir -p $TEST_DIR
+mkdir -p $REPORT_DIR
+
+# Download aws-k8s-tester if not yet
+if [[ ! -e $TESTER_PATH ]]; then
+  echo "Downloading aws-k8s-tester from $TESTER_DOWNLOAD_URL to $TESTER_PATH"
+  curl -L -X GET $TESTER_DOWNLOAD_URL -o $TESTER_PATH
+  chmod +x $TESTER_PATH
+fi
+
+if [[ "$PROVISION" = true ]]; then
+    up-test-cluster
+fi
+
+if [[ "$BUILD" = true ]]; then
+    # Push test image
+    eval $(aws ecr get-login --region $REGION --no-include-email)
+    AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
+    IMAGE_NAME="$AWS_ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/amazon/amazon-k8s-cni"
+    IMAGE_VERSION=$(git describe --tags --always --dirty)
+
+    make docker IMAGE=$IMAGE_NAME VERSION=$IMAGE_VERSION
+    docker push $IMAGE_NAME:$IMAGE_VERSION
+
+    sed -i'.bak' "s,602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni,$IMAGE_NAME," ./config/v1.5/aws-k8s-cni.yaml
+    sed -i'.bak' "s,v1.5.3,$IMAGE_VERSION," ./config/v1.5/aws-k8s-cni.yaml
+fi
+
+echo "Deploying CNI"
+export KUBECONFIG=/tmp/aws-k8s-tester/kubeconfig
+kubectl apply -f ./config/v1.5/aws-k8s-cni.yaml
+
+# Run the test
+pushd ./test/integration
+go test -v -timeout 0 ./... --kubeconfig=$KUBECONFIG --ginkgo.focus="\[cni-integration\]" --ginkgo.skip="\[Disruptive\]" \
+    --assets=${DIR}/../test/integration/assets
+TEST_PASS=$?
+popd
+
+if [[ "$DEPROVISION" = true ]]; then
+    down-test-cluster
+fi
+
+rm -rf $TEST_DIR
+
+if [[ $TEST_PASS -ne 0 ]]; then
+  exit 1
+fi

scripts/up-test-cluster.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+function up-test-cluster() {
+    echo "Creating cluster $CLUSTER_NAME"
+    CLUSTER_YAML_PATH=$TEST_DIR/$CLUSTER_NAME.yaml
+    SSH_KEY_PATH=$TEST_DIR/id_rsa
+    ssh-keygen -P cni-test -f $SSH_KEY_PATH
+
+
+    $TESTER_PATH eks create config --path /tmp/aws-k8s-tester-eks.yaml
+
+    AWS_K8S_TESTER_EKS_KUBERNETES_VERSION=$K8S_VERSION \
+        AWS_K8S_TESTER_EKS_ENABLE_WORKER_NODE_PRIVILEGED_PORT_ACCESS=true \
+        AWS_K8S_TESTER_EKS_WORKER_NODE_ASG_MIN=3 \
+        AWS_K8S_TESTER_EKS_WORKER_NODE_ASG_MAX=3 \
+        AWS_K8S_TESTER_EKS_WORKER_NODE_ASG_DESIRED_CAPACITY=3 \
+        AWS_K8S_TESTER_EKS_WORKER_NODE_PRIVATE_KEY_PATH=$SSH_KEY_PATH \
+        AWS_K8S_TESTER_EKS_WORKER_NODE_INSTANCE_TYPE=m3.xlarge \
+        $TESTER_PATH eks create cluster --path /tmp/aws-k8s-tester-eks.yaml
+
+    # Wait for cluster creation
+    while [[ 1 ]]; do
+        $TESTER_PATH eks check cluster --path /tmp/aws-k8s-tester-eks.yaml
+        ret=$?
+        if [[ $ret -eq 0 ]]; then
+            break
+        else
+            echo "Waiting cluster to be created"
+            sleep 30
+        fi
+    done;
+}

test/integration/assets/test-deployment.yaml

@@ -0,0 +1,21 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: integration-test-app
+  labels:
+    app: test
+spec:
+  replicas: 0
+  selector:
+    matchLabels:
+      app: test
+  template:
+    metadata:
+      labels:
+        app: test
+    spec:
+      containers:
+        - name: nginx
+          image: nginx:1.17.5-alpine
+          ports:
+            - containerPort: 80

test/integration/integration_test.go

@@ -0,0 +1,96 @@
+package integration
+
+import (
+	//flag "github.com/spf13/pflag"
+	"flag"
+	"fmt"
+	"os"
+	"os/exec"
+	"testing"
+
+	"github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"k8s.io/client-go/tools/clientcmd"
+)
+
+const (
+	defaultHost = "http://127.0.0.1:8080"
+)
+
+func init() {
+	RegisterFlags(flag.CommandLine)
+}
+
+type TestContextType struct {
+	KubeConfig  string
+	KubeContext string
+	KubectlPath string
+	Host        string
+
+	AssetsDir string
+}
+
+var TestContext TestContextType
+
+func RegisterFlags(flags *flag.FlagSet) {
+	// Flags also used by the upstream test framework
+	flags.StringVar(&TestContext.KubeConfig, clientcmd.RecommendedConfigPathFlag, os.Getenv(clientcmd.RecommendedConfigPathEnvVar), "Path to kubeconfig containing embedded authinfo.")
+	flags.StringVar(&TestContext.KubeContext, clientcmd.FlagContext, "", "kubeconfig context to use/override. If unset, will use value from 'current-context'")
+	flags.StringVar(&TestContext.KubectlPath, "kubectl-path", "kubectl", "The kubectl binary to use. For development, you might use 'cluster/kubectl.sh' here.")
+	flags.StringVar(&TestContext.Host, "host", "", fmt.Sprintf("The host, or apiserver, to connect to. Will default to %s if this argument and --kubeconfig are not set", defaultHost))
+
+	// Custom flags
+	flags.StringVar(&TestContext.AssetsDir, "assets", "assets", "The directory that holds assets used by the integration test.")
+}
+
+func TestIntegration(t *testing.T) {
+	flag.Parse()
+	RegisterFailHandler(ginkgo.Fail)
+	ginkgo.RunSpecs(t, "Amazon VPC CNI Integration Tests")
+}
+
+var _ = ginkgo.BeforeSuite(func() {
+	fmt.Printf("Using KUBECONFIG=\"%s\"\n", TestContext.KubeConfig)
+})
+
+var _ = ginkgo.Describe("[cni-integration]", func() {
+	ginkgo.BeforeEach(func() {
+		//
+	})
+	ginkgo.Context("Host ip rule test", func() {
+		ginkgo.It("Should test something 1", func() {
+			Expect(1).Should(Equal(1))
+		})
+		ginkgo.It("Should test something 2", func() {
+			Expect(1).ShouldNot(Equal(2))
+		})
+	})
+
+})
+
+// KubectlCmd runs the kubectl executable through the wrapper script.
+func KubectlCmd(args ...string) *exec.Cmd {
+	defaultArgs := []string{}
+
+	// Reference a --server option so tests can run anywhere.
+	if TestContext.Host != "" {
+		defaultArgs = append(defaultArgs, "--"+clientcmd.FlagAPIServer+"="+TestContext.Host)
+	}
+	if TestContext.KubeConfig != "" {
+		defaultArgs = append(defaultArgs, "--"+clientcmd.RecommendedConfigPathFlag+"="+TestContext.KubeConfig)
+
+		// Reference the KubeContext
+		if TestContext.KubeContext != "" {
+			defaultArgs = append(defaultArgs, "--"+clientcmd.FlagContext+"="+TestContext.KubeContext)
+		}
+
+	}
+	kubectlArgs := append(defaultArgs, args...)
+
+	//We allow users to specify path to kubectl, so you can test either "kubectl" or "cluster/kubectl.sh"
+	//and so on.
+	cmd := exec.Command(TestContext.KubectlPath, kubectlArgs...)
+
+	//caller will invoke this and wait on it.
+	return cmd
+}