Skip to content

Commit

Permalink
Fix test runner script
Browse files Browse the repository at this point in the history
- Fix env vars
- Fix Deprovision command
- Split test and cluster config into separate /tmp directories
- combine cluster functions into lib/cluster.sh
  • Loading branch information
nckturner committed Nov 11, 2019
1 parent fe63eb7 commit 9394228
Show file tree
Hide file tree
Showing 8 changed files with 221 additions and 77 deletions.
2 changes: 1 addition & 1 deletion config/v1.5/aws-k8s-cni.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ spec:
tolerations:
- operator: Exists
containers:
- image: 602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:v1.5.3
- image: 352684330888.dkr.ecr.us-west-2.amazonaws.com/amazon/amazon-k8s-cni:v1.6.0-rc1-31-g11512e5f-dirty
imagePullPolicy: Always
ports:
- containerPort: 61678
Expand Down
148 changes: 148 additions & 0 deletions config/v1.5/aws-k8s-cni.yaml.bak
Original file line number Diff line number Diff line change
@@ -0,0 +1,148 @@
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: aws-node
rules:
- apiGroups:
- crd.k8s.amazonaws.com
resources:
- "*"
- namespaces
verbs:
- "*"
- apiGroups: [""]
resources:
- pods
- nodes
- namespaces
verbs: ["list", "watch", "get"]
- apiGroups: ["extensions"]
resources:
- daemonsets
verbs: ["list", "watch"]

---
apiVersion: v1
kind: ServiceAccount
metadata:
name: aws-node
namespace: kube-system

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: aws-node
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: aws-node
subjects:
- kind: ServiceAccount
name: aws-node
namespace: kube-system

---
kind: DaemonSet
apiVersion: apps/v1
metadata:
name: aws-node
namespace: kube-system
labels:
k8s-app: aws-node
spec:
updateStrategy:
type: RollingUpdate
selector:
matchLabels:
k8s-app: aws-node
template:
metadata:
labels:
k8s-app: aws-node
spec:
priorityClassName: system-node-critical
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: "beta.kubernetes.io/os"
operator: In
values:
- linux
- key: "beta.kubernetes.io/arch"
operator: In
values:
- amd64
serviceAccountName: aws-node
hostNetwork: true
tolerations:
- operator: Exists
containers:
- image: 352684330888.dkr.ecr.us-west-2.amazonaws.com/amazon/amazon-k8s-cni:v1.5.3
imagePullPolicy: Always
ports:
- containerPort: 61678
name: metrics
name: aws-node
#readinessProbe:
# exec:
# command: ["/app/grpc_health_probe", "-addr=:50051"]
# initialDelaySeconds: 25
#livenessProbe:
# exec:
# command: ["/app/grpc_health_probe", "-addr=:50051"]
# initialDelaySeconds: 25
env:
- name: AWS_VPC_K8S_CNI_LOGLEVEL
value: DEBUG
- name: MY_NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
resources:
requests:
cpu: 10m
securityContext:
privileged: true
volumeMounts:
- mountPath: /host/opt/cni/bin
name: cni-bin-dir
- mountPath: /host/etc/cni/net.d
name: cni-net-dir
- mountPath: /host/var/log
name: log-dir
- mountPath: /var/run/docker.sock
name: dockersock
volumes:
- name: cni-bin-dir
hostPath:
path: /opt/cni/bin
- name: cni-net-dir
hostPath:
path: /etc/cni/net.d
- name: log-dir
hostPath:
path: /var/log
- name: dockersock
hostPath:
path: /var/run/docker.sock

---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: eniconfigs.crd.k8s.amazonaws.com
spec:
scope: Cluster
group: crd.k8s.amazonaws.com
versions:
- name: v1alpha1
served: true
storage: true
names:
plural: eniconfigs
singular: eniconfig
kind: ENIConfig
6 changes: 0 additions & 6 deletions scripts/down-test-cluster.sh

This file was deleted.

27 changes: 27 additions & 0 deletions scripts/lib/cluster.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
#!/usr/bin/env bash

function down-test-cluster() {
echo "Deleting cluster $CLUSTER_NAME"
$TESTER_PATH eks delete cluster --path $CLUSTER_CONFIG
}

function up-test-cluster() {
echo "Creating cluster $CLUSTER_NAME"
ssh-keygen -P cni-test -f $SSH_KEY_PATH

$TESTER_PATH eks create config --path $CLUSTER_CONFIG

AWS_K8S_TESTER_EKS_CLUSTER_NAME=$CLUSTER_NAME \
AWS_K8S_TESTER_EKS_KUBECONFIG_PATH=$KUBECONFIG_PATH \
AWS_K8S_TESTER_EKS_KUBERNETES_VERSION=${K8S_VERSION%.*} \
AWS_K8S_TESTER_EKS_ENABLE_WORKER_NODE_PRIVILEGED_PORT_ACCESS=true \
AWS_K8S_TESTER_EKS_WORKER_NODE_ASG_MIN=3 \
AWS_K8S_TESTER_EKS_WORKER_NODE_ASG_MAX=3 \
AWS_K8S_TESTER_EKS_WORKER_NODE_ASG_DESIRED_CAPACITY=3 \
AWS_K8S_TESTER_EKS_WORKER_NODE_PRIVATE_KEY_PATH=$SSH_KEY_PATH \
AWS_K8S_TESTER_EKS_WORKER_NODE_INSTANCE_TYPE=m3.xlarge \
AWS_K8S_TESTER_EKS_AWS_K8S_TESTER_PATH=$TESTER_PATH \
AWS_K8S_TESTER_EKS_AWS_IAM_AUTHENTICATOR_PATH=$AUTHENTICATOR_PATH \
AWS_K8S_TESTER_EKS_KUBECTL_PATH=$KUBECTL_PATH \
$TESTER_PATH eks create cluster --path $CLUSTER_CONFIG 1>&2
}
60 changes: 43 additions & 17 deletions scripts/run-integration-tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,29 +3,51 @@
set -euo pipefail

DIR=$(cd "$(dirname "$0")"; pwd)
source $DIR/up-test-cluster.sh
source $DIR/down-test-cluster.sh
source $DIR/lib/cluster.sh

OS=$(go env GOOS)
ARCH=amd64
TEST_ID=$RANDOM
CLUSTER_NAME=test-cluster-$TEST_ID
BASE_DIR=$(dirname $0)
TEST_DIR=/tmp/cni-test
REPORT_DIR=${TEST_DIR}/report
REGION=${AWS_REGION:-us-west-2}
K8S_VERSION=${K8S_VERSION:-1.14.1}
K8S_VERSION=${K8S_VERSION:-1.14.6}
PROVISION=${PROVISION:-true}
DEPROVISION=${DEPROVISION:-true}
BUILD=${BUILD:-true}

echo "Testing in region: $REGION"
# test specific config, results location
TEST_ID=${TEST_ID:-$RANDOM}
TEST_DIR=/tmp/cni-test/$(date "+%Y%M%d%H%M%S")-$TEST_ID
REPORT_DIR=${TEST_DIR}/report

# test cluster config location
# Pass in CLUSTER_ID to reuse a test cluster
CLUSTER_ID=${CLUSTER_ID:-$RANDOM}
CLUSTER_NAME=cni-test-$CLUSTER_ID
TEST_CLUSTER_DIR=/tmp/cni-test/cluster-$CLUSTER_NAME
CLUSTER_CONFIG=${CLUSTER_CONFIG:-${TEST_CLUSTER_DIR}/${CLUSTER_NAME}.yaml}
SSH_KEY_PATH=${SSH_KEY_PATH:-${TEST_CLUSTER_DIR}/id_rsa}
KUBECONFIG_PATH=${KUBECONFIG_PATH:-${TEST_CLUSTER_DIR}/kubeconfig}

# shared binaries
TESTER_DOWNLOAD_URL=https://github.com/aws/aws-k8s-tester/releases/download/v0.4.3/aws-k8s-tester-v0.4.3-$OS-$ARCH
TESTER_PATH=$TEST_DIR/aws-k8s-tester
TESTER_PATH=${TESTER_PATH:-/tmp/aws-k8s-tester/aws-k8s-tester}
AUTHENTICATOR_PATH=${AUTHENTICATOR_PATH:-/tmp/aws-k8s-tester/aws-iam-authenticator}
KUBECTL_PATH=${KUBECTL_PATH:-/tmp/aws-k8s-tester/kubectl}

# The version substituted in ./config/X/aws-k8s-cni.yaml
CNI_TEMPLATE_VERSION=${CNI_TEMPLATE_VERSION:-v1.5}

echo "Running $TEST_ID on $CLUSTER_NAME in $REGION"
echo "+ Cluster config dir: $TEST_CLUSTER_DIR"
echo "+ Result dir: $TEST_DIR"
echo "+ Tester: $TESTER_PATH"
echo "+ Kubeconfig: $KUBECONFIG_PATH"
echo "+ Node SSH key: $SSH_KEY_PATH"
echo "+ Cluster config: $CLUSTER_CONFIG"
echo ""

mkdir -p $TEST_DIR
mkdir -p $REPORT_DIR
mkdir -p $TEST_CLUSTER_DIR

# Download aws-k8s-tester if not yet
if [[ ! -e $TESTER_PATH ]]; then
Expand All @@ -48,27 +70,31 @@ if [[ "$BUILD" = true ]]; then
make docker IMAGE=$IMAGE_NAME VERSION=$IMAGE_VERSION
docker push $IMAGE_NAME:$IMAGE_VERSION

sed -i'.bak' "s,602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni,$IMAGE_NAME," ./config/v1.5/aws-k8s-cni.yaml
sed -i'.bak' "s,v1.5.3,$IMAGE_VERSION," ./config/v1.5/aws-k8s-cni.yaml
echo "Using ./config/$CNI_TEMPLATE_VERSION/aws-k8s-cni.yaml as a template"
if [[ ! -f "./config/$CNI_TEMPLATE_VERSION/aws-k8s-cni.yaml" ]]; then
echo "./config/$CNI_TEMPLATE_VERSION/aws-k8s-cni.yaml DOES NOT exist. Set \$CNI_TEMPLATE_VERSION to an existing directory in ./config/"
exit
fi

sed -i'.bak' "s,602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni,$IMAGE_NAME," ./config/$CNI_TEMPLATE_VERSION/aws-k8s-cni.yaml
sed -i'.bak' "s,v1.5.3,$IMAGE_VERSION," ./config/$CNI_TEMPLATE_VERSION/aws-k8s-cni.yaml
fi

echo "Deploying CNI"
export KUBECONFIG=/tmp/aws-k8s-tester/kubeconfig
kubectl apply -f ./config/v1.5/aws-k8s-cni.yaml
export KUBECONFIG=$KUBECONFIG_PATH
kubectl apply -f ./config/$CNI_TEMPLATE_VERSION/aws-k8s-cni.yaml

# Run the test
pushd ./test/integration
go test -v -timeout 0 ./... --kubeconfig=$KUBECONFIG --ginkgo.focus="\[cni-integration\]" --ginkgo.skip="\[Disruptive\]" \
--assets=${DIR}/../test/integration/assets
--assets=./assets
TEST_PASS=$?
popd

if [[ "$DEPROVISION" = true ]]; then
down-test-cluster
fi

rm -rf $TEST_DIR

if [[ $TEST_PASS -ne 0 ]]; then
exit 1
fi
32 changes: 0 additions & 32 deletions scripts/up-test-cluster.sh

This file was deleted.

21 changes: 0 additions & 21 deletions test/integration/assets/test-deployment.yaml

This file was deleted.

2 changes: 2 additions & 0 deletions test/integration/integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,8 @@ var _ = ginkgo.Describe("[cni-integration]", func() {

framework.ExpectNoError(
framework.CheckConnectivityToHost(f, "", "client-pod", serverPod.Status.PodIP, framework.IPv4PingCommand, 30))
err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete("server-pod", &metav1.DeleteOptions{})
framework.ExpectNoError(err, "deleting pod")
})

ginkgo.It("should enable pod-node communication", func() {
Expand Down

0 comments on commit 9394228

Please sign in to comment.