From 1398c565fdc3677c3109a09ea12d909638516fc4 Mon Sep 17 00:00:00 2001 From: Chinmay Gadgil Date: Mon, 14 Feb 2022 13:43:02 -0800 Subject: [PATCH 1/4] Added upgrade/downgrade script template Refactored code Addon upgrade/downgrade test similar to https://github.com/aws/amazon-vpc-cni-k8s/pull/1795 Added tests for addon upgrade/downgrade Changed DEFAULT version Added addon status checks Fetch latest addon version for given K8s Cluster Update kops cluster config used in weekly tests (#1862) * Change to kops cluster creation scripts * Add logging for retry attempt * Switch kops cluster to use docker container runtime Co-authored-by: Jayanth Varavani <1111446+jayanthvn@users.noreply.github.com> Added upgrade/downgrade test for custom cni-manifest-file Added missing files remove upgrade-downgrade.sh --- scripts/upload-resources-to-github.sh | 249 ------------------ test/framework/options.go | 22 +- test/framework/resources/aws/services/eks.go | 64 ----- test/go.mod | 1 + test/go.sum | 2 + .../host_networking_test.go | 99 +++++++ .../upgrade_downgrade_suite_test.go | 82 ++++++ .../cni/host_networking_test.go | 132 ++-------- test/integration-new/cni/pod_traffic_test.go | 68 +---- test/integration-new/common/util.go | 156 +++++++++++ 10 files changed, 381 insertions(+), 494 deletions(-) delete mode 100755 scripts/upload-resources-to-github.sh delete mode 100644 test/framework/resources/aws/services/eks.go create mode 100644 test/integration-new/cni-upgrade-downgrade/host_networking_test.go create mode 100644 test/integration-new/cni-upgrade-downgrade/upgrade_downgrade_suite_test.go create mode 100644 test/integration-new/common/util.go diff --git a/scripts/upload-resources-to-github.sh b/scripts/upload-resources-to-github.sh deleted file mode 100755 index 163aceb52f..0000000000 --- a/scripts/upload-resources-to-github.sh +++ /dev/null @@ -1,249 +0,0 @@ -#!/bin/bash -set -euo pipefail - -# Script to upload release assets to Github. -# This script cleans up after itself in cases of parital failures. i.e. either all assets are uploaded or none -SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" -VERSION=$(make -s -f $SCRIPTPATH/../Makefile version) -BUILD_DIR=$SCRIPTPATH/../build/cni-rel-yamls/$VERSION -BINARY_DIR=$SCRIPTPATH/../build/bin - -CNI_TAR_RESOURCES_FILE=$BUILD_DIR/cni_individual-resources.tar -METRICS_TAR_RESOURCES_FILE=$BUILD_DIR/cni_metrics_individual-resources.tar -CALICO_TAR_RESOURCES_FILE=$BUILD_DIR/calico_individual-resources.tar -CNI_RESOURCES_YAML=$BUILD_DIR/aws-k8s-cni -METRICS_RESOURCES_YAML=$BUILD_DIR/cni-metrics-helper -CALICO_RESOURCES_YAML=$BUILD_DIR/calico.yaml -CALICO_OPERATOR_RESOURCES_YAML=$BUILD_DIR/calico-operator.yaml -CALICO_CRS_RESOURCES_YAML=$BUILD_DIR/calico-crs.yaml -REGIONS_FILE=$SCRIPTPATH/../charts/regions.json - -BINARIES_ONLY="false" -PR_ID=$(uuidgen | cut -d '-' -f1) -BINARY_BASE="aws-vpc-cni-k8s" - -REPO="aws/amazon-vpc-cni-k8s" -GH_CLI_VERSION="0.10.1" -GH_CLI_CONFIG_PATH="${HOME}/.config/gh/config.yml" -KERNEL=$(uname -s | tr '[:upper:]' '[:lower:]') -OS="${KERNEL}" -if [[ "${KERNEL}" == "darwin" ]]; then - OS="macOS" -fi - -USAGE=$(cat << 'EOM' - Usage: upload-resources-to-github [-b] - Upload release assets to GitHub - - Example: upload-resources-to-github -b - Optional: - -b Upload binaries only [DEFAULT: upload all the assets] -EOM -) - -# Process our input arguments -while getopts "b" opt; do - case ${opt} in - b ) # Binaries only - BINARIES_ONLY="true" - ;; - \? ) - echo "$USAGE" 1>&2 - exit - ;; - esac -done - -RELEASE_ID=$(curl -s -H "Authorization: token $GITHUB_TOKEN" \ - https://api.github.com/repos/aws/amazon-vpc-cni-k8s/releases | \ - jq --arg VERSION "$VERSION" '.[] | select(.tag_name==$VERSION) | .id') -ASSET_IDS_UPLOADED=() - -trap 'handle_errors_and_cleanup $?' EXIT - -handle_errors_and_cleanup() { - if [ $1 -eq 0 ]; then - exit 0 - fi - - if [[ ${#ASSET_IDS_UPLOADED[@]} -ne 0 ]]; then - echo -e "\nCleaning up assets uploaded in the current execution of the script" - for asset_id in "${ASSET_IDS_UPLOADED[@]}"; do - echo "Deleting asset $asset_id" - curl -X DELETE \ - -H "Authorization: token $GITHUB_TOKEN" \ - "https://api.github.com/repos/aws/amazon-vpc-cni-k8s/releases/assets/$asset_id" - done - exit $1 - fi -} - -# $1: absolute path to asset -upload_asset() { - resp=$(curl --write-out '%{http_code}' --silent \ - -H "Authorization: token $GITHUB_TOKEN" \ - -H "Content-Type: $(file -b --mime-type $1)" \ - --data-binary @$1 \ - "https://uploads.github.com/repos/aws/amazon-vpc-cni-k8s/releases/$RELEASE_ID/assets?name=$(basename $1)") - - response_code=$(echo $resp | sed 's/\(.*\)}//') - response_content=$(echo $resp | sed "s/$response_code//") - - # HTTP success code expected - 201 Created - if [[ $response_code -eq 201 ]]; then - asset_id=$(echo $response_content | jq '.id') - ASSET_IDS_UPLOADED+=("$asset_id") - echo "✅ Created asset ID $asset_id successfully" - else - echo -e "❌ Upload failed with response code $response_code and message \n$response_content ❌" - exit 1 - fi -} - -RESOURCES_TO_UPLOAD=("$CALICO_OPERATOR_RESOURCES_YAML" "$CALICO_CRS_RESOURCES_YAML" "$CNI_TAR_RESOURCES_FILE" "$METRICS_TAR_RESOURCES_FILE" "$CALICO_TAR_RESOURCES_FILE") -RESOURCES_TO_COPY=("$CALICO_OPERATOR_RESOURCES_YAML" "$CALICO_CRS_RESOURCES_YAML") - -COUNT=1 -echo -e "\nUploading release assets for release id '$RELEASE_ID' to Github" -for asset in ${RESOURCES_TO_UPLOAD[@]}; do - name=$(echo $asset | tr '/' '\n' | tail -1) - echo -e "\n $((COUNT++)). $name" - upload_asset $asset -done - -while read i; do - ecrRegion=`echo $i | jq '.ecrRegion' -r` - ecrAccount=`echo $i | jq '.ecrAccount' -r` - ecrDomain=`echo $i | jq '.ecrDomain' -r` - - if [ "$ecrRegion" = "us-west-2" ]; then - NEW_CNI_RESOURCES_YAML="${CNI_RESOURCES_YAML}.yaml" - NEW_METRICS_RESOURCES_YAML="${METRICS_RESOURCES_YAML}.yaml" - elif [ "$ecrRegion" = "cn-northwest-1" ]; then - NEW_CNI_RESOURCES_YAML="${CNI_RESOURCES_YAML}-cn.yaml" - NEW_METRICS_RESOURCES_YAML="${METRICS_RESOURCES_YAML}-cn.yaml" - else - NEW_CNI_RESOURCES_YAML="${CNI_RESOURCES_YAML}-${ecrRegion}.yaml" - NEW_METRICS_RESOURCES_YAML="${METRICS_RESOURCES_YAML}-${ecrRegion}.yaml" - fi - RESOURCES_TO_UPLOAD=("$NEW_CNI_RESOURCES_YAML" "$NEW_METRICS_RESOURCES_YAML") - RESOURCES_TO_COPY=(${RESOURCES_TO_COPY[@]} "$NEW_CNI_RESOURCES_YAML" "$NEW_METRICS_RESOURCES_YAML") - COUNT=1 - echo -e "\nUploading release assets for release id '$RELEASE_ID' to Github" - for asset in ${RESOURCES_TO_UPLOAD[@]}; do - name=$(echo $asset | tr '/' '\n' | tail -1) - echo -e "\n $((COUNT++)). $name" - upload_asset $asset - done -done < <(jq -c '.[]' $REGIONS_FILE) - -echo "✅ Attach artifacts to release page done" - -echo $REPO - -if [[ -z $(command -v gh) ]] || [[ ! $(gh --version) =~ $GH_CLI_VERSION ]]; then - mkdir -p "${BUILD_DIR}"/gh - curl -Lo "${BUILD_DIR}"/gh/gh.tar.gz "https://github.com/cli/cli/releases/download/v${GH_CLI_VERSION}/gh_${GH_CLI_VERSION}_${OS}_amd64.tar.gz" - tar -C "${BUILD_DIR}"/gh -xvf "${BUILD_DIR}/gh/gh.tar.gz" - export PATH="${BUILD_DIR}/gh/gh_${GH_CLI_VERSION}_${OS}_amd64/bin:$PATH" - if [[ ! $(gh --version) =~ $GH_CLI_VERSION ]]; then - echo "❌ Failed install of github cli" - exit 4 - fi -fi - -function fail() { - echo "❌ Create PR failed" - exit 5 -} - -CLONE_DIR="${BUILD_DIR}/config-sync" -SYNC_DIR="$CLONE_DIR" -echo $SYNC_DIR -rm -rf "${SYNC_DIR}" -mkdir -p "${SYNC_DIR}" -cd "${SYNC_DIR}" -gh repo clone aws/amazon-vpc-cni-k8s -DEFAULT_BRANCH=$(git rev-parse --abbrev-ref HEAD | tr -d '\n') - CONFIG_DIR=amazon-vpc-cni-k8s/config/master - cd $CONFIG_DIR - REPO_NAME=$(echo ${REPO} | cut -d'/' -f2) - git remote set-url origin https://"${GITHUB_USERNAME}":"${GITHUB_TOKEN}"@github.com/"${GITHUB_USERNAME}"/"${REPO_NAME}".git - - git config user.name "eks-bot" - git config user.email "eks-bot@users.noreply.github.com" - - FORK_RELEASE_BRANCH="${BINARY_BASE}-${VERSION}-${PR_ID}" - git checkout -b "${FORK_RELEASE_BRANCH}" origin - - COUNT=1 - for asset in ${RESOURCES_TO_COPY[@]}; do - name=$(echo $asset | tr '/' '\n' | tail -1) - echo -e "\n $((COUNT++)). $name" - cp "$asset" . - done - - git add --all - git commit -m "${BINARY_BASE}: ${VERSION}" - -PR_BODY=$(cat << EOM -## ${BINARY_BASE} ${VERSION} Automated manifest folder Sync! 🤖🤖 - -### Description 📝 - -Updating all the generated release artifacts in master/config for master branch. - -EOM -) - - git push -u origin "${FORK_RELEASE_BRANCH}" - gh pr create --title "🥳 ${BINARY_BASE} ${VERSION} Automated manifest sync! 🥑" \ - --body "${PR_BODY}" --repo ${REPO} - - echo "✅ Manifest folder PR created for master" - -CLONE_DIR="${BUILD_DIR}/config-sync-release" -SYNC_DIR="$CLONE_DIR" -echo $SYNC_DIR -rm -rf "${SYNC_DIR}" -mkdir -p "${SYNC_DIR}" -cd "${SYNC_DIR}" -gh repo clone aws/amazon-vpc-cni-k8s -RELEASE_BRANCH=$(git branch -a --contains $VERSION | grep "upstream" | cut -d '/' -f3) -echo "Release branch $RELEASE_BRANCH" - CONFIG_DIR=amazon-vpc-cni-k8s/config/master - cd $CONFIG_DIR - REPO_NAME=$(echo ${REPO} | cut -d'/' -f2) - git remote set-url origin https://"${GITHUB_USERNAME}":"${GITHUB_TOKEN}"@github.com/"${GITHUB_USERNAME}"/"${REPO_NAME}".git - - git config user.name "eks-bot" - git config user.email "eks-bot@users.noreply.github.com" - - FORK_RELEASE_BRANCH="${BINARY_BASE}-${VERSION}-${PR_ID}" - git checkout -b "${FORK_RELEASE_BRANCH}" origin/$RELEASE_BRANCH - - COUNT=1 - for asset in ${RESOURCES_TO_COPY[@]}; do - name=$(echo $asset | tr '/' '\n' | tail -1) - echo -e "\n $((COUNT++)). $name" - cp "$asset" . - done - - git add --all - git commit -m "${BINARY_BASE}: ${VERSION}" - -PR_BODY=$(cat << EOM -## ${BINARY_BASE} ${VERSION} Automated manifest folder Sync! 🤖🤖 - -### Description 📝 - -Updating all the generated release artifacts in master/config for $RELEASE_BRANCH branch. - -EOM -) - - git push -u origin "${FORK_RELEASE_BRANCH}":$RELEASE_BRANCH - gh pr create --title "🥳 ${BINARY_BASE} ${VERSION} Automated manifest sync! 🥑" \ - --body "${PR_BODY}" --repo ${REPO} --base ${RELEASE_BRANCH} - - echo "✅ Manifest folder PR created for $RELEASE_BRANCH" \ No newline at end of file diff --git a/test/framework/options.go b/test/framework/options.go index 4c7cf34065..87e911a2e5 100644 --- a/test/framework/options.go +++ b/test/framework/options.go @@ -27,13 +27,17 @@ func init() { } type Options struct { - KubeConfig string - ClusterName string - AWSRegion string - AWSVPCID string - NgNameLabelKey string - NgNameLabelVal string - EKSEndpoint string + KubeConfig string + ClusterName string + AWSRegion string + AWSVPCID string + NgNameLabelKey string + NgNameLabelVal string + EKSEndpoint string + InitialAddon string + TargetAddon string + InitialManifest string + TargetManifest string } func (options *Options) BindFlags() { @@ -44,6 +48,10 @@ func (options *Options) BindFlags() { flag.StringVar(&options.NgNameLabelKey, "ng-name-label-key", "eks.amazonaws.com/nodegroup", "label key used to identify nodegroup name") flag.StringVar(&options.NgNameLabelVal, "ng-name-label-val", "", "label value with the nodegroup name") flag.StringVar(&options.EKSEndpoint, "eks-endpoint", "", "optional eks api server endpoint") + flag.StringVar(&options.InitialAddon, "initial-addon-version", "", "Initial CNI addon version before upgrade applied") + flag.StringVar(&options.TargetAddon, "target-addon-version", "", "Target CNI addon version after upgrade applied") + flag.StringVar(&options.InitialManifest, "initial-manifest-file", "", "Initial CNI manifest, can be local file path or remote Url") + flag.StringVar(&options.TargetManifest, "target-manifest-file", "", "Target CNI manifest, can be local file path or remote Url") } func (options *Options) Validate() error { diff --git a/test/framework/resources/aws/services/eks.go b/test/framework/resources/aws/services/eks.go deleted file mode 100644 index 838870d9b4..0000000000 --- a/test/framework/resources/aws/services/eks.go +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"). You may -// not use this file except in compliance with the License. A copy of the -// License is located at -// -// http://aws.amazon.com/apache2.0/ -// -// or in the "license" file accompanying this file. This file is distributed -// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -// express or implied. See the License for the specific language governing -// permissions and limitations under the License. - -package services - -import ( - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/eks" - "github.com/aws/aws-sdk-go/service/eks/eksiface" -) - -type EKS interface { - DescribeCluster(clusterName string) (*eks.DescribeClusterOutput, error) - CreateAddon(addon string, clusterName string) (*eks.CreateAddonOutput, error) - DeleteAddon(addon string, clusterName string) (*eks.DeleteAddonOutput, error) -} - -type defaultEKS struct { - eksiface.EKSAPI -} - -func NewEKS(session *session.Session, endpoint string) EKS { - return &defaultEKS{ - EKSAPI: eks.New(session, &aws.Config{ - Endpoint: aws.String(endpoint), - Region: session.Config.Region, - }), - } -} - -func (d defaultEKS) CreateAddon(addon string, clusterName string) (*eks.CreateAddonOutput, error) { - createAddonInput := &eks.CreateAddonInput{ - AddonName: aws.String(addon), - ClusterName: aws.String(clusterName), - } - return d.EKSAPI.CreateAddon(createAddonInput) -} - -func (d defaultEKS) DeleteAddon(addon string, clusterName string) (*eks.DeleteAddonOutput, error) { - deleteAddonInput := &eks.DeleteAddonInput{ - AddonName: aws.String(addon), - ClusterName: aws.String(clusterName), - } - return d.EKSAPI.DeleteAddon(deleteAddonInput) -} - -func (d defaultEKS) DescribeCluster(clusterName string) (*eks.DescribeClusterOutput, error) { - describeClusterInput := &eks.DescribeClusterInput{ - Name: aws.String(clusterName), - } - - return d.EKSAPI.DescribeCluster(describeClusterInput) -} diff --git a/test/go.mod b/test/go.mod index c41350c387..e79186d456 100644 --- a/test/go.mod +++ b/test/go.mod @@ -8,6 +8,7 @@ require ( github.com/aws/amazon-vpc-cni-k8s/test/agent v0.0.0-20211209222755-86ece934e91a github.com/aws/amazon-vpc-resource-controller-k8s v1.0.7 github.com/aws/aws-sdk-go v1.40.6 + github.com/go-errors/errors v1.4.2 github.com/onsi/ginkgo v1.12.1 github.com/onsi/gomega v1.11.0 github.com/pkg/errors v0.9.1 diff --git a/test/go.sum b/test/go.sum index 4143b5919d..c7e5b3a905 100644 --- a/test/go.sum +++ b/test/go.sum @@ -207,6 +207,8 @@ github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= diff --git a/test/integration-new/cni-upgrade-downgrade/host_networking_test.go b/test/integration-new/cni-upgrade-downgrade/host_networking_test.go new file mode 100644 index 0000000000..6dbec4185f --- /dev/null +++ b/test/integration-new/cni-upgrade-downgrade/host_networking_test.go @@ -0,0 +1,99 @@ +package cni_upgrade_downgrade + +import ( + "bytes" + "fmt" + "io" + "os" + "os/exec" + "time" + + "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest" + "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" + "github.com/aws/amazon-vpc-cni-k8s/test/integration-new/common" + "github.com/go-errors/errors" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + v1 "k8s.io/api/apps/v1" +) + +var _ = Describe("test host networking", func() { + var err error + var podLabelKey = "app" + var podLabelVal = "host-networking-test" + var deployment *v1.Deployment + var podInput string + var initialManifest string + var targetManifest string + + Context("when pods using IP from primary and secondary ENI are created", func() { + It("should have correct host networking setup when pods are running and cleaned up when pods are terminated", func() { + initialManifest = f.Options.InitialManifest + targetManifest = f.Options.TargetManifest + if len(targetManifest) == 0 { + err = errors.Errorf("Target Manifest file must be specified") + } + Expect(err).NotTo(HaveOccurred()) + + if len(initialManifest) != 0 { + ApplyCNIManifest(initialManifest) + } else { + By("Using existing cni manifest") + } + + // Launch enough pods so some pods end up using primary ENI IP and some using secondary + // ENI IP + deployment = manifest.NewBusyBoxDeploymentBuilder(). + Replicas(maxIPPerInterface*2). + PodLabel(podLabelKey, podLabelVal). + NodeName(primaryNode.Name). + Build() + + By("creating a deployment to launch pod using primary and secondary ENI IP") + deployment, err = f.K8sResourceManagers.DeploymentManager(). + CreateAndWaitTillDeploymentIsReady(deployment, utils.DefaultDeploymentReadyTimeout) + Expect(err).ToNot(HaveOccurred()) + + By("getting the list of pods using IP from primary and secondary ENI") + interfaceTypeToPodList := common.GetPodsOnPrimaryAndSecondaryInterface(primaryNode, podLabelKey, podLabelVal, f) + + // Primary ENI and Secondary ENI IPs are handled differently when setting up + // the host networking rule hence this check + Expect(len(interfaceTypeToPodList.PodsOnSecondaryENI)). + Should(BeNumerically(">", 0)) + Expect(len(interfaceTypeToPodList.PodsOnPrimaryENI)). + Should(BeNumerically(">", 0)) + + By("generating the pod networking validation input to be passed to tester") + podInput, err = common.GetPodNetworkingValidationInput(interfaceTypeToPodList, vpcCIDRs).Serialize() + Expect(err).NotTo(HaveOccurred()) + + By("validating host networking setup is setup correctly") + common.ValidateHostNetworking(common.NetworkingSetupSucceeds, podInput, primaryNode.Name, f) + + By("update cni to target manifest") + ApplyCNIManifest(targetManifest) + + By("deleting the deployment to test teardown") + err = f.K8sResourceManagers.DeploymentManager(). + DeleteAndWaitTillDeploymentIsDeleted(deployment) + Expect(err).ToNot(HaveOccurred()) + + By("waiting to allow CNI to tear down networking for terminated pods") + time.Sleep(time.Second * 60) + + By("validating host networking is teared down correctly") + common.ValidateHostNetworking(common.NetworkingTearDownSucceeds, podInput, primaryNode.Name, f) + }) + }) +}) + +func ApplyCNIManifest(filepath string) { + var stdoutBuf, stderrBuf bytes.Buffer + By(fmt.Sprintf("Applying manifest: %s", filepath)) + cmd := exec.Command("kubectl", "apply", "-f", filepath) + cmd.Stdout = io.MultiWriter(os.Stdout, &stdoutBuf) + cmd.Stderr = io.MultiWriter(os.Stderr, &stderrBuf) + err := cmd.Run() + Expect(err).NotTo(HaveOccurred()) +} diff --git a/test/integration-new/cni-upgrade-downgrade/upgrade_downgrade_suite_test.go b/test/integration-new/cni-upgrade-downgrade/upgrade_downgrade_suite_test.go new file mode 100644 index 0000000000..e0f8b76fd1 --- /dev/null +++ b/test/integration-new/cni-upgrade-downgrade/upgrade_downgrade_suite_test.go @@ -0,0 +1,82 @@ +package cni_upgrade_downgrade + +import ( + "fmt" + "testing" + + "github.com/aws/amazon-vpc-cni-k8s/test/framework" + k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils" + "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" +) + +const InstanceTypeNodeLabelKey = "beta.kubernetes.io/instance-type" + +var f *framework.Framework +var maxIPPerInterface int +var primaryNode v1.Node +var secondaryNode v1.Node +var vpcCIDRs []string + +func TestCNIPodNetworking(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "CNI Pod Networking Suite") +} + +var _ = BeforeSuite(func() { + f = framework.New(framework.GlobalOptions) + + By("creating test namespace") + f.K8sResourceManagers.NamespaceManager(). + CreateNamespace(utils.DefaultTestNamespace) + + By(fmt.Sprintf("getting the node with the node label key %s and value %s", + f.Options.NgNameLabelKey, f.Options.NgNameLabelVal)) + nodes, err := f.K8sResourceManagers.NodeManager().GetNodes(f.Options.NgNameLabelKey, f.Options.NgNameLabelVal) + Expect(err).ToNot(HaveOccurred()) + + By("verifying more than 1 nodes are present for the test") + Expect(len(nodes.Items)).Should(BeNumerically(">", 1)) + + // Set the primary and secondary node for testing + primaryNode = nodes.Items[0] + secondaryNode = nodes.Items[1] + + By("getting the instance type from node label " + InstanceTypeNodeLabelKey) + instanceType := primaryNode.Labels[InstanceTypeNodeLabelKey] + + By("getting the network interface details from ec2") + instanceOutput, err := f.CloudServices.EC2().DescribeInstanceType(instanceType) + Expect(err).ToNot(HaveOccurred()) + + maxIPPerInterface = int(*instanceOutput[0].NetworkInfo.Ipv4AddressesPerInterface) + + By("describing the VPC to get the VPC CIDRs") + describeVPCOutput, err := f.CloudServices.EC2().DescribeVPC(f.Options.AWSVPCID) + Expect(err).ToNot(HaveOccurred()) + + for _, cidrBlockAssociationSet := range describeVPCOutput.Vpcs[0].CidrBlockAssociationSet { + vpcCIDRs = append(vpcCIDRs, *cidrBlockAssociationSet.CidrBlock) + } + + // Set the WARM_ENI_TARGET to 0 to prevent all pods being scheduled on secondary ENI + k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, "aws-node", "kube-system", + "aws-node", map[string]string{"WARM_IP_TARGET": "3", "WARM_ENI_TARGET": "0"}) +}) + +var _ = AfterSuite(func() { + By("deleting test namespace") + f.K8sResourceManagers.NamespaceManager(). + DeleteAndWaitTillNamespaceDeleted(utils.DefaultTestNamespace) + + k8sUtils.UpdateEnvVarOnDaemonSetAndWaitUntilReady(f, "aws-node", "kube-system", + "aws-node", + nil, + map[string]struct{}{ + "WARM_IP_TARGET": {}, + "WARM_ENI_TARGET": {}, + }) +}) diff --git a/test/integration-new/cni/host_networking_test.go b/test/integration-new/cni/host_networking_test.go index 03a3abe687..2cbe5c917e 100644 --- a/test/integration-new/cni/host_networking_test.go +++ b/test/integration-new/cni/host_networking_test.go @@ -14,32 +14,21 @@ package cni import ( - "fmt" "strconv" "time" - "github.com/aws/amazon-vpc-cni-k8s/test/agent/pkg/input" v1 "k8s.io/api/core/v1" "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest" k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils" "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" + "github.com/aws/amazon-vpc-cni-k8s/test/integration-new/common" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) // TODO: Instead of passing the list of pods to the test helper, have the test helper get the pod on node - -type TestType int - -const ( - NetworkingTearDownSucceeds TestType = iota - NetworkingTearDownFails - NetworkingSetupSucceeds - NetworkingSetupFails -) - const ( AWS_VPC_ENI_MTU = "AWS_VPC_ENI_MTU" AWS_VPC_K8S_CNI_VETHPREFIX = "AWS_VPC_K8S_CNI_VETHPREFIX" @@ -76,8 +65,7 @@ var _ = Describe("test host networking", func() { Expect(err).ToNot(HaveOccurred()) By("getting the list of pods using IP from primary and secondary ENI") - interfaceTypeToPodList := - GetPodsOnPrimaryAndSecondaryInterface(primaryNode, podLabelKey, podLabelVal) + interfaceTypeToPodList := common.GetPodsOnPrimaryAndSecondaryInterface(primaryNode, podLabelKey, podLabelVal, f) // Primary ENI and Secondary ENI IPs are handled differently when setting up // the host networking rule hence this check @@ -87,11 +75,11 @@ var _ = Describe("test host networking", func() { Should(BeNumerically(">", 0)) By("generating the pod networking validation input to be passed to tester") - input, err := GetPodNetworkingValidationInput(interfaceTypeToPodList).Serialize() + input, err := common.GetPodNetworkingValidationInput(interfaceTypeToPodList, vpcCIDRs).Serialize() Expect(err).NotTo(HaveOccurred()) By("validating host networking setup is setup correctly") - ValidateHostNetworking(NetworkingSetupSucceeds, input) + common.ValidateHostNetworking(common.NetworkingSetupSucceeds, input, primaryNode.Name, f) By("deleting the deployment to test teardown") err = f.K8sResourceManagers.DeploymentManager(). @@ -102,7 +90,7 @@ var _ = Describe("test host networking", func() { time.Sleep(time.Second * 60) By("validating host networking is teared down correctly") - ValidateHostNetworking(NetworkingTearDownSucceeds, input) + common.ValidateHostNetworking(common.NetworkingTearDownSucceeds, input, primaryNode.Name, f) }) It("Validate Host Networking setup after changing MTU and Veth Prefix", func() { @@ -125,10 +113,10 @@ var _ = Describe("test host networking", func() { By("getting the list of pods using IP from primary and secondary ENI") interfaceTypeToPodList := - GetPodsOnPrimaryAndSecondaryInterface(primaryNode, podLabelKey, podLabelVal) + common.GetPodsOnPrimaryAndSecondaryInterface(primaryNode, podLabelKey, podLabelVal, f) By("generating the pod networking validation input to be passed to tester") - podNetworkingValidationInput := GetPodNetworkingValidationInput(interfaceTypeToPodList) + podNetworkingValidationInput := common.GetPodNetworkingValidationInput(interfaceTypeToPodList, vpcCIDRs) podNetworkingValidationInput.VethPrefix = NEW_VETH_PREFIX podNetworkingValidationInput.ValidateMTU = true podNetworkingValidationInput.MTU = NEW_MTU_VAL @@ -136,7 +124,7 @@ var _ = Describe("test host networking", func() { Expect(err).NotTo(HaveOccurred()) By("validating host networking setup is setup correctly with MTU check as well") - ValidateHostNetworking(NetworkingSetupSucceeds, input) + common.ValidateHostNetworking(common.NetworkingSetupSucceeds, input, primaryNode.Name, f) By("deleting the deployment to test teardown") err = f.K8sResourceManagers.DeploymentManager(). @@ -147,7 +135,7 @@ var _ = Describe("test host networking", func() { time.Sleep(time.Second * 60) By("validating host networking is teared down correctly") - ValidateHostNetworking(NetworkingTearDownSucceeds, input) + common.ValidateHostNetworking(common.NetworkingTearDownSucceeds, input, primaryNode.Name, f) }) }) @@ -165,39 +153,39 @@ var _ = Describe("test host networking", func() { CreatAndWaitTillRunning(parkingPod) Expect(err).ToNot(HaveOccurred()) - validInput, err := GetPodNetworkingValidationInput(InterfaceTypeToPodList{ + validInput, err := common.GetPodNetworkingValidationInput(common.InterfaceTypeToPodList{ PodsOnPrimaryENI: []v1.Pod{*parkingPod}, - }).Serialize() + }, vpcCIDRs).Serialize() Expect(err).NotTo(HaveOccurred()) By("first validating the tester work on valid input") - ValidateHostNetworking(NetworkingSetupSucceeds, validInput) + common.ValidateHostNetworking(common.NetworkingSetupSucceeds, validInput, primaryNode.Name, f) By("validating tester fails when invalid IP is passed") invalidPod := parkingPod.DeepCopy() invalidPod.Status.PodIP = "1.1.1.1" - invalidInput, err := GetPodNetworkingValidationInput(InterfaceTypeToPodList{ + invalidInput, err := common.GetPodNetworkingValidationInput(common.InterfaceTypeToPodList{ PodsOnPrimaryENI: []v1.Pod{*invalidPod}, - }).Serialize() + }, vpcCIDRs).Serialize() Expect(err).NotTo(HaveOccurred()) - ValidateHostNetworking(NetworkingSetupFails, invalidInput) + common.ValidateHostNetworking(common.NetworkingSetupFails, invalidInput, primaryNode.Name, f) By("validating the tester fails when invalid namespace is passed") invalidPod = parkingPod.DeepCopy() // veth pair name is generated using namespace+name so the test should fail invalidPod.Namespace = "different" - invalidInput, err = GetPodNetworkingValidationInput(InterfaceTypeToPodList{ + invalidInput, err = common.GetPodNetworkingValidationInput(common.InterfaceTypeToPodList{ PodsOnPrimaryENI: []v1.Pod{*invalidPod}, - }).Serialize() + }, vpcCIDRs).Serialize() Expect(err).NotTo(HaveOccurred()) - ValidateHostNetworking(NetworkingSetupFails, invalidInput) + common.ValidateHostNetworking(common.NetworkingSetupFails, invalidInput, primaryNode.Name, f) By("validating the tester fails when tear down check is run on running pod") - ValidateHostNetworking(NetworkingTearDownFails, validInput) + common.ValidateHostNetworking(common.NetworkingTearDownFails, validInput, primaryNode.Name, f) By("deleting the parking pod") err = f.K8sResourceManagers.PodManager(). @@ -206,85 +194,3 @@ var _ = Describe("test host networking", func() { }) }) }) - -// Validate host networking for the list of pods supplied -func ValidateHostNetworking(testType TestType, podValidationInputString string) { - testerArgs := []string{fmt.Sprintf("-pod-networking-validation-input=%s", - podValidationInputString)} - - var shouldTestPodError bool - if NetworkingSetupSucceeds == testType { - testerArgs = append(testerArgs, "-test-setup=true") - } else if NetworkingSetupFails == testType { - testerArgs = append(testerArgs, "-test-setup=true") - shouldTestPodError = true - } else if NetworkingTearDownSucceeds == testType { - testerArgs = append(testerArgs, "-test-cleanup=true") - } else if NetworkingTearDownFails == testType { - testerArgs = append(testerArgs, "-test-cleanup=true") - shouldTestPodError = true - } - - testContainer := manifest.NewTestHelperContainer(). - Command([]string{"./networking"}). - Args(testerArgs). - Build() - - testPod := manifest.NewDefaultPodBuilder(). - Container(testContainer). - NodeName(primaryNode.Name). - HostNetwork(true). - Build() - - By("creating pod to test host networking setup") - testPod, err := f.K8sResourceManagers.PodManager(). - CreateAndWaitTillPodCompleted(testPod) - logs, errLogs := f.K8sResourceManagers.PodManager(). - PodLogs(testPod.Namespace, testPod.Name) - Expect(errLogs).ToNot(HaveOccurred()) - - fmt.Fprintln(GinkgoWriter, logs) - - if shouldTestPodError { - Expect(err).To(HaveOccurred()) - } else { - Expect(err).ToNot(HaveOccurred()) - } - - By("deleting the host networking setup pod") - err = f.K8sResourceManagers.PodManager(). - DeleteAndWaitTillPodDeleted(testPod) - Expect(err).ToNot(HaveOccurred()) -} - -// GetPodNetworkingValidationInput returns input string containing the list of pods for which -// the host networking has to be tested -func GetPodNetworkingValidationInput(interfaceTypeToPodList InterfaceTypeToPodList) input.PodNetworkingValidationInput { - - ip := input.PodNetworkingValidationInput{ - VPCCidrRange: vpcCIDRs, - VethPrefix: "eni", - PodList: []input.Pod{}, - ValidateMTU: false, - } - - for _, primaryENIPod := range interfaceTypeToPodList.PodsOnPrimaryENI { - ip.PodList = append(ip.PodList, input.Pod{ - PodName: primaryENIPod.Name, - PodNamespace: primaryENIPod.Namespace, - PodIPv4Address: primaryENIPod.Status.PodIP, - IsIPFromSecondaryENI: false, - }) - } - - for _, secondaryENIPod := range interfaceTypeToPodList.PodsOnSecondaryENI { - ip.PodList = append(ip.PodList, input.Pod{ - PodName: secondaryENIPod.Name, - PodNamespace: secondaryENIPod.Namespace, - PodIPv4Address: secondaryENIPod.Status.PodIP, - IsIPFromSecondaryENI: true, - }) - } - - return ip -} diff --git a/test/integration-new/cni/pod_traffic_test.go b/test/integration-new/cni/pod_traffic_test.go index 7014ddac61..f832f5b532 100644 --- a/test/integration-new/cni/pod_traffic_test.go +++ b/test/integration-new/cni/pod_traffic_test.go @@ -18,9 +18,9 @@ import ( "strconv" "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" + "github.com/aws/amazon-vpc-cni-k8s/test/integration-new/common" "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest" - k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils" "github.com/aws/aws-sdk-go/service/ec2" . "github.com/onsi/ginkgo" @@ -64,9 +64,9 @@ var _ = Describe("test pod networking", func() { secondaryNodeDeployment *v1.Deployment // Map of Pods placed on primary/secondary ENI IP on primary node - interfaceToPodListOnPrimaryNode InterfaceTypeToPodList + interfaceToPodListOnPrimaryNode common.InterfaceTypeToPodList // Map of Pods placed on primary/secondary ENI IP on secondary node - interfaceToPodListOnSecondaryNode InterfaceTypeToPodList + interfaceToPodListOnSecondaryNode common.InterfaceTypeToPodList ) JustBeforeEach(func() { @@ -102,7 +102,7 @@ var _ = Describe("test pod networking", func() { Expect(err).ToNot(HaveOccurred()) interfaceToPodListOnPrimaryNode = - GetPodsOnPrimaryAndSecondaryInterface(primaryNode, "node", "primary") + common.GetPodsOnPrimaryAndSecondaryInterface(primaryNode, "node", "primary", f) // At least two Pods should be placed on the Primary and Secondary Interface // on the Primary and Secondary Node in order to test all possible scenarios @@ -127,7 +127,7 @@ var _ = Describe("test pod networking", func() { Expect(err).ToNot(HaveOccurred()) interfaceToPodListOnSecondaryNode = - GetPodsOnPrimaryAndSecondaryInterface(secondaryNode, "node", "secondary") + common.GetPodsOnPrimaryAndSecondaryInterface(secondaryNode, "node", "secondary", f) // Same reason as mentioned above Expect(len(interfaceToPodListOnSecondaryNode.PodsOnPrimaryENI)). @@ -282,8 +282,8 @@ func VerifyConnectivityFailsForNegativeCase(senderPod coreV1.Pod, receiverPod co // CheckConnectivityForMultiplePodPlacement checks connectivity for various scenarios, an example // connection from Pod on Node 1 having IP from Primary Network Interface to Pod on Node 2 having // IP from Secondary Network Interface -func CheckConnectivityForMultiplePodPlacement(interfaceToPodListOnPrimaryNode InterfaceTypeToPodList, - interfaceToPodListOnSecondaryNode InterfaceTypeToPodList, port int, +func CheckConnectivityForMultiplePodPlacement(interfaceToPodListOnPrimaryNode common.InterfaceTypeToPodList, + interfaceToPodListOnSecondaryNode common.InterfaceTypeToPodList, port int, testerExpectedStdOut string, testerExpectedStdErr string, getTestCommandFunc func(receiverPod coreV1.Pod, port int) []string) { @@ -343,57 +343,3 @@ func testConnectivity(senderPod coreV1.Pod, receiverPod coreV1.Pod, expectedStdo Expect(stdErr).To(ContainSubstring(expectedStderr)) Expect(stdOut).To(ContainSubstring(expectedStdout)) } - -type InterfaceTypeToPodList struct { - PodsOnPrimaryENI []coreV1.Pod - PodsOnSecondaryENI []coreV1.Pod -} - -// GetPodsOnPrimaryAndSecondaryInterface returns the list of Pods on Primary Networking -// Interface and Secondary Network Interface on a given Node -func GetPodsOnPrimaryAndSecondaryInterface(node coreV1.Node, - podLabelKey string, podLabelVal string) InterfaceTypeToPodList { - podList, err := f.K8sResourceManagers. - PodManager(). - GetPodsWithLabelSelector(podLabelKey, podLabelVal) - Expect(err).ToNot(HaveOccurred()) - - instance, err := f.CloudServices.EC2(). - DescribeInstance(k8sUtils.GetInstanceIDFromNode(node)) - Expect(err).ToNot(HaveOccurred()) - - interfaceToPodList := InterfaceTypeToPodList{ - PodsOnPrimaryENI: []coreV1.Pod{}, - PodsOnSecondaryENI: []coreV1.Pod{}, - } - - ipToPod := map[string]coreV1.Pod{} - for _, pod := range podList.Items { - ipToPod[pod.Status.PodIP] = pod - } - - for _, nwInterface := range instance.NetworkInterfaces { - isPrimary := IsPrimaryENI(nwInterface, instance.PrivateIpAddress) - for _, ip := range nwInterface.PrivateIpAddresses { - if pod, found := ipToPod[*ip.PrivateIpAddress]; found { - if isPrimary { - interfaceToPodList.PodsOnPrimaryENI = - append(interfaceToPodList.PodsOnPrimaryENI, pod) - } else { - interfaceToPodList.PodsOnSecondaryENI = - append(interfaceToPodList.PodsOnSecondaryENI, pod) - } - } - } - } - return interfaceToPodList -} - -func IsPrimaryENI(nwInterface *ec2.InstanceNetworkInterface, instanceIPAddr *string) bool { - for _, privateIPAddress := range nwInterface.PrivateIpAddresses { - if *privateIPAddress.PrivateIpAddress == *instanceIPAddr { - return true - } - } - return false -} diff --git a/test/integration-new/common/util.go b/test/integration-new/common/util.go new file mode 100644 index 0000000000..ea777aebb3 --- /dev/null +++ b/test/integration-new/common/util.go @@ -0,0 +1,156 @@ +package common + +import ( + "fmt" + + "github.com/aws/amazon-vpc-cni-k8s/test/agent/pkg/input" + "github.com/aws/amazon-vpc-cni-k8s/test/framework" + "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest" + k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils" + "github.com/aws/aws-sdk-go/service/ec2" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + coreV1 "k8s.io/api/core/v1" +) + +type TestType int + +const ( + NetworkingTearDownSucceeds TestType = iota + NetworkingTearDownFails + NetworkingSetupSucceeds + NetworkingSetupFails +) + +type InterfaceTypeToPodList struct { + PodsOnPrimaryENI []coreV1.Pod + PodsOnSecondaryENI []coreV1.Pod +} + +func GetPodNetworkingValidationInput(interfaceTypeToPodList InterfaceTypeToPodList, vpcCIDRs []string) input.PodNetworkingValidationInput { + + ip := input.PodNetworkingValidationInput{ + VPCCidrRange: vpcCIDRs, + VethPrefix: "eni", + PodList: []input.Pod{}, + ValidateMTU: false, + } + + for _, primaryENIPod := range interfaceTypeToPodList.PodsOnPrimaryENI { + ip.PodList = append(ip.PodList, input.Pod{ + PodName: primaryENIPod.Name, + PodNamespace: primaryENIPod.Namespace, + PodIPv4Address: primaryENIPod.Status.PodIP, + IsIPFromSecondaryENI: false, + }) + } + + for _, secondaryENIPod := range interfaceTypeToPodList.PodsOnSecondaryENI { + ip.PodList = append(ip.PodList, input.Pod{ + PodName: secondaryENIPod.Name, + PodNamespace: secondaryENIPod.Namespace, + PodIPv4Address: secondaryENIPod.Status.PodIP, + IsIPFromSecondaryENI: true, + }) + } + + return ip +} + +// Validate host networking for the list of pods supplied +func ValidateHostNetworking(testType TestType, podValidationInputString string, nodeName string, f *framework.Framework) { + testerArgs := []string{fmt.Sprintf("-pod-networking-validation-input=%s", + podValidationInputString)} + + var shouldTestPodError bool + if NetworkingSetupSucceeds == testType { + testerArgs = append(testerArgs, "-test-setup=true") + } else if NetworkingSetupFails == testType { + testerArgs = append(testerArgs, "-test-setup=true") + shouldTestPodError = true + } else if NetworkingTearDownSucceeds == testType { + testerArgs = append(testerArgs, "-test-cleanup=true") + } else if NetworkingTearDownFails == testType { + testerArgs = append(testerArgs, "-test-cleanup=true") + shouldTestPodError = true + } + + testContainer := manifest.NewTestHelperContainer(). + Command([]string{"./networking"}). + Args(testerArgs). + Build() + + testPod := manifest.NewDefaultPodBuilder(). + Container(testContainer). + NodeName(nodeName). + HostNetwork(true). + Build() + + By("creating pod to test host networking setup") + testPod, err := f.K8sResourceManagers.PodManager(). + CreateAndWaitTillPodCompleted(testPod) + logs, errLogs := f.K8sResourceManagers.PodManager(). + PodLogs(testPod.Namespace, testPod.Name) + Expect(errLogs).ToNot(HaveOccurred()) + + fmt.Fprintln(GinkgoWriter, logs) + + if shouldTestPodError { + Expect(err).To(HaveOccurred()) + } else { + Expect(err).ToNot(HaveOccurred()) + } + + By("deleting the host networking setup pod") + err = f.K8sResourceManagers.PodManager(). + DeleteAndWaitTillPodDeleted(testPod) + Expect(err).ToNot(HaveOccurred()) +} + +// GetPodsOnPrimaryAndSecondaryInterface returns the list of Pods on Primary Networking +// Interface and Secondary Network Interface on a given Node +func GetPodsOnPrimaryAndSecondaryInterface(node coreV1.Node, + podLabelKey string, podLabelVal string, f *framework.Framework) InterfaceTypeToPodList { + podList, err := f.K8sResourceManagers.PodManager(). + GetPodsWithLabelSelector(podLabelKey, podLabelVal) + Expect(err).ToNot(HaveOccurred()) + + instance, err := f.CloudServices.EC2(). + DescribeInstance(k8sUtils.GetInstanceIDFromNode(node)) + Expect(err).ToNot(HaveOccurred()) + + interfaceToPodList := InterfaceTypeToPodList{ + PodsOnPrimaryENI: []coreV1.Pod{}, + PodsOnSecondaryENI: []coreV1.Pod{}, + } + + ipToPod := map[string]coreV1.Pod{} + for _, pod := range podList.Items { + ipToPod[pod.Status.PodIP] = pod + } + + for _, nwInterface := range instance.NetworkInterfaces { + isPrimary := IsPrimaryENI(nwInterface, instance.PrivateIpAddress) + for _, ip := range nwInterface.PrivateIpAddresses { + if pod, found := ipToPod[*ip.PrivateIpAddress]; found { + if isPrimary { + interfaceToPodList.PodsOnPrimaryENI = + append(interfaceToPodList.PodsOnPrimaryENI, pod) + } else { + interfaceToPodList.PodsOnSecondaryENI = + append(interfaceToPodList.PodsOnSecondaryENI, pod) + } + } + } + } + return interfaceToPodList +} + +func IsPrimaryENI(nwInterface *ec2.InstanceNetworkInterface, instanceIPAddr *string) bool { + for _, privateIPAddress := range nwInterface.PrivateIpAddresses { + if *privateIPAddress.PrivateIpAddress == *instanceIPAddr { + return true + } + } + return false +} From 33fb6c0f3d7ec550bbcd5f14568b877f8a494ff7 Mon Sep 17 00:00:00 2001 From: Chinmay Gadgil Date: Thu, 7 Apr 2022 14:53:19 -0700 Subject: [PATCH 2/4] Add eks.go file , deleted by mistake --- test/framework/resources/aws/services/eks.go | 64 ++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 test/framework/resources/aws/services/eks.go diff --git a/test/framework/resources/aws/services/eks.go b/test/framework/resources/aws/services/eks.go new file mode 100644 index 0000000000..838870d9b4 --- /dev/null +++ b/test/framework/resources/aws/services/eks.go @@ -0,0 +1,64 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package services + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/eks" + "github.com/aws/aws-sdk-go/service/eks/eksiface" +) + +type EKS interface { + DescribeCluster(clusterName string) (*eks.DescribeClusterOutput, error) + CreateAddon(addon string, clusterName string) (*eks.CreateAddonOutput, error) + DeleteAddon(addon string, clusterName string) (*eks.DeleteAddonOutput, error) +} + +type defaultEKS struct { + eksiface.EKSAPI +} + +func NewEKS(session *session.Session, endpoint string) EKS { + return &defaultEKS{ + EKSAPI: eks.New(session, &aws.Config{ + Endpoint: aws.String(endpoint), + Region: session.Config.Region, + }), + } +} + +func (d defaultEKS) CreateAddon(addon string, clusterName string) (*eks.CreateAddonOutput, error) { + createAddonInput := &eks.CreateAddonInput{ + AddonName: aws.String(addon), + ClusterName: aws.String(clusterName), + } + return d.EKSAPI.CreateAddon(createAddonInput) +} + +func (d defaultEKS) DeleteAddon(addon string, clusterName string) (*eks.DeleteAddonOutput, error) { + deleteAddonInput := &eks.DeleteAddonInput{ + AddonName: aws.String(addon), + ClusterName: aws.String(clusterName), + } + return d.EKSAPI.DeleteAddon(deleteAddonInput) +} + +func (d defaultEKS) DescribeCluster(clusterName string) (*eks.DescribeClusterOutput, error) { + describeClusterInput := &eks.DescribeClusterInput{ + Name: aws.String(clusterName), + } + + return d.EKSAPI.DescribeCluster(describeClusterInput) +} From 9f8373798992c6eda68736cad49f5409e9b58cb5 Mon Sep 17 00:00:00 2001 From: Chinmay Gadgil Date: Tue, 26 Apr 2022 11:00:29 -0700 Subject: [PATCH 3/4] Extract apply manifest logic in common Remove redundant code --- .../host_networking_test.go | 19 ++----------------- test/integration-new/common/util.go | 14 ++++++++++++++ .../integration-new/ipamd/ipamd_suite_test.go | 12 ------------ 3 files changed, 16 insertions(+), 29 deletions(-) diff --git a/test/integration-new/cni-upgrade-downgrade/host_networking_test.go b/test/integration-new/cni-upgrade-downgrade/host_networking_test.go index 6dbec4185f..e7fcf17107 100644 --- a/test/integration-new/cni-upgrade-downgrade/host_networking_test.go +++ b/test/integration-new/cni-upgrade-downgrade/host_networking_test.go @@ -1,11 +1,6 @@ package cni_upgrade_downgrade import ( - "bytes" - "fmt" - "io" - "os" - "os/exec" "time" "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest" @@ -36,7 +31,7 @@ var _ = Describe("test host networking", func() { Expect(err).NotTo(HaveOccurred()) if len(initialManifest) != 0 { - ApplyCNIManifest(initialManifest) + common.ApplyCNIManifest(initialManifest) } else { By("Using existing cni manifest") } @@ -72,7 +67,7 @@ var _ = Describe("test host networking", func() { common.ValidateHostNetworking(common.NetworkingSetupSucceeds, podInput, primaryNode.Name, f) By("update cni to target manifest") - ApplyCNIManifest(targetManifest) + common.ApplyCNIManifest(targetManifest) By("deleting the deployment to test teardown") err = f.K8sResourceManagers.DeploymentManager(). @@ -87,13 +82,3 @@ var _ = Describe("test host networking", func() { }) }) }) - -func ApplyCNIManifest(filepath string) { - var stdoutBuf, stderrBuf bytes.Buffer - By(fmt.Sprintf("Applying manifest: %s", filepath)) - cmd := exec.Command("kubectl", "apply", "-f", filepath) - cmd.Stdout = io.MultiWriter(os.Stdout, &stdoutBuf) - cmd.Stderr = io.MultiWriter(os.Stderr, &stderrBuf) - err := cmd.Run() - Expect(err).NotTo(HaveOccurred()) -} diff --git a/test/integration-new/common/util.go b/test/integration-new/common/util.go index ea777aebb3..5ee15efc1d 100644 --- a/test/integration-new/common/util.go +++ b/test/integration-new/common/util.go @@ -1,7 +1,11 @@ package common import ( + "bytes" "fmt" + "io" + "os" + "os/exec" "github.com/aws/amazon-vpc-cni-k8s/test/agent/pkg/input" "github.com/aws/amazon-vpc-cni-k8s/test/framework" @@ -154,3 +158,13 @@ func IsPrimaryENI(nwInterface *ec2.InstanceNetworkInterface, instanceIPAddr *str } return false } + +func ApplyCNIManifest(filepath string) { + var stdoutBuf, stderrBuf bytes.Buffer + By(fmt.Sprintf("Applying manifest: %s", filepath)) + cmd := exec.Command("kubectl", "apply", "-f", filepath) + cmd.Stdout = io.MultiWriter(os.Stdout, &stdoutBuf) + cmd.Stderr = io.MultiWriter(os.Stderr, &stderrBuf) + err := cmd.Run() + Expect(err).NotTo(HaveOccurred()) +} diff --git a/test/integration-new/ipamd/ipamd_suite_test.go b/test/integration-new/ipamd/ipamd_suite_test.go index cfa9319c2e..15735d056c 100644 --- a/test/integration-new/ipamd/ipamd_suite_test.go +++ b/test/integration-new/ipamd/ipamd_suite_test.go @@ -52,16 +52,4 @@ var _ = BeforeSuite(func() { instanceID := k8sUtils.GetInstanceIDFromNode(primaryNode) primaryInstance, err = f.CloudServices.EC2().DescribeInstance(instanceID) Expect(err).ToNot(HaveOccurred()) - - By("Delete coredns addon if it exists") - _, addonDeleteError = f.CloudServices.EKS().DeleteAddon("coredns", f.Options.ClusterName) - -}) - -var _ = AfterSuite(func() { - if addonDeleteError == nil { - By("Restore coredns addon") - _, err := f.CloudServices.EKS().CreateAddon("coredns", f.Options.ClusterName) - Expect(err).NotTo(HaveOccurred()) - } }) From 72ddee219b9be82da38c332d50b7976631faa8cf Mon Sep 17 00:00:00 2001 From: Chinmay Gadgil Date: Wed, 18 May 2022 18:47:11 -0700 Subject: [PATCH 4/4] Add PD traffic test for cni upgrade downgrade test --- test/go.mod | 2 - .../host_networking_test.go | 28 ++---- .../pod_traffic_PD_enabled_test.go | 94 +++++++++++++++++++ .../upgrade_downgrade_suite_test.go | 40 +++++++- .../cni/pod_traffic_test_PD_enabled.go | 42 +-------- test/integration/common/util.go | 36 ++++++- 6 files changed, 178 insertions(+), 64 deletions(-) create mode 100644 test/integration/cni-upgrade-downgrade/pod_traffic_PD_enabled_test.go diff --git a/test/go.mod b/test/go.mod index 2dc8932d0f..4011eddd46 100644 --- a/test/go.mod +++ b/test/go.mod @@ -8,8 +8,6 @@ require ( github.com/aws/amazon-vpc-cni-k8s/test/agent v0.0.0-20211209222755-86ece934e91a github.com/aws/amazon-vpc-resource-controller-k8s v1.0.7 github.com/aws/aws-sdk-go v1.40.6 - github.com/go-errors/errors v1.0.1 - github.com/onsi/ginkgo v1.16.5 github.com/onsi/ginkgo/v2 v2.1.3 github.com/onsi/gomega v1.18.1 github.com/pkg/errors v0.9.1 diff --git a/test/integration/cni-upgrade-downgrade/host_networking_test.go b/test/integration/cni-upgrade-downgrade/host_networking_test.go index e7fcf17107..50e5540f6e 100644 --- a/test/integration/cni-upgrade-downgrade/host_networking_test.go +++ b/test/integration/cni-upgrade-downgrade/host_networking_test.go @@ -5,9 +5,8 @@ import ( "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest" "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" - "github.com/aws/amazon-vpc-cni-k8s/test/integration-new/common" - "github.com/go-errors/errors" - . "github.com/onsi/ginkgo" + "github.com/aws/amazon-vpc-cni-k8s/test/integration/common" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" v1 "k8s.io/api/apps/v1" ) @@ -18,23 +17,11 @@ var _ = Describe("test host networking", func() { var podLabelVal = "host-networking-test" var deployment *v1.Deployment var podInput string - var initialManifest string - var targetManifest string Context("when pods using IP from primary and secondary ENI are created", func() { It("should have correct host networking setup when pods are running and cleaned up when pods are terminated", func() { - initialManifest = f.Options.InitialManifest - targetManifest = f.Options.TargetManifest - if len(targetManifest) == 0 { - err = errors.Errorf("Target Manifest file must be specified") - } - Expect(err).NotTo(HaveOccurred()) - - if len(initialManifest) != 0 { - common.ApplyCNIManifest(initialManifest) - } else { - By("Using existing cni manifest") - } + By("applying initial cni manifest") + common.ApplyCNIManifest(initialManifest) // Launch enough pods so some pods end up using primary ENI IP and some using secondary // ENI IP @@ -66,7 +53,7 @@ var _ = Describe("test host networking", func() { By("validating host networking setup is setup correctly") common.ValidateHostNetworking(common.NetworkingSetupSucceeds, podInput, primaryNode.Name, f) - By("update cni to target manifest") + By("applying target cni manifest") common.ApplyCNIManifest(targetManifest) By("deleting the deployment to test teardown") @@ -80,5 +67,10 @@ var _ = Describe("test host networking", func() { By("validating host networking is teared down correctly") common.ValidateHostNetworking(common.NetworkingTearDownSucceeds, podInput, primaryNode.Name, f) }) + + AfterEach(func() { + By("revert to initial cni manifest") + common.ApplyCNIManifest(initialManifest) + }) }) }) diff --git a/test/integration/cni-upgrade-downgrade/pod_traffic_PD_enabled_test.go b/test/integration/cni-upgrade-downgrade/pod_traffic_PD_enabled_test.go new file mode 100644 index 0000000000..01e2074f52 --- /dev/null +++ b/test/integration/cni-upgrade-downgrade/pod_traffic_PD_enabled_test.go @@ -0,0 +1,94 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package cni_upgrade_downgrade + +import ( + "fmt" + + "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest" + k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils" + "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" + "github.com/aws/amazon-vpc-cni-k8s/test/integration/common" + + . "github.com/onsi/ginkgo/v2" +) + +var _ = Describe("Test pod networking with prefix delegation enabled", func() { + var ( + serverDeploymentBuilder *manifest.DeploymentBuilder + // Value for the Environment variable ENABLE_PREFIX_DELEGATION + enableIPv4PrefixDelegation string + ) + + JustBeforeEach(func() { + By("applying initial cni manifest") + common.ApplyCNIManifest(initialManifest) + + By("creating deployment") + serverDeploymentBuilder = manifest.NewDefaultDeploymentBuilder(). + Name("traffic-server"). + NodeSelector(f.Options.NgNameLabelKey, f.Options.NgNameLabelVal) + + By("set PD") + k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, + utils.AwsNodeNamespace, utils.AwsNodeName, + map[string]string{"ENABLE_PREFIX_DELEGATION": enableIPv4PrefixDelegation}) + }) + + JustAfterEach(func() { + // Revert to Initial manifest file + By("revert to initial cni manifest") + common.ApplyCNIManifest(initialManifest) + + k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, + utils.AwsNodeNamespace, utils.AwsNodeName, + map[string]string{"ENABLE_PREFIX_DELEGATION": "false"}) + }) + + Context("when testing TCP traffic between client and server pods", func() { + BeforeEach(func() { + enableIPv4PrefixDelegation = "true" + }) + + //TODO : Add pod IP validation if IP belongs to prefix or SIP + //TODO : remove hardcoding from client/server count + It("should have 99+% success rate", func() { + By("test with initial cni manifest file") + common.ValidateTraffic(f, serverDeploymentBuilder, 99, "tcp") + targetManifestStr := fmt.Sprintf("Testing with Target CNI Manifest: %s", targetManifest) + By(targetManifestStr) + By("Applying Taget CNI Manifest") + common.ApplyCNIManifest(targetManifest) + common.ValidateTraffic(f, serverDeploymentBuilder, 99, "tcp") + }) + }) + + Context("when testing UDP traffic between client and server pods", func() { + BeforeEach(func() { + enableIPv4PrefixDelegation = "true" + }) + + //TODO : Add pod IP validation if IP belongs to prefix or SIP + //TODO : remove hardcoding from client/server count + It("should have 99+% success rate", func() { + By("test with initial cni manifest file") + common.ValidateTraffic(f, serverDeploymentBuilder, 99, "udp") + targetManifestStr := fmt.Sprintf("Testing with Target CNI Manifest: %s", targetManifest) + By(targetManifestStr) + By("Applying Taget CNI Manifest") + common.ApplyCNIManifest(targetManifest) + common.ValidateTraffic(f, serverDeploymentBuilder, 99, "udp") + }) + }) +}) diff --git a/test/integration/cni-upgrade-downgrade/upgrade_downgrade_suite_test.go b/test/integration/cni-upgrade-downgrade/upgrade_downgrade_suite_test.go index e0f8b76fd1..0250f9740a 100644 --- a/test/integration/cni-upgrade-downgrade/upgrade_downgrade_suite_test.go +++ b/test/integration/cni-upgrade-downgrade/upgrade_downgrade_suite_test.go @@ -7,19 +7,26 @@ import ( "github.com/aws/amazon-vpc-cni-k8s/test/framework" k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils" "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" + "github.com/pkg/errors" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" v1 "k8s.io/api/core/v1" ) -const InstanceTypeNodeLabelKey = "beta.kubernetes.io/instance-type" +const ( + InstanceTypeNodeLabelKey = "beta.kubernetes.io/instance-type" + DefaultManifestfile = "https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/master/config/master/aws-k8s-cni.yaml" +) var f *framework.Framework var maxIPPerInterface int var primaryNode v1.Node var secondaryNode v1.Node +var instanceSecurityGroupID string var vpcCIDRs []string +var initialManifest string +var targetManifest string func TestCNIPodNetworking(t *testing.T) { RegisterFailHandler(Fail) @@ -45,6 +52,15 @@ var _ = BeforeSuite(func() { primaryNode = nodes.Items[0] secondaryNode = nodes.Items[1] + // Get the node security group + instanceID := k8sUtils.GetInstanceIDFromNode(primaryNode) + primaryInstance, err := f.CloudServices.EC2().DescribeInstance(instanceID) + Expect(err).ToNot(HaveOccurred()) + + // This won't work if the first SG is only associated with the primary instance. + // Need a robust substring in the SGP name to identify node SGP + instanceSecurityGroupID = *primaryInstance.NetworkInterfaces[0].Groups[0].GroupId + By("getting the instance type from node label " + InstanceTypeNodeLabelKey) instanceType := primaryNode.Labels[InstanceTypeNodeLabelKey] @@ -52,7 +68,8 @@ var _ = BeforeSuite(func() { instanceOutput, err := f.CloudServices.EC2().DescribeInstanceType(instanceType) Expect(err).ToNot(HaveOccurred()) - maxIPPerInterface = int(*instanceOutput[0].NetworkInfo.Ipv4AddressesPerInterface) + // Subtract 2 for coredns pods if present, and both could be on same ENI + maxIPPerInterface = int(*instanceOutput[0].NetworkInfo.Ipv4AddressesPerInterface) - 2 By("describing the VPC to get the VPC CIDRs") describeVPCOutput, err := f.CloudServices.EC2().DescribeVPC(f.Options.AWSVPCID) @@ -62,6 +79,23 @@ var _ = BeforeSuite(func() { vpcCIDRs = append(vpcCIDRs, *cidrBlockAssociationSet.CidrBlock) } + initialManifest = f.Options.InitialManifest + targetManifest = f.Options.TargetManifest + if len(targetManifest) == 0 { + err = errors.Errorf("Target Manifest file must be specified") + } + Expect(err).NotTo(HaveOccurred()) + + if len(initialManifest) == 0 { + initialManifest = DefaultManifestfile + } + + initialManifestStr := fmt.Sprintf("using initial cni manifest: %s", initialManifest) + targetManifestStr := fmt.Sprintf("using target cni manifest: %s", targetManifest) + + By(initialManifestStr) + By(targetManifestStr) + // Set the WARM_ENI_TARGET to 0 to prevent all pods being scheduled on secondary ENI k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, "aws-node", "kube-system", "aws-node", map[string]string{"WARM_IP_TARGET": "3", "WARM_ENI_TARGET": "0"}) diff --git a/test/integration/cni/pod_traffic_test_PD_enabled.go b/test/integration/cni/pod_traffic_test_PD_enabled.go index efd0af7f4b..4a716fd625 100644 --- a/test/integration/cni/pod_traffic_test_PD_enabled.go +++ b/test/integration/cni/pod_traffic_test_PD_enabled.go @@ -14,22 +14,16 @@ package cni import ( - "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/agent" "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest" k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils" "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" + "github.com/aws/amazon-vpc-cni-k8s/test/integration/common" . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" ) var _ = Describe("Test pod networking with prefix delegation enabled", func() { var ( - // The Pod labels for client and server in order to retrieve the - // client and server Pods belonging to a Deployment/Jobs - labelKey = "app" - serverPodLabelVal = "server-pod" - clientPodLabelVal = "client-pod" serverDeploymentBuilder *manifest.DeploymentBuilder // Value for the Environment variable ENABLE_PREFIX_DELEGATION enableIPv4PrefixDelegation string @@ -61,22 +55,7 @@ var _ = Describe("Test pod networking with prefix delegation enabled", func() { //TODO : Add pod IP validation if IP belongs to prefix or SIP //TODO : remove hardcoding from client/server count It("should have 99+% success rate", func() { - trafficTester := agent.TrafficTest{ - Framework: f, - TrafficServerDeploymentBuilder: serverDeploymentBuilder, - ServerPort: 2273, - ServerProtocol: "tcp", - ClientCount: 20, - ServerCount: 20, - ServerPodLabelKey: labelKey, - ServerPodLabelVal: serverPodLabelVal, - ClientPodLabelKey: labelKey, - ClientPodLabelVal: clientPodLabelVal, - } - - successRate, err := trafficTester.TestTraffic() - Expect(err).ToNot(HaveOccurred()) - Expect(successRate).Should(BeNumerically(">=", float64(99))) + common.ValidateTraffic(f, serverDeploymentBuilder, 99, "tcp") }) }) @@ -88,22 +67,7 @@ var _ = Describe("Test pod networking with prefix delegation enabled", func() { //TODO : Add pod IP validation if IP belongs to prefix or SIP //TODO : remove hardcoding from client/server count It("should have 99+% success rate", func() { - trafficTester := agent.TrafficTest{ - Framework: f, - TrafficServerDeploymentBuilder: serverDeploymentBuilder, - ServerPort: 2273, - ServerProtocol: "udp", - ClientCount: 20, - ServerCount: 20, - ServerPodLabelKey: labelKey, - ServerPodLabelVal: serverPodLabelVal, - ClientPodLabelKey: labelKey, - ClientPodLabelVal: clientPodLabelVal, - } - - successRate, err := trafficTester.TestTraffic() - Expect(err).ToNot(HaveOccurred()) - Expect(successRate).Should(BeNumerically(">=", float64(99))) + common.ValidateTraffic(f, serverDeploymentBuilder, 99, "udp") }) }) }) diff --git a/test/integration/common/util.go b/test/integration/common/util.go index 5ee15efc1d..9252ae4fde 100644 --- a/test/integration/common/util.go +++ b/test/integration/common/util.go @@ -9,16 +9,25 @@ import ( "github.com/aws/amazon-vpc-cni-k8s/test/agent/pkg/input" "github.com/aws/amazon-vpc-cni-k8s/test/framework" + "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/agent" "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest" k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils" "github.com/aws/aws-sdk-go/service/ec2" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" coreV1 "k8s.io/api/core/v1" ) type TestType int +var ( + // The Pod labels for client and server in order to retrieve the + // client and server Pods belonging to a Deployment/Jobs + labelKey = "app" + serverPodLabelVal = "server-pod" + clientPodLabelVal = "client-pod" +) + const ( NetworkingTearDownSucceeds TestType = iota NetworkingTearDownFails @@ -150,6 +159,21 @@ func GetPodsOnPrimaryAndSecondaryInterface(node coreV1.Node, return interfaceToPodList } +func GetTrafficTestConfig(f *framework.Framework, protocol string, serverDeploymentBuilder *manifest.DeploymentBuilder, clientCount int, serverCount int) agent.TrafficTest { + return agent.TrafficTest{ + Framework: f, + TrafficServerDeploymentBuilder: serverDeploymentBuilder, + ServerPort: 2273, + ServerProtocol: protocol, + ClientCount: clientCount, + ServerCount: serverCount, + ServerPodLabelKey: labelKey, + ServerPodLabelVal: serverPodLabelVal, + ClientPodLabelKey: labelKey, + ClientPodLabelVal: clientPodLabelVal, + } +} + func IsPrimaryENI(nwInterface *ec2.InstanceNetworkInterface, instanceIPAddr *string) bool { for _, privateIPAddress := range nwInterface.PrivateIpAddresses { if *privateIPAddress.PrivateIpAddress == *instanceIPAddr { @@ -161,10 +185,18 @@ func IsPrimaryENI(nwInterface *ec2.InstanceNetworkInterface, instanceIPAddr *str func ApplyCNIManifest(filepath string) { var stdoutBuf, stderrBuf bytes.Buffer - By(fmt.Sprintf("Applying manifest: %s", filepath)) + By(fmt.Sprintf("applying manifest: %s", filepath)) cmd := exec.Command("kubectl", "apply", "-f", filepath) cmd.Stdout = io.MultiWriter(os.Stdout, &stdoutBuf) cmd.Stderr = io.MultiWriter(os.Stderr, &stderrBuf) err := cmd.Run() Expect(err).NotTo(HaveOccurred()) } + +func ValidateTraffic(f *framework.Framework, serverDeploymentBuilder *manifest.DeploymentBuilder, succesRate float64, protocol string) { + trafficTester := GetTrafficTestConfig(f, protocol, serverDeploymentBuilder, 20, 20) + successRate, err := trafficTester.TestTraffic() + Expect(err).ToNot(HaveOccurred()) + Expect(successRate).Should(BeNumerically(">=", succesRate)) + +}