add test to verify host networking setup & cleanup (#1457)

test/framework/resources/aws/services/ec2.go

@@ -25,6 +25,7 @@ import (
 type EC2 interface {
 	DescribeInstanceType(instanceType string) ([]*ec2.InstanceTypeInfo, error)
 	DescribeInstance(instanceID string) (*ec2.Instance, error)
+	DescribeVPC(vpcID string) (*ec2.DescribeVpcsOutput, error)
 	DescribeNetworkInterface(interfaceIDs []string) (*ec2.DescribeNetworkInterfacesOutput, error)
 	AuthorizeSecurityGroupIngress(groupID string, protocol string, fromPort int, toPort int, cidrIP string) error
 	RevokeSecurityGroupIngress(groupID string, protocol string, fromPort int, toPort int, cidrIP string) error
@@ -268,6 +269,13 @@ func (d *defaultEC2) TerminateInstance(instanceIDs []string) error {
 	return err
 }
 
+func (d *defaultEC2) DescribeVPC(vpcID string) (*ec2.DescribeVpcsOutput, error) {
+	describeVPCInput := &ec2.DescribeVpcsInput{
+		VpcIds: aws.StringSlice([]string{vpcID}),
+	}
+	return d.EC2API.DescribeVpcs(describeVPCInput)
+}
+
 func NewEC2(session *session.Session) EC2 {
 	return &defaultEC2{
 		EC2API: ec2.New(session),

test/framework/resources/k8s/manifest/container.go

@@ -14,6 +14,8 @@
 package manifest
 
 import (
+	"github.com/aws/amazon-vpc-cni-k8s/test/framework/utils"
+
 	v1 "k8s.io/api/core/v1"
 )
 
@@ -43,6 +45,15 @@ func NewCurlContainer() *Container {
 	}
 }
 
+// See test/agent/README.md in this repository for more details
+func NewTestHelperContainer() *Container {
+	return &Container{
+		name:            "test-helper",
+		image:           utils.TestAgentImage,
+		imagePullPolicy: v1.PullIfNotPresent,
+	}
+}
+
 func NewNetCatAlpineContainer() *Container {
 	return &Container{
 		name:            "net-cat",

test/framework/resources/k8s/manifest/pod.go

@@ -0,0 +1,100 @@
+// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//     http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package manifest
+
+import (
+	"github.com/aws/amazon-vpc-cni-k8s/test/framework/utils"
+
+	"github.com/aws/aws-sdk-go/aws"
+	v1 "k8s.io/api/core/v1"
+	metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+type PodBuilder struct {
+	name                   string
+	namespace              string
+	hostNetwork            bool
+	container              v1.Container
+	labels                 map[string]string
+	terminationGracePeriod int
+	nodeName               string
+	restartPolicy          v1.RestartPolicy
+}
+
+func NewDefaultPodBuilder() *PodBuilder {
+	return &PodBuilder{
+		name:                   "test-pod",
+		namespace:              utils.DefaultTestNamespace,
+		labels:                 map[string]string{},
+		terminationGracePeriod: 0,
+		restartPolicy:          v1.RestartPolicyNever,
+	}
+}
+
+func (p *PodBuilder) Name(name string) *PodBuilder {
+	p.name = name
+	return p
+}
+
+func (p *PodBuilder) Namespace(namespace string) *PodBuilder {
+	p.namespace = namespace
+	return p
+}
+
+func (p *PodBuilder) HostNetwork(hostNetwork bool) *PodBuilder {
+	p.hostNetwork = hostNetwork
+	return p
+}
+
+func (p *PodBuilder) Container(container v1.Container) *PodBuilder {
+	p.container = container
+	return p
+}
+
+func (p *PodBuilder) PodLabel(labelKey string, labelVal string) *PodBuilder {
+	p.labels[labelKey] = labelVal
+	return p
+}
+
+func (p *PodBuilder) NodeName(nodeName string) *PodBuilder {
+	p.nodeName = nodeName
+	return p
+}
+
+func (p *PodBuilder) TerminationGracePeriod(period int) *PodBuilder {
+	p.terminationGracePeriod = period
+	return p
+}
+
+func (p *PodBuilder) RestartPolicy(policy v1.RestartPolicy) *PodBuilder {
+	p.restartPolicy = policy
+	return p
+}
+
+func (p *PodBuilder) Build() *v1.Pod {
+	return &v1.Pod{
+		ObjectMeta: metaV1.ObjectMeta{
+			Name:      p.name,
+			Namespace: p.namespace,
+			Labels:    p.labels,
+		},
+		Spec: v1.PodSpec{
+			Containers:                    []v1.Container{p.container},
+			RestartPolicy:                 p.restartPolicy,
+			TerminationGracePeriodSeconds: aws.Int64(int64(p.terminationGracePeriod)),
+			NodeName:                      p.nodeName,
+			HostNetwork:                   p.hostNetwork,
+		},
+	}
+}

test/framework/resources/k8s/resources/pod.go

@@ -16,7 +16,13 @@ package resources
 import (
 	"bytes"
 	"context"
+	"fmt"
 	"net/http"
+	"time"
+
+	"github.com/aws/amazon-vpc-cni-k8s/test/framework/utils"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/util/wait"
 
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -30,7 +36,11 @@ import (
 
 type PodManager interface {
 	PodExec(namespace string, name string, command []string) (string, string, error)
+	PodLogs(namespace string, name string) (string, error)
 	GetPodsWithLabelSelector(labelKey string, labelVal string) (v1.PodList, error)
+	CreatAndWaitTillRunning(pod *v1.Pod) (*v1.Pod, error)
+	CreateAndWaitTillPodCompleted(pod *v1.Pod) (*v1.Pod, error)
+	DeleteAndWaitTillPodDeleted(pod *v1.Pod) error
 }
 
 type defaultPodManager struct {
@@ -49,21 +59,97 @@ func NewDefaultPodManager(k8sClient client.DelegatingClient, k8sSchema *runtime.
 	}
 }
 
-func (d *defaultPodManager) PodExec(namespace string, name string, command []string) (string, string, error) {
-	pod := &v1.Pod{}
-	err := d.k8sClient.Get(context.Background(), types.NamespacedName{
-		Namespace: namespace,
-		Name:      name,
-	}, pod)
+func (d *defaultPodManager) CreatAndWaitTillRunning(pod *v1.Pod) (*v1.Pod, error) {
+	err := d.k8sClient.Create(context.Background(), pod)
 	if err != nil {
-		return "", "", err
+		return nil, fmt.Errorf("faield to create pod: %v", err)
 	}
+	// Allow the cache to sync
+	time.Sleep(utils.PollIntervalShort)
 
-	gkv, err := apiutil.GVKForObject(pod, d.k8sSchema)
+	observedPod := &v1.Pod{}
+	err = wait.PollImmediate(utils.PollIntervalShort, time.Second*120, func() (done bool, err error) {
+		err = d.k8sClient.Get(context.Background(), utils.NamespacedName(pod), observedPod)
+		if err != nil {
+			return true, err
+		}
+		if observedPod.Status.Phase == v1.PodRunning {
+			return true, nil
+		}
+		return false, nil
+	})
+
+	return observedPod, err
+}
+
+func (d *defaultPodManager) CreateAndWaitTillPodCompleted(pod *v1.Pod) (*v1.Pod, error) {
+	err := d.k8sClient.Create(context.Background(), pod)
 	if err != nil {
-		return "", "", err
+		return nil, fmt.Errorf("faield to create pod: %v", err)
+	}
+	// Allow the cache to sync
+	time.Sleep(utils.PollIntervalShort)
+
+	observedPod := &v1.Pod{}
+	err = wait.PollImmediate(utils.PollIntervalShort, time.Second*120, func() (done bool, err error) {
+		err = d.k8sClient.Get(context.Background(), utils.NamespacedName(pod), observedPod)
+		if err != nil {
+			return true, err
+		}
+		if observedPod.Status.Phase == v1.PodSucceeded {
+			return true, nil
+		} else if observedPod.Status.Phase == v1.PodFailed {
+			return false, fmt.Errorf("pod failed to run")
+		}
+		return false, nil
+	})
+
+	return observedPod, err
+}
+
+func (d *defaultPodManager) DeleteAndWaitTillPodDeleted(pod *v1.Pod) error {
+	err := d.k8sClient.Delete(context.Background(), pod)
+	if err != nil {
+		return err
+	}
+	observedPod := &v1.Pod{}
+	return wait.PollImmediate(utils.PollIntervalShort, time.Second*120, func() (done bool, err error) {
+		err = d.k8sClient.Get(context.Background(), utils.NamespacedName(pod), observedPod)
+		if err != nil {
+			if errors.IsNotFound(err) {
+				return true, nil
+			}
+			return true, err
+		}
+		return false, nil
+	})
+}
+
+func (d *defaultPodManager) PodLogs(namespace string, name string) (string, error) {
+	restClient, err := d.getRestClientForPod(namespace, name)
+	if err != nil {
+		return "", err
+	}
+
+	req := restClient.Get().
+		Resource("pods").
+		Namespace(namespace).
+		Name(name).
+		SubResource("log")
+
+	readCloser, err := req.Stream(context.Background())
+	if err != nil {
+		return "", err
 	}
-	restClient, err := apiutil.RESTClientForGVK(gkv, d.config, serializer.NewCodecFactory(d.k8sSchema))
+	defer readCloser.Close()
+	buf := new(bytes.Buffer)
+	buf.ReadFrom(readCloser)
+
+	return string(buf.Bytes()), err
+}
+
+func (d *defaultPodManager) PodExec(namespace string, name string, command []string) (string, string, error) {
+	restClient, err := d.getRestClientForPod(namespace, name)
 	if err != nil {
 		return "", "", err
 	}
@@ -74,10 +160,11 @@ func (d *defaultPodManager) PodExec(namespace string, name string, command []str
 		Command: command,
 	}
 
+	restClient.Get()
 	req := restClient.Post().
 		Resource("pods").
-		Name(pod.Name).
-		Namespace(pod.Namespace).
+		Name(name).
+		Namespace(namespace).
 		SubResource("exec").
 		VersionedParams(execOptions, runtime.NewParameterCodec(d.k8sSchema))
 
@@ -103,3 +190,20 @@ func (d *defaultPodManager) GetPodsWithLabelSelector(labelKey string, labelVal s
 	})
 	return podList, err
 }
+
+func (d *defaultPodManager) getRestClientForPod(namespace string, name string) (rest.Interface, error) {
+	pod := &v1.Pod{}
+	err := d.k8sClient.Get(context.Background(), types.NamespacedName{
+		Namespace: namespace,
+		Name:      name,
+	}, pod)
+	if err != nil {
+		return nil, err
+	}
+
+	gkv, err := apiutil.GVKForObject(pod, d.k8sSchema)
+	if err != nil {
+		return nil, err
+	}
+	return apiutil.RESTClientForGVK(gkv, d.config, serializer.NewCodecFactory(d.k8sSchema))
+}

test/framework/utils/const.go

@@ -19,6 +19,9 @@ const (
 	DefaultTestNamespace = "cni-automation"
 	AwsNodeNamespace     = "kube-system"
 	AwsNodeName          = "aws-node"
+
+	// See https://gallery.ecr.aws/r3i6j7b0/aws-vpc-cni-test-helper
+	TestAgentImage = "public.ecr.aws/r3i6j7b0/aws-vpc-cni-test-helper:07ac7548"
 )
 
 const (

test/go.mod

@@ -3,8 +3,9 @@ module github.com/aws/amazon-vpc-cni-k8s/test
 go 1.14
 
 require (
-	github.com/aws/amazon-vpc-cni-k8s v1.7.10
 	github.com/apparentlymart/go-cidr v1.0.1
+	github.com/aws/amazon-vpc-cni-k8s v1.7.10
+	github.com/aws/amazon-vpc-cni-k8s/test/agent v0.0.0-20210504235351-07ac754880d8
 	github.com/aws/aws-sdk-go v1.37.23
 	github.com/google/gopacket v1.1.19 // indirect
 	github.com/gophercloud/gophercloud v0.1.0 // indirect