diff --git a/cmd/aws-k8s-agent/main.go b/cmd/aws-k8s-agent/main.go index e0da594491..c6b731d543 100644 --- a/cmd/aws-k8s-agent/main.go +++ b/cmd/aws-k8s-agent/main.go @@ -45,15 +45,12 @@ func _main() int { return 1 } - discoverController := k8sapi.NewController(kubeClient) - go discoverController.DiscoverLocalK8SPods() - eniConfigController := eniconfig.NewENIConfigController() if ipamd.UseCustomNetworkCfg() { go eniConfigController.Start() } - ipamContext, err := ipamd.New(discoverController, eniConfigController) + ipamContext, err := ipamd.New(kubeClient, eniConfigController) if err != nil { log.Errorf("Initialization failure: %v", err) diff --git a/cmd/routed-eni-cni-plugin/cni.go b/cmd/routed-eni-cni-plugin/cni.go index 38152d75db..97a7933dd4 100644 --- a/cmd/routed-eni-cni-plugin/cni.go +++ b/cmd/routed-eni-cni-plugin/cni.go @@ -47,14 +47,7 @@ var version string // NetConf stores the common network config for the CNI plugin type NetConf struct { - // CNIVersion is the plugin version - CNIVersion string `json:"cniVersion,omitempty"` - - // Name is the plugin name - Name string `json:"name"` - - // Type is the plugin type - Type string `json:"type"` + types.NetConf // VethPrefix is the prefix to use when constructing the host-side // veth device name. It should be no more than four characters, and @@ -91,8 +84,8 @@ func init() { // LoadNetConf converts inputs (i.e. stdin) to NetConf func LoadNetConf(bytes []byte) (*NetConf, logger.Logger, error) { - conf := &NetConf{} - if err := json.Unmarshal(bytes, conf); err != nil { + var conf NetConf + if err := json.Unmarshal(bytes, &conf); err != nil { return nil, nil, errors.Wrap(err, "add cmd: error loading config from args") } @@ -118,7 +111,7 @@ func LoadNetConf(bytes []byte) (*NetConf, logger.Logger, error) { return nil, nil, errors.New("conf.VethPrefix can be at most 4 characters long") } - return conf, log, nil + return &conf, log, nil } func cmdAdd(args *skel.CmdArgs) error { @@ -136,7 +129,7 @@ func add(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap log.Infof("Received CNI add request: ContainerID(%s) Netns(%s) IfName(%s) Args(%s) Path(%s) argsStdinData(%s)", args.ContainerID, args.Netns, args.IfName, args.Args, args.Path, args.StdinData) - k8sArgs := K8sArgs{} + var k8sArgs K8sArgs if err := cniTypes.LoadArgs(args.Args, &k8sArgs); err != nil { log.Errorf("Failed to load k8s config from arg: %v", err) return errors.Wrap(err, "add cmd: failed to load k8s config from arg") @@ -148,11 +141,8 @@ func add(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap // Set up a connection to the ipamD server. conn, err := grpcClient.Dial(ipamdAddress, grpc.WithInsecure()) if err != nil { - log.Errorf("Failed to connect to backend server for pod %s namespace %s sandbox %s: %v", - string(k8sArgs.K8S_POD_NAME), - string(k8sArgs.K8S_POD_NAMESPACE), - string(k8sArgs.K8S_POD_INFRA_CONTAINER_ID), - err) + log.Errorf("Failed to connect to backend server for container %s: %v", + args.ContainerID, err) return errors.Wrap(err, "add cmd: failed to connect to backend server") } defer conn.Close() @@ -161,31 +151,30 @@ func add(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap r, err := c.AddNetwork(context.Background(), &pb.AddNetworkRequest{ - Netns: args.Netns, K8S_POD_NAME: string(k8sArgs.K8S_POD_NAME), K8S_POD_NAMESPACE: string(k8sArgs.K8S_POD_NAMESPACE), K8S_POD_INFRA_CONTAINER_ID: string(k8sArgs.K8S_POD_INFRA_CONTAINER_ID), - IfName: args.IfName}) + Netns: args.Netns, + ContainerID: args.ContainerID, + NetworkName: conf.Name, + IfName: args.IfName, + }) if err != nil { - log.Errorf("Error received from AddNetwork grpc call for pod %s namespace %s sandbox %s: %v", - string(k8sArgs.K8S_POD_NAME), - string(k8sArgs.K8S_POD_NAMESPACE), - string(k8sArgs.K8S_POD_INFRA_CONTAINER_ID), + log.Errorf("Error received from AddNetwork grpc call for containerID %s: %v", + args.ContainerID, err) return errors.Wrap(err, "add cmd: Error received from AddNetwork gRPC call") } if !r.Success { - log.Errorf("Failed to assign an IP address to pod %s, namespace %s sandbox %s", - string(k8sArgs.K8S_POD_NAME), - string(k8sArgs.K8S_POD_NAMESPACE), - string(k8sArgs.K8S_POD_INFRA_CONTAINER_ID)) + log.Errorf("Failed to assign an IP address to container %s", + args.ContainerID) return errors.New("add cmd: failed to assign an IP address to container") } - log.Infof("Received add network response for pod %s namespace %s sandbox %s: %s, table %d, external-SNAT: %v, vpcCIDR: %v", - string(k8sArgs.K8S_POD_NAME), string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_INFRA_CONTAINER_ID), + log.Infof("Received add network response for container %s interface %s: %s, table %d, external-SNAT: %v, vpcCIDR: %v", + args.ContainerID, args.IfName, r.IPv4Addr, r.DeviceNumber, r.UseExternalSNAT, r.VPCcidrs) addr := &net.IPNet{ @@ -195,30 +184,33 @@ func add(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap // build hostVethName // Note: the maximum length for linux interface name is 15 - hostVethName := generateHostVethName(conf.VethPrefix, string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME)) + hostVethName := generateHostVethName(conf.VethPrefix, conf.Name, args.ContainerID, args.IfName) err = driverClient.SetupNS(hostVethName, args.IfName, args.Netns, addr, int(r.DeviceNumber), r.VPCcidrs, r.UseExternalSNAT, mtu, log) if err != nil { - log.Errorf("Failed SetupPodNetwork for pod %s namespace %s sandbox %s: %v", - string(k8sArgs.K8S_POD_NAME), string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_INFRA_CONTAINER_ID), err) + log.Errorf("Failed SetupPodNetwork for container %s: %v", + args.ContainerID, err) // return allocated IP back to IP pool - r, delErr := c.DelNetwork(context.Background(), - &pb.DelNetworkRequest{ - K8S_POD_NAME: string(k8sArgs.K8S_POD_NAME), - K8S_POD_NAMESPACE: string(k8sArgs.K8S_POD_NAMESPACE), - K8S_POD_INFRA_CONTAINER_ID: string(k8sArgs.K8S_POD_INFRA_CONTAINER_ID), - Reason: "SetupNSFailed"}) + r, delErr := c.DelNetwork(context.Background(), &pb.DelNetworkRequest{ + K8S_POD_NAME: string(k8sArgs.K8S_POD_NAME), + K8S_POD_NAMESPACE: string(k8sArgs.K8S_POD_NAMESPACE), + K8S_POD_INFRA_CONTAINER_ID: string(k8sArgs.K8S_POD_INFRA_CONTAINER_ID), + ContainerID: args.ContainerID, + IfName: args.IfName, + NetworkName: conf.Name, + Reason: "SetupNSFailed", + }) if delErr != nil { - log.Errorf("Error received from DelNetwork grpc call for pod %s namespace %s sandbox %s: %v", - string(k8sArgs.K8S_POD_NAME), string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_INFRA_CONTAINER_ID), delErr) + log.Errorf("Error received from DelNetwork grpc call for container %s: %v", + args.ContainerID, delErr) } if !r.Success { - log.Errorf("Failed to release IP of pod %s namespace %s sandbox %s: %v", - string(k8sArgs.K8S_POD_NAME), string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_INFRA_CONTAINER_ID), delErr) + log.Errorf("Failed to release IP of container %s: %v", + args.ContainerID, delErr) } return errors.Wrap(err, "add command: failed to setup network") } @@ -238,9 +230,9 @@ func add(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap } // generateHostVethName returns a name to be used on the host-side veth device. -func generateHostVethName(prefix, namespace, podname string) string { +func generateHostVethName(prefix, netname, containerid, ifname string) string { h := sha1.New() - h.Write([]byte(fmt.Sprintf("%s.%s", namespace, podname))) + fmt.Fprintf(h, "%s.%s.%s", netname, containerid, ifname) return fmt.Sprintf("%s%s", prefix, hex.EncodeToString(h.Sum(nil))[:11]) } @@ -251,7 +243,7 @@ func cmdDel(args *skel.CmdArgs) error { func del(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrapper.GRPC, rpcClient rpcwrapper.RPC, driverClient driver.NetworkAPIs) error { - _, log, err := LoadNetConf(args.StdinData) + conf, log, err := LoadNetConf(args.StdinData) if err != nil { return errors.Wrap(err, "add cmd: error loading config from args") } @@ -259,7 +251,7 @@ func del(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap log.Infof("Received CNI del request: ContainerID(%s) Netns(%s) IfName(%s) Args(%s) Path(%s) argsStdinData(%s)", args.ContainerID, args.Netns, args.IfName, args.Args, args.Path, args.StdinData) - k8sArgs := K8sArgs{} + var k8sArgs K8sArgs if err := cniTypes.LoadArgs(args.Args, &k8sArgs); err != nil { log.Errorf("Failed to load k8s config from args: %v", err) return errors.Wrap(err, "del cmd: failed to load k8s config from args") @@ -269,11 +261,8 @@ func del(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap // Set up a connection to the server. conn, err := grpcClient.Dial(ipamdAddress, grpc.WithInsecure()) if err != nil { - log.Errorf("Failed to connect to backend server for pod %s namespace %s sandbox %s: %v", - string(k8sArgs.K8S_POD_NAME), - string(k8sArgs.K8S_POD_NAMESPACE), - string(k8sArgs.K8S_POD_INFRA_CONTAINER_ID), - err) + log.Errorf("Failed to connect to backend server for container %s: %v", + args.ContainerID, err) return errors.Wrap(err, "del cmd: failed to connect to backend server") } @@ -281,30 +270,32 @@ func del(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap c := rpcClient.NewCNIBackendClient(conn) - r, err := c.DelNetwork(context.Background(), - &pb.DelNetworkRequest{ - K8S_POD_NAME: string(k8sArgs.K8S_POD_NAME), - K8S_POD_NAMESPACE: string(k8sArgs.K8S_POD_NAMESPACE), - K8S_POD_INFRA_CONTAINER_ID: string(k8sArgs.K8S_POD_INFRA_CONTAINER_ID), - Reason: "PodDeleted"}) + r, err := c.DelNetwork(context.Background(), &pb.DelNetworkRequest{ + K8S_POD_NAME: string(k8sArgs.K8S_POD_NAME), + K8S_POD_NAMESPACE: string(k8sArgs.K8S_POD_NAMESPACE), + K8S_POD_INFRA_CONTAINER_ID: string(k8sArgs.K8S_POD_INFRA_CONTAINER_ID), + NetworkName: conf.Name, + ContainerID: args.ContainerID, + IfName: args.IfName, + Reason: "PodDeleted", + }) if err != nil { if strings.Contains(err.Error(), datastore.ErrUnknownPod.Error()) { // Plugins should generally complete a DEL action without error even if some resources are missing. For example, // an IPAM plugin should generally release an IP allocation and return success even if the container network // namespace no longer exists, unless that network namespace is critical for IPAM management - log.Infof("Pod %s in namespace %s not found", string(k8sArgs.K8S_POD_NAME), string(k8sArgs.K8S_POD_NAMESPACE)) + log.Infof("Container %s not found", args.ContainerID) return nil - } else { - log.Errorf("Error received from DelNetwork gRPC call for pod %s namespace %s sandbox %s: %v", - string(k8sArgs.K8S_POD_NAME), string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_INFRA_CONTAINER_ID), err) - return errors.Wrap(err, "del cmd: error received from DelNetwork gRPC call") } + log.Errorf("Error received from DelNetwork gRPC call for container %s: %v", + args.ContainerID, err) + return errors.Wrap(err, "del cmd: error received from DelNetwork gRPC call") } if !r.Success { - log.Errorf("Failed to process delete request for pod %s namespace %s sandbox %s: Success == false", - string(k8sArgs.K8S_POD_NAME), string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_INFRA_CONTAINER_ID)) + log.Errorf("Failed to process delete request for container %s: Success == false", + args.ContainerID) return errors.New("del cmd: failed to process delete request") } @@ -316,13 +307,12 @@ func del(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap } err = driverClient.TeardownNS(addr, int(r.DeviceNumber), log) if err != nil { - log.Errorf("Failed on TeardownPodNetwork for pod %s namespace %s sandbox %s: %v", - string(k8sArgs.K8S_POD_NAME), string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_INFRA_CONTAINER_ID), err) + log.Errorf("Failed on TeardownPodNetwork for container ID %s: %v", + args.ContainerID, err) return errors.Wrap(err, "del cmd: failed on tear down pod network") } } else { - log.Warnf("Pod %s in namespace %s did not have a valid IP %s", string(k8sArgs.K8S_POD_NAME), - string(k8sArgs.K8S_POD_NAMESPACE), r.IPv4Addr) + log.Warnf("Container %s did not have a valid IP %s", args.ContainerID, r.IPv4Addr) } return nil } diff --git a/cmd/routed-eni-cni-plugin/cni_test.go b/cmd/routed-eni-cni-plugin/cni_test.go index c9f6da7773..71962fbcdb 100644 --- a/cmd/routed-eni-cni-plugin/cni_test.go +++ b/cmd/routed-eni-cni-plugin/cni_test.go @@ -20,6 +20,7 @@ import ( "testing" "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/types" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "google.golang.org/grpc" @@ -45,9 +46,12 @@ const ( devNum = 4 ) -var netConf = &NetConf{CNIVersion: cniVersion, - Name: cniName, - Type: cniType, +var netConf = &NetConf{ + NetConf: types.NetConf{ + CNIVersion: cniVersion, + Name: cniName, + Type: cniType, + }, PluginLogLevel: pluginLogLevel, PluginLogFile: pluginLogFile, } diff --git a/config/master/aws-k8s-cni-cn.yaml b/config/master/aws-k8s-cni-cn.yaml index 3042b23610..ec2c9e14ef 100644 --- a/config/master/aws-k8s-cni-cn.yaml +++ b/config/master/aws-k8s-cni-cn.yaml @@ -20,24 +20,9 @@ - "apiGroups": - "crd.k8s.amazonaws.com" "resources": - - "*" + - "eniconfigs" "verbs": - - "*" -- "apiGroups": - - "" - "resources": - - "pods" - - "nodes" - - "namespaces" - "verbs": - - "list" - - "watch" - "get" -- "apiGroups": - - "extensions" - "resources": - - "daemonsets" - "verbs": - "list" - "watch" --- @@ -151,6 +136,8 @@ "name": "cni-net-dir" - "mountPath": "/host/var/log/aws-routed-eni" "name": "log-dir" + - "mountPath": "/var/run/aws-node" + "name": "run-dir" - "mountPath": "/var/run/docker.sock" "name": "dockersock" - "mountPath": "/var/run/dockershim.sock" @@ -176,16 +163,20 @@ - "hostPath": "path": "/etc/cni/net.d" "name": "cni-net-dir" - - "hostPath": - "path": "/var/log/aws-routed-eni" - "type": "DirectoryOrCreate" - "name": "log-dir" - "hostPath": "path": "/var/run/docker.sock" "name": "dockersock" - "hostPath": "path": "/var/run/dockershim.sock" "name": "dockershim" + - "hostPath": + "path": "/var/log/aws-routed-eni" + "type": "DirectoryOrCreate" + "name": "log-dir" + - "hostPath": + "path": "/var/run/aws-node" + "type": "DirectoryOrCreate" + "name": "run-dir" "updateStrategy": "rollingUpdate": "maxUnavailable": "10%" diff --git a/config/master/aws-k8s-cni-us-gov-east-1.yaml b/config/master/aws-k8s-cni-us-gov-east-1.yaml index 8b947661c9..26386d68f5 100644 --- a/config/master/aws-k8s-cni-us-gov-east-1.yaml +++ b/config/master/aws-k8s-cni-us-gov-east-1.yaml @@ -20,24 +20,9 @@ - "apiGroups": - "crd.k8s.amazonaws.com" "resources": - - "*" + - "eniconfigs" "verbs": - - "*" -- "apiGroups": - - "" - "resources": - - "pods" - - "nodes" - - "namespaces" - "verbs": - - "list" - - "watch" - "get" -- "apiGroups": - - "extensions" - "resources": - - "daemonsets" - "verbs": - "list" - "watch" --- @@ -151,6 +136,8 @@ "name": "cni-net-dir" - "mountPath": "/host/var/log/aws-routed-eni" "name": "log-dir" + - "mountPath": "/var/run/aws-node" + "name": "run-dir" - "mountPath": "/var/run/docker.sock" "name": "dockersock" - "mountPath": "/var/run/dockershim.sock" @@ -176,16 +163,20 @@ - "hostPath": "path": "/etc/cni/net.d" "name": "cni-net-dir" - - "hostPath": - "path": "/var/log/aws-routed-eni" - "type": "DirectoryOrCreate" - "name": "log-dir" - "hostPath": "path": "/var/run/docker.sock" "name": "dockersock" - "hostPath": "path": "/var/run/dockershim.sock" "name": "dockershim" + - "hostPath": + "path": "/var/log/aws-routed-eni" + "type": "DirectoryOrCreate" + "name": "log-dir" + - "hostPath": + "path": "/var/run/aws-node" + "type": "DirectoryOrCreate" + "name": "run-dir" "updateStrategy": "rollingUpdate": "maxUnavailable": "10%" diff --git a/config/master/aws-k8s-cni-us-gov-west-1.yaml b/config/master/aws-k8s-cni-us-gov-west-1.yaml index 93b81c704b..9a8cd99c59 100644 --- a/config/master/aws-k8s-cni-us-gov-west-1.yaml +++ b/config/master/aws-k8s-cni-us-gov-west-1.yaml @@ -20,24 +20,9 @@ - "apiGroups": - "crd.k8s.amazonaws.com" "resources": - - "*" + - "eniconfigs" "verbs": - - "*" -- "apiGroups": - - "" - "resources": - - "pods" - - "nodes" - - "namespaces" - "verbs": - - "list" - - "watch" - "get" -- "apiGroups": - - "extensions" - "resources": - - "daemonsets" - "verbs": - "list" - "watch" --- @@ -151,6 +136,8 @@ "name": "cni-net-dir" - "mountPath": "/host/var/log/aws-routed-eni" "name": "log-dir" + - "mountPath": "/var/run/aws-node" + "name": "run-dir" - "mountPath": "/var/run/docker.sock" "name": "dockersock" - "mountPath": "/var/run/dockershim.sock" @@ -176,16 +163,20 @@ - "hostPath": "path": "/etc/cni/net.d" "name": "cni-net-dir" - - "hostPath": - "path": "/var/log/aws-routed-eni" - "type": "DirectoryOrCreate" - "name": "log-dir" - "hostPath": "path": "/var/run/docker.sock" "name": "dockersock" - "hostPath": "path": "/var/run/dockershim.sock" "name": "dockershim" + - "hostPath": + "path": "/var/log/aws-routed-eni" + "type": "DirectoryOrCreate" + "name": "log-dir" + - "hostPath": + "path": "/var/run/aws-node" + "type": "DirectoryOrCreate" + "name": "run-dir" "updateStrategy": "rollingUpdate": "maxUnavailable": "10%" diff --git a/config/master/aws-k8s-cni.yaml b/config/master/aws-k8s-cni.yaml index 53ac36e7c4..a4af8724b9 100644 --- a/config/master/aws-k8s-cni.yaml +++ b/config/master/aws-k8s-cni.yaml @@ -20,24 +20,9 @@ - "apiGroups": - "crd.k8s.amazonaws.com" "resources": - - "*" + - "eniconfigs" "verbs": - - "*" -- "apiGroups": - - "" - "resources": - - "pods" - - "nodes" - - "namespaces" - "verbs": - - "list" - - "watch" - "get" -- "apiGroups": - - "extensions" - "resources": - - "daemonsets" - "verbs": - "list" - "watch" --- @@ -151,6 +136,8 @@ "name": "cni-net-dir" - "mountPath": "/host/var/log/aws-routed-eni" "name": "log-dir" + - "mountPath": "/var/run/aws-node" + "name": "run-dir" - "mountPath": "/var/run/docker.sock" "name": "dockersock" - "mountPath": "/var/run/dockershim.sock" @@ -176,16 +163,20 @@ - "hostPath": "path": "/etc/cni/net.d" "name": "cni-net-dir" - - "hostPath": - "path": "/var/log/aws-routed-eni" - "type": "DirectoryOrCreate" - "name": "log-dir" - "hostPath": "path": "/var/run/docker.sock" "name": "dockersock" - "hostPath": "path": "/var/run/dockershim.sock" "name": "dockershim" + - "hostPath": + "path": "/var/log/aws-routed-eni" + "type": "DirectoryOrCreate" + "name": "log-dir" + - "hostPath": + "path": "/var/run/aws-node" + "type": "DirectoryOrCreate" + "name": "run-dir" "updateStrategy": "rollingUpdate": "maxUnavailable": "10%" diff --git a/config/master/manifests.jsonnet b/config/master/manifests.jsonnet index c7024ea3bd..9a24843ffc 100644 --- a/config/master/manifests.jsonnet +++ b/config/master/manifests.jsonnet @@ -35,18 +35,8 @@ local awsnode = { rules: [ { apiGroups: ["crd.k8s.amazonaws.com"], - resources: ["*"], - verbs: ["*"], - }, - { - apiGroups: [""], - resources: ["pods", "nodes", "namespaces"], - verbs: ["list", "watch", "get"], - }, - { - apiGroups: ["extensions"], - resources: ["daemonsets"], - verbs: ["list", "watch"], + resources: ["eniconfigs"], + verbs: ["get", "list", "watch"], }, ], }, @@ -177,6 +167,7 @@ local awsnode = { {mountPath: "/host/opt/cni/bin", name: "cni-bin-dir"}, {mountPath: "/host/etc/cni/net.d", name: "cni-net-dir"}, {mountPath: "/host/var/log/aws-routed-eni", name: "log-dir"}, + {mountPath: "/var/run/aws-node", name: "run-dir"}, {mountPath: "/var/run/docker.sock", name: "dockersock"}, {mountPath: "/var/run/dockershim.sock", name: "dockershim"}, ], @@ -186,14 +177,20 @@ local awsnode = { volumes: [ {name: "cni-bin-dir", hostPath: {path: "/opt/cni/bin"}}, {name: "cni-net-dir", hostPath: {path: "/etc/cni/net.d"}}, + {name: "dockersock", hostPath: {path: "/var/run/docker.sock"}}, + {name: "dockershim", hostPath: {path: "/var/run/dockershim.sock"}}, {name: "log-dir", hostPath: { path: "/var/log/aws-routed-eni", type: "DirectoryOrCreate", }, }, - {name: "dockersock", hostPath: {path: "/var/run/docker.sock"}}, - {name: "dockershim", hostPath: {path: "/var/run/dockershim.sock"}}, + {name: "run-dir", + hostPath: { + path: "/var/run/aws-node", + type: "DirectoryOrCreate", + }, + }, ], initContainers: [ { diff --git a/go.mod b/go.mod index 0f3273612c..0d27c1313c 100644 --- a/go.mod +++ b/go.mod @@ -41,6 +41,7 @@ require ( golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 // indirect google.golang.org/grpc v1.29.0 + gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 gopkg.in/yaml.v2 v2.2.7 // indirect diff --git a/go.sum b/go.sum index 806edaf1e5..30825c83ae 100644 --- a/go.sum +++ b/go.sum @@ -144,7 +144,6 @@ github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8 h1:2c1EFnZHIPCW8qKWgHMH/fX2PkSabFc5mrVzfUNdg5U= github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= -github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k= diff --git a/pkg/cri/cri.go b/pkg/cri/cri.go index b50e7b05c7..38978fa8ca 100644 --- a/pkg/cri/cri.go +++ b/pkg/cri/cri.go @@ -15,7 +15,6 @@ package cri import ( "context" - "errors" "os" "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" @@ -30,14 +29,12 @@ const ( // SandboxInfo provides container information type SandboxInfo struct { - ID string - Namespace string - Name string - K8SUID string + ID string + IP string } type APIs interface { - GetRunningPodSandboxes(log logger.Logger) (map[string]*SandboxInfo, error) + GetRunningPodSandboxes(log logger.Logger) ([]*SandboxInfo, error) } type Client struct{} @@ -47,7 +44,9 @@ func New() *Client { } //GetRunningPodSandboxes get running sandboxIDs -func (c *Client) GetRunningPodSandboxes(log logger.Logger) (map[string]*SandboxInfo, error) { +func (c *Client) GetRunningPodSandboxes(log logger.Logger) ([]*SandboxInfo, error) { + ctx := context.TODO() + socketPath := dockerSocketPath if info, err := os.Stat("/var/run/cri.sock"); err == nil && !info.IsDir() { socketPath = criSocketPath @@ -62,37 +61,48 @@ func (c *Client) GetRunningPodSandboxes(log logger.Logger) (map[string]*SandboxI client := runtimeapi.NewRuntimeServiceClient(conn) // List all ready sandboxes from the CRI - filter := &runtimeapi.PodSandboxFilter{ - State: &runtimeapi.PodSandboxStateValue{ - State: runtimeapi.PodSandboxState_SANDBOX_READY, + sandboxes, err := client.ListPodSandbox(ctx, &runtimeapi.ListPodSandboxRequest{ + Filter: &runtimeapi.PodSandboxFilter{ + State: &runtimeapi.PodSandboxStateValue{ + State: runtimeapi.PodSandboxState_SANDBOX_READY, + }, }, - } - sandboxes, err := client.ListPodSandbox(context.Background(), &runtimeapi.ListPodSandboxRequest{Filter: filter}) + }) if err != nil { return nil, err } - sandboxInfos := make(map[string]*SandboxInfo) - for _, sandbox := range sandboxes.Items { - if sandbox.Metadata == nil { + sandboxInfos := make([]*SandboxInfo, 0, len(sandboxes.GetItems())) + for _, sandbox := range sandboxes.GetItems() { + status, err := client.PodSandboxStatus(ctx, &runtimeapi.PodSandboxStatusRequest{ + PodSandboxId: sandbox.GetId(), + }) + if err != nil { + return nil, err + } + + if state := status.GetStatus().GetState(); state != runtimeapi.PodSandboxState_SANDBOX_READY { + log.Debugf("Ignoring sandbox %s in unready state %s", sandbox.Id, state) continue } - uid := sandbox.Metadata.Uid - - // Verify each pod only has one active sandbox. Kubelet will clean this - // up if it happens, so we should abort and wait until it does. - if other, ok := sandboxInfos[uid]; ok { - log.Errorf("GetRunningPodSandboxes: More than one sandbox with the same pod UID %s", uid) - log.Errorf(" Sandbox %s: namespace=%s name=%s", other.ID, other.Namespace, other.Name) - log.Errorf(" Sandbox %s: namespace=%s name=%s", sandbox.Id, sandbox.Metadata.Namespace, sandbox.Metadata.Name) - return nil, errors.New("UID conflict in container runtime") + + if netmode := status.GetStatus().GetLinux().GetNamespaces().GetOptions().GetNetwork(); netmode != runtimeapi.NamespaceMode_POD { + log.Debugf("Ignoring sandbox %s with non-pod netns mode %s", sandbox.Id, netmode) + continue + } + + ips := []string{status.GetStatus().GetNetwork().GetIp()} + for _, ip := range status.GetStatus().GetNetwork().GetAdditionalIps() { + ips = append(ips, ip.GetIp()) } - sandboxInfos[uid] = &SandboxInfo{ - ID: sandbox.Id, - Namespace: sandbox.Metadata.Namespace, - Name: sandbox.Metadata.Name, - K8SUID: uid} + for _, ip := range ips { + info := SandboxInfo{ + ID: sandbox.GetId(), + IP: ip, + } + sandboxInfos = append(sandboxInfos, &info) + } } return sandboxInfos, nil } diff --git a/pkg/cri/mocks/cri_mocks.go b/pkg/cri/mocks/cri_mocks.go index 3154f6d66a..1b40fab151 100644 --- a/pkg/cri/mocks/cri_mocks.go +++ b/pkg/cri/mocks/cri_mocks.go @@ -50,10 +50,10 @@ func (m *MockAPIs) EXPECT() *MockAPIsMockRecorder { } // GetRunningPodSandboxes mocks base method -func (m *MockAPIs) GetRunningPodSandboxes(arg0 logger.Logger) (map[string]*cri.SandboxInfo, error) { +func (m *MockAPIs) GetRunningPodSandboxes(arg0 logger.Logger) ([]*cri.SandboxInfo, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetRunningPodSandboxes", arg0) - ret0, _ := ret[0].(map[string]*cri.SandboxInfo) + ret0, _ := ret[0].([]*cri.SandboxInfo) ret1, _ := ret[1].(error) return ret0, ret1 } diff --git a/pkg/ipamd/datastore/checkpoint.go b/pkg/ipamd/datastore/checkpoint.go new file mode 100644 index 0000000000..a65513908e --- /dev/null +++ b/pkg/ipamd/datastore/checkpoint.go @@ -0,0 +1,113 @@ +package datastore + +import ( + "encoding/json" + "io/ioutil" + "os" + "path/filepath" +) + +// Checkpointer can persist data and (hopefully) restore it later +type Checkpointer interface { + Checkpoint(data interface{}) error + Restore(into interface{}) error +} + +// NullCheckpoint discards data and always returns "not found". For testing only! +type NullCheckpoint struct{} + +// Checkpoint implements the Checkpointer interface in the most +// trivial sense, by just discarding data. +func (c NullCheckpoint) Checkpoint(data interface{}) error { + return nil +} + +// Restore implements the Checkpointer interface in the most trivial +// sense, by always returning "not found". +func (c NullCheckpoint) Restore(into interface{}) error { + return os.ErrNotExist +} + +// TestCheckpoint maintains a snapshot in memory. +type TestCheckpoint struct { + Error error + Data interface{} +} + +// NewTestCheckpoint creates a new TestCheckpoint. +func NewTestCheckpoint(data interface{}) *TestCheckpoint { + return &TestCheckpoint{Data: data} +} + +// Checkpoint implements the Checkpointer interface. +func (c *TestCheckpoint) Checkpoint(data interface{}) error { + if c.Error != nil { + return c.Error + } + c.Data = data + return nil +} + +// Restore implements the Checkpointer interface. +func (c *TestCheckpoint) Restore(into interface{}) error { + if c.Error != nil { + return c.Error + } + // `into` is always a pointer to interface{}, but we can't + // actually make the Restore() function *interface{}, because + // that doesn't match the (widely used) `encoding.Unmarshal` + // interface :( + // Round trip through json strings instead because copying is + // hard. + buf, err := json.Marshal(c.Data) + if err != nil { + return err + } + return json.Unmarshal(buf, into) +} + +// JSONFile is a checkpointer that writes to a JSON file +type JSONFile struct { + path string +} + +// NewJSONFile creates a new JsonFile +func NewJSONFile(path string) *JSONFile { + return &JSONFile{path: path} +} + +// Checkpoint implements the Checkpointer interface +func (c *JSONFile) Checkpoint(data interface{}) error { + f, err := ioutil.TempFile(filepath.Dir(c.path), filepath.Base(c.path)+".tmp*") + if err != nil { + return err + } + + if err := json.NewEncoder(f).Encode(&data); err != nil { + os.Remove(f.Name()) + return err + } + + if err := f.Sync(); err != nil { + os.Remove(f.Name()) + return err + } + + if err := os.Rename(f.Name(), c.path); err != nil { + os.Remove(f.Name()) + return err + } + + return nil +} + +// Restore implements the Checkpointer interface +func (c *JSONFile) Restore(into interface{}) error { + f, err := os.Open(c.path) + if err != nil { + return err + } + defer f.Close() + + return json.NewDecoder(f).Decode(into) +} diff --git a/pkg/ipamd/datastore/data_store.go b/pkg/ipamd/datastore/data_store.go index 4a2bda2941..53726c3487 100644 --- a/pkg/ipamd/datastore/data_store.go +++ b/pkg/ipamd/datastore/data_store.go @@ -14,11 +14,12 @@ package datastore import ( - "sort" + "fmt" + "os" "sync" "time" - "github.com/aws/amazon-vpc-cni-k8s/pkg/k8sapi" + "github.com/aws/amazon-vpc-cni-k8s/pkg/cri" "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" @@ -53,6 +54,13 @@ const ( UnknownENIError = "datastore: unknown ENI" ) +// Enable temporary checkpoint upgrade logic. If checkpoint file is +// not found, query CRI to learn current pods and currently-allocated +// IPs. Will be disabled/removed after next release. +const BackfillMissingCheckpointFromCRI = true +const BackfillNetworkName = "_migrated-from-cri" +const BackfillNetworkIface = "unknown" + // ErrUnknownPod is an error when there is no pod in data store matching pod name, namespace, sandbox id var ErrUnknownPod = errors.New("datastore: unknown pod") @@ -93,17 +101,34 @@ var ( prometheusRegistered = false ) -// ENIIPPool contains ENI/IP Pool information. Exported fields will be marshaled for introspection. -type ENIIPPool struct { - createTime time.Time - lastUnassignedTime time.Time +// IPAMKey is the IPAM primary key. Quoting CNI spec: +// Plugins that store state should do so using a primary key of +// (network name, CNI_CONTAINERID, CNI_IFNAME). +type IPAMKey struct { + NetworkName string `json:"networkName"` + ContainerID string `json:"containerID"` + IfName string `json:"ifName"` +} + +// IsZero returns true iff object is equal to the golang zero/null value. +func (k IPAMKey) IsZero() bool { + return k == IPAMKey{} +} + +// String() implements the fmt.Stringer interface. +func (k IPAMKey) String() string { + return fmt.Sprintf("%s/%s/%s", k.NetworkName, k.ContainerID, k.IfName) +} + +// ENI represents a single ENI. +type ENI struct { + // AWS ENI ID + ID string + createTime time.Time // IsPrimary indicates whether ENI is a primary ENI IsPrimary bool - ID string // DeviceNumber is the device number of ENI (0 means the primary ENI) DeviceNumber int - // AssignedIPv4Addresses is the number of IP addresses already been assigned - AssignedIPv4Addresses int // IPv4Addresses shows whether each address is assigned, the key is IP address, which must // be in dot-decimal notation with no leading zeros and no whitespace(eg: "10.1.0.253") IPv4Addresses map[string]*AddressInfo @@ -111,21 +136,67 @@ type ENIIPPool struct { // AddressInfo contains information about an IP, Exported fields will be marshaled for introspection. type AddressInfo struct { + IPAMKey IPAMKey Address string - Assigned bool // true if it is assigned to a pod UnassignedTime time.Time } -// PodKey is used to locate pod IP -type PodKey struct { - name string - namespace string - sandbox string +func (e *ENI) findAddressForSandbox(ipamKey IPAMKey) *AddressInfo { + for _, addr := range e.IPv4Addresses { + if addr.IPAMKey == ipamKey { + return addr + } + } + return nil +} + +// AssignedIPv4Addresses is the number of IP addresses already assigned +func (e *ENI) AssignedIPv4Addresses() int { + count := 0 + for _, addr := range e.IPv4Addresses { + if addr.Assigned() { + count++ + } + } + return count +} + +// Assigned returns true iff the address is allocated to a pod/sandbox. +func (addr AddressInfo) Assigned() bool { + return !addr.IPAMKey.IsZero() +} + +// InCoolingPeriod checks whether an addr is in addressCoolingPeriod +func (addr AddressInfo) inCoolingPeriod() bool { + return time.Since(addr.UnassignedTime) <= addressCoolingPeriod +} + +// ENIPool is a collection of ENI, keyed by ENI ID +type ENIPool map[string]*ENI + +// AssignedIPv4Addresses is the number of IP addresses already assigned +func (p *ENIPool) AssignedIPv4Addresses() int { + count := 0 + for _, eni := range *p { + count += eni.AssignedIPv4Addresses() + } + return count +} + +// FindAddressForSandbox returns ENI and AddressInfo or (nil, nil) if not found +func (p *ENIPool) FindAddressForSandbox(ipamKey IPAMKey) (*ENI, *AddressInfo) { + for _, eni := range *p { + if addr := eni.findAddressForSandbox(ipamKey); addr != nil { + return eni, addr + } + } + return nil, nil } // PodIPInfo contains pod's IP and the device number of the ENI type PodIPInfo struct { - // IP is the IP address of pod + IPAMKey IPAMKey + // IP is the IPv4 address of pod IP string // DeviceNumber is the device number of the ENI DeviceNumber int @@ -133,25 +204,23 @@ type PodIPInfo struct { // DataStore contains node level ENI/IP type DataStore struct { - total int - assigned int - eniIPPools map[string]*ENIIPPool - podsIP map[PodKey]PodIPInfo - lock sync.RWMutex - log logger.Logger + total int + assigned int + eniPool ENIPool + lock sync.Mutex + log logger.Logger + backingStore Checkpointer + cri cri.APIs } -// PodInfos contains pods IP information which uses key name_namespace_sandbox -type PodInfos map[string]PodIPInfo - // ENIInfos contains ENI IP information type ENIInfos struct { // TotalIPs is the total number of IP addresses TotalIPs int // assigned is the number of IP addresses that has been assigned AssignedIPs int - // ENIIPPools contains ENI IP pool information - ENIIPPools map[string]ENIIPPool + // ENIs contains ENI IP pool information + ENIs map[string]ENI } func prometheusRegister() { @@ -166,13 +235,128 @@ func prometheusRegister() { } // NewDataStore returns DataStore structure -func NewDataStore(log logger.Logger) *DataStore { +func NewDataStore(log logger.Logger, backingStore Checkpointer) *DataStore { prometheusRegister() return &DataStore{ - eniIPPools: make(map[string]*ENIIPPool), - podsIP: make(map[PodKey]PodIPInfo), - log: log, + eniPool: make(ENIPool), + log: log, + backingStore: backingStore, + cri: cri.New(), + } +} + +// CheckpointFormatVersion is the version stamp used on stored checkpoints. +const CheckpointFormatVersion = "vpc-cni-ipam/1" + +// CheckpointData is the format of stored checkpoints. Note this is +// deliberately a "dumb" format since efficiency is less important +// than version stability here. +type CheckpointData struct { + Version string `json:"version"` + Allocations []CheckpointEntry `json:"allocations"` +} + +// CheckpointEntry is a "row" in the conceptual IPAM datastore, as stored +// in checkpoints. +type CheckpointEntry struct { + IPAMKey + IPv4 string `json:"ipv4"` +} + +// ReadBackingStore initialises the IP allocation state from the +// configured backing store. Should be called before using data +// store. +func (ds *DataStore) ReadBackingStore() error { + ds.log.Infof("Reading ipam state from backing store") + + var data CheckpointData + err := ds.backingStore.Restore(&data) + ds.log.Debugf("backing store restore returned err %v", err) + if BackfillMissingCheckpointFromCRI && os.IsNotExist(err) { + ds.log.Infof("Backing store not found. Querying CRI.") + sandboxes, err := ds.cri.GetRunningPodSandboxes(ds.log) + if err != nil { + return err + } + + entries := make([]CheckpointEntry, 0, len(sandboxes)) + for _, s := range sandboxes { + entries = append(entries, CheckpointEntry{ + // NB: These Backfill values are also assumed in UnassignPodIPv4Address + IPAMKey: IPAMKey{ + NetworkName: BackfillNetworkName, + ContainerID: s.ID, + IfName: BackfillNetworkIface, + }, + IPv4: s.IP, + }) + } + data = CheckpointData{ + Version: CheckpointFormatVersion, + Allocations: entries, + } + + } else if os.IsNotExist(err) { + // Assume that no file == no containers are + // currently in use, eg a fresh reboot just + // cleared everything out. This is ok, and a + // no-op. + return nil + } else if err != nil { + return fmt.Errorf("datastore: error reading backing store: %v", err) + } + + if data.Version != CheckpointFormatVersion { + return fmt.Errorf("datastore: unknown backing store format (%s != %s) - wrong CNI/ipamd version? (Rebooting this node will restart local pods and probably help)", data.Version, CheckpointFormatVersion) + } + + ds.lock.Lock() + defer ds.lock.Unlock() + + eniIPs := make(ENIPool) + for _, eni := range ds.eniPool { + for _, addr := range eni.IPv4Addresses { + eniIPs[addr.Address] = eni + } + } + + for _, allocation := range data.Allocations { + eni := eniIPs[allocation.IPv4] + if eni == nil { + ds.log.Infof("datastore: Sandbox %s uses unknown IPv4 %s - presuming stale/dead", allocation.IPAMKey, allocation.IPv4) + continue + } + + addr := eni.IPv4Addresses[allocation.IPv4] + ds.assignPodIPv4AddressUnsafe(allocation.IPAMKey, eni, addr) + ds.log.Debugf("Recovered %s => %s/%s", allocation.IPAMKey, eni.ID, addr.Address) } + + ds.log.Debugf("Completed reading ipam state from backing store") + return nil +} + +func (ds *DataStore) writeBackingStoreUnsafe() error { + allocations := make([]CheckpointEntry, 0, ds.assigned) + + for _, eni := range ds.eniPool { + for _, addr := range eni.IPv4Addresses { + if addr.Assigned() { + entry := CheckpointEntry{ + IPAMKey: addr.IPAMKey, + IPv4: addr.Address, + } + allocations = append(allocations, entry) + } + } + } + + data := CheckpointData{ + Version: CheckpointFormatVersion, + Allocations: allocations, + } + + return ds.backingStore.Checkpoint(&data) } // AddENI add ENI to data store @@ -182,17 +366,17 @@ func (ds *DataStore) AddENI(eniID string, deviceNumber int, isPrimary bool) erro ds.log.Debugf("DataStore Add an ENI %s", eniID) - _, ok := ds.eniIPPools[eniID] + _, ok := ds.eniPool[eniID] if ok { return errors.New(DuplicatedENIError) } - ds.eniIPPools[eniID] = &ENIIPPool{ + ds.eniPool[eniID] = &ENI{ createTime: time.Now(), IsPrimary: isPrimary, ID: eniID, DeviceNumber: deviceNumber, IPv4Addresses: make(map[string]*AddressInfo)} - enis.Set(float64(len(ds.eniIPPools))) + enis.Set(float64(len(ds.eniPool))) return nil } @@ -201,7 +385,7 @@ func (ds *DataStore) AddIPv4AddressToStore(eniID string, ipv4 string) error { ds.lock.Lock() defer ds.lock.Unlock() - curENI, ok := ds.eniIPPools[eniID] + curENI, ok := ds.eniPool[eniID] if !ok { return errors.New("add ENI's IP to datastore: unknown ENI") } @@ -216,7 +400,7 @@ func (ds *DataStore) AddIPv4AddressToStore(eniID string, ipv4 string) error { // Prometheus gauge totalIPs.Set(float64(ds.total)) - curENI.IPv4Addresses[ipv4] = &AddressInfo{Address: ipv4, Assigned: false} + curENI.IPv4Addresses[ipv4] = &AddressInfo{Address: ipv4} ds.log.Infof("Added ENI(%s)'s IP %s to datastore", eniID, ipv4) return nil } @@ -226,7 +410,7 @@ func (ds *DataStore) DelIPv4AddressFromStore(eniID string, ipv4 string, force bo ds.lock.Lock() defer ds.lock.Unlock() - curENI, ok := ds.eniIPPools[eniID] + curENI, ok := ds.eniPool[eniID] if !ok { return errors.New(UnknownENIError) } @@ -236,18 +420,16 @@ func (ds *DataStore) DelIPv4AddressFromStore(eniID string, ipv4 string, force bo return errors.New(UnknownIPError) } - if ipAddr.Assigned { + if ipAddr.Assigned() { if !force { return errors.New(IPInUseError) } ds.log.Warnf("Force deleting assigned ip %s on eni %s", ipv4, eniID) forceRemovedIPs.Inc() - decrementAssignedCount(ds, curENI, ipAddr) - for key, info := range ds.podsIP { - if info.IP == ipv4 { - delete(ds.podsIP, key) - break - } + ds.unassignPodIPv4AddressUnsafe(curENI, ipAddr) + if err := ds.writeBackingStoreUnsafe(); err != nil { + ds.log.Warnf("Unable to update backing store: %v", err) + // Continuing because 'force' } } @@ -263,79 +445,62 @@ func (ds *DataStore) DelIPv4AddressFromStore(eniID string, ipv4 string, force bo // AssignPodIPv4Address assigns an IPv4 address to pod // It returns the assigned IPv4 address, device number, error -func (ds *DataStore) AssignPodIPv4Address(k8sPod *k8sapi.K8SPodInfo) (ip string, deviceNumber int, err error) { +func (ds *DataStore) AssignPodIPv4Address(ipamKey IPAMKey) (string, int, error) { ds.lock.Lock() defer ds.lock.Unlock() ds.log.Debugf("AssignIPv4Address: IP address pool stats: total: %d, assigned %d", ds.total, ds.assigned) - podKey := PodKey{ - name: k8sPod.Name, - namespace: k8sPod.Namespace, - sandbox: k8sPod.Sandbox, + if eni, addr := ds.eniPool.FindAddressForSandbox(ipamKey); addr != nil { + ds.log.Infof("AssignPodIPv4Address: duplicate pod assign for sandbox %s", ipamKey) + return addr.Address, eni.DeviceNumber, nil } - ipAddr, ok := ds.podsIP[podKey] - if ok { - if ipAddr.IP == k8sPod.IP && k8sPod.IP != "" { - // The caller invoke multiple times to assign(PodName/NameSpace --> same IPAddress). It is not a error, but not very efficient. - ds.log.Infof("AssignPodIPv4Address: duplicate pod assign for IP %s, name %s, namespace %s, sandbox %s", - k8sPod.IP, k8sPod.Name, k8sPod.Namespace, k8sPod.Sandbox) - return ipAddr.IP, ipAddr.DeviceNumber, nil - } - ds.log.Errorf("AssignPodIPv4Address: current IP %s is changed to IP %s for pod(name %s, namespace %s, sandbox %s)", - ipAddr, k8sPod.IP, k8sPod.Name, k8sPod.Namespace, k8sPod.Sandbox) - return "", 0, errors.New("AssignPodIPv4Address: invalid pod with multiple IP addresses") - } - return ds.assignPodIPv4AddressUnsafe(podKey, k8sPod) -} -// It returns the assigned IPv4 address, device number, error -func (ds *DataStore) assignPodIPv4AddressUnsafe(podKey PodKey, k8sPod *k8sapi.K8SPodInfo) (ip string, deviceNumber int, err error) { - for _, eni := range ds.eniIPPools { - if (k8sPod.IP == "") && (len(eni.IPv4Addresses) == eni.AssignedIPv4Addresses) { - // Skip this ENI, since it has no available IP addresses - ds.log.Debugf("AssignPodIPv4Address: Skip ENI %s that does not have available addresses", eni.ID) - continue - } + for _, eni := range ds.eniPool { for _, addr := range eni.IPv4Addresses { - if k8sPod.IP == addr.Address { - // After L-IPAM restart and built IP warm-pool, it needs to take the existing running pod IP out of the pool. - if !addr.Assigned { - incrementAssignedCount(ds, eni, addr) + if !addr.Assigned() && !addr.inCoolingPeriod() { + ds.assignPodIPv4AddressUnsafe(ipamKey, eni, addr) + if err := ds.writeBackingStoreUnsafe(); err != nil { + ds.log.Warnf("Failed to update backing store: %v", err) + // Important! Unwind assignment + ds.unassignPodIPv4AddressUnsafe(eni, addr) + return "", 0, err } - ds.log.Infof("AssignPodIPv4Address: Reassign IP %v to pod (name %s, namespace %s)", - addr.Address, k8sPod.Name, k8sPod.Namespace) - ds.podsIP[podKey] = PodIPInfo{IP: addr.Address, DeviceNumber: eni.DeviceNumber} - return addr.Address, eni.DeviceNumber, nil - } - if !addr.Assigned && k8sPod.IP == "" && !addr.inCoolingPeriod() { - // This is triggered by a pod's Add Network command from CNI plugin - incrementAssignedCount(ds, eni, addr) - ds.log.Infof("AssignPodIPv4Address: Assign IP %v to pod (name %s, namespace %s sandbox %s)", - addr.Address, k8sPod.Name, k8sPod.Namespace, k8sPod.Sandbox) - ds.podsIP[podKey] = PodIPInfo{IP: addr.Address, DeviceNumber: eni.DeviceNumber} + return addr.Address, eni.DeviceNumber, nil } } + ds.log.Debugf("AssignPodIPv4Address: ENI %s does not have available addresses", eni.ID) } ds.log.Errorf("DataStore has no available IP addresses") return "", 0, errors.New("assignPodIPv4AddressUnsafe: no available IP addresses") } -func incrementAssignedCount(ds *DataStore, eni *ENIIPPool, addr *AddressInfo) { +// It returns the assigned IPv4 address, device number +func (ds *DataStore) assignPodIPv4AddressUnsafe(ipamKey IPAMKey, eni *ENI, addr *AddressInfo) (string, int) { + ds.log.Infof("AssignPodIPv4Address: Assign IP %v to sandbox %s", + addr.Address, ipamKey) + + if addr.Assigned() { + panic("addr already assigned") + } + addr.IPAMKey = ipamKey // This marks the addr as assigned + ds.assigned++ - eni.AssignedIPv4Addresses++ - addr.Assigned = true // Prometheus gauge assignedIPs.Set(float64(ds.assigned)) + + return addr.Address, eni.DeviceNumber } -func decrementAssignedCount(ds *DataStore, eni *ENIIPPool, addr *AddressInfo) { +func (ds *DataStore) unassignPodIPv4AddressUnsafe(eni *ENI, addr *AddressInfo) { + if !addr.Assigned() { + // Already unassigned + return + } + ds.log.Infof("UnAssignPodIPv4Address: Unassign IP %v from sandbox %s", + addr.Address, addr.IPAMKey) + addr.IPAMKey = IPAMKey{} // unassign the addr ds.assigned-- - eni.AssignedIPv4Addresses-- - addr.Assigned = false - curTime := time.Now() - eni.lastUnassignedTime = curTime - addr.UnassignedTime = curTime // Prometheus gauge assignedIPs.Set(float64(ds.assigned)) } @@ -347,11 +512,11 @@ func (ds *DataStore) GetStats() (int, int) { // IsRequiredForWarmIPTarget determines if this ENI has warm IPs that are required to fulfill whatever WARM_IP_TARGET is // set to. -func (ds *DataStore) isRequiredForWarmIPTarget(warmIPTarget int, eni *ENIIPPool) bool { +func (ds *DataStore) isRequiredForWarmIPTarget(warmIPTarget int, eni *ENI) bool { otherWarmIPs := 0 - for _, other := range ds.eniIPPools { + for _, other := range ds.eniPool { if other.ID != eni.ID { - otherWarmIPs += len(other.IPv4Addresses) - other.AssignedIPv4Addresses + otherWarmIPs += len(other.IPv4Addresses) - other.AssignedIPv4Addresses() } } return otherWarmIPs < warmIPTarget @@ -359,9 +524,9 @@ func (ds *DataStore) isRequiredForWarmIPTarget(warmIPTarget int, eni *ENIIPPool) // IsRequiredForMinimumIPTarget determines if this ENI is necessary to fulfill whatever MINIMUM_IP_TARGET is // set to. -func (ds *DataStore) isRequiredForMinimumIPTarget(minimumIPTarget int, eni *ENIIPPool) bool { +func (ds *DataStore) isRequiredForMinimumIPTarget(minimumIPTarget int, eni *ENI) bool { otherIPs := 0 - for _, other := range ds.eniIPPools { + for _, other := range ds.eniPool { if other.ID != eni.ID { otherIPs += len(other.IPv4Addresses) } @@ -369,8 +534,8 @@ func (ds *DataStore) isRequiredForMinimumIPTarget(minimumIPTarget int, eni *ENII return otherIPs < minimumIPTarget } -func (ds *DataStore) getDeletableENI(warmIPTarget int, minimumIPTarget int) *ENIIPPool { - for _, eni := range ds.eniIPPools { +func (ds *DataStore) getDeletableENI(warmIPTarget int, minimumIPTarget int) *ENI { + for _, eni := range ds.eniPool { if eni.IsPrimary { ds.log.Debugf("ENI %s cannot be deleted because it is primary", eni.ID) continue @@ -408,37 +573,35 @@ func (ds *DataStore) getDeletableENI(warmIPTarget int, minimumIPTarget int) *ENI } // IsTooYoung returns true if the ENI hasn't been around long enough to be deleted. -func (e *ENIIPPool) isTooYoung() bool { +func (e *ENI) isTooYoung() bool { return time.Since(e.createTime) < minLifeTime } // HasIPInCooling returns true if an IP address was unassigned recently. -func (e *ENIIPPool) hasIPInCooling() bool { - return time.Since(e.lastUnassignedTime) < addressENICoolingPeriod +func (e *ENI) hasIPInCooling() bool { + for _, addr := range e.IPv4Addresses { + if addr.inCoolingPeriod() { + return true + } + } + return false } // HasPods returns true if the ENI has pods assigned to it. -func (e *ENIIPPool) hasPods() bool { - return e.AssignedIPv4Addresses != 0 +func (e *ENI) hasPods() bool { + return e.AssignedIPv4Addresses() != 0 } // GetENINeedsIP finds an ENI in the datastore that needs more IP addresses allocated -func (ds *DataStore) GetENINeedsIP(maxIPperENI int, skipPrimary bool) *ENIIPPool { - // NOTE(jaypipes): Some tests rely on key order so we iterate over the IP - // pool structs here in sorted key order. - // TODO(jaypipes): Don't use a map as the primary iterator vehicle. - // Instead, use a slice of *ENIPool and use a map for existence checks only - eniIDs := make([]string, 0) - for eniID, eni := range ds.eniIPPools { +func (ds *DataStore) GetENINeedsIP(maxIPperENI int, skipPrimary bool) *ENI { + ds.lock.Lock() + defer ds.lock.Unlock() + + for _, eni := range ds.eniPool { if skipPrimary && eni.IsPrimary { ds.log.Debugf("Skip the primary ENI for need IP check") continue } - eniIDs = append(eniIDs, eniID) - } - sort.Strings(eniIDs) - for _, eniID := range eniIDs { - eni := ds.eniIPPools[eniID] if len(eni.IPv4Addresses) < maxIPperENI { ds.log.Debugf("Found ENI %s that has less than the maximum number of IP addresses allocated: cur=%d, max=%d", eni.ID, len(eni.IPv4Addresses), maxIPperENI) @@ -461,111 +624,136 @@ func (ds *DataStore) RemoveUnusedENIFromStore(warmIPTarget int, minimumIPTarget } removableENI := deletableENI.ID - eniIPCount := len(ds.eniIPPools[removableENI].IPv4Addresses) + eniIPCount := len(ds.eniPool[removableENI].IPv4Addresses) ds.total -= eniIPCount ds.log.Infof("RemoveUnusedENIFromStore %s: IP address pool stats: free %d addresses, total: %d, assigned: %d", removableENI, eniIPCount, ds.total, ds.assigned) - delete(ds.eniIPPools, removableENI) + delete(ds.eniPool, removableENI) // Prometheus update - enis.Set(float64(len(ds.eniIPPools))) + enis.Set(float64(len(ds.eniPool))) totalIPs.Set(float64(ds.total)) return removableENI } // RemoveENIFromDataStore removes an ENI from the datastore. It return nil on success or an error. -func (ds *DataStore) RemoveENIFromDataStore(eni string, force bool) error { +func (ds *DataStore) RemoveENIFromDataStore(eniID string, force bool) error { ds.lock.Lock() defer ds.lock.Unlock() - eniIPPool, ok := ds.eniIPPools[eni] + eni, ok := ds.eniPool[eniID] if !ok { return errors.New(UnknownENIError) } - if eniIPPool.hasPods() { + if eni.hasPods() { if !force { return errors.New(ENIInUseError) } // This scenario can occur if the reconciliation process discovered this eni was detached // from the EC2 instance outside of the control of ipamd. If this happens, there's nothing // we can do other than force all pods to be unassigned from the IPs on this eni. - ds.log.Warnf("Force removing eni %s with %d assigned pods", eni, eniIPPool.AssignedIPv4Addresses) + ds.log.Warnf("Force removing eni %s with %d assigned pods", eniID, eni.AssignedIPv4Addresses()) forceRemovedENIs.Inc() - forceRemovedIPs.Add(float64(eniIPPool.AssignedIPv4Addresses)) - for _, addr := range eniIPPool.IPv4Addresses { - if addr.Assigned { - decrementAssignedCount(ds, eniIPPool, addr) + forceRemovedIPs.Add(float64(eni.AssignedIPv4Addresses())) + for _, addr := range eni.IPv4Addresses { + if addr.Assigned() { + ds.unassignPodIPv4AddressUnsafe(eni, addr) } } - for key, info := range ds.podsIP { - if info.DeviceNumber == eniIPPool.DeviceNumber { - delete(ds.podsIP, key) - } + if err := ds.writeBackingStoreUnsafe(); err != nil { + ds.log.Warnf("Unable to update backing store: %v", err) + // Continuing, because 'force' } } - ds.total -= len(eniIPPool.IPv4Addresses) + ds.total -= len(eni.IPv4Addresses) ds.log.Infof("RemoveENIFromDataStore %s: IP address pool stats: free %d addresses, total: %d, assigned: %d", - eni, len(eniIPPool.IPv4Addresses), ds.total, ds.assigned) - delete(ds.eniIPPools, eni) + eni, len(eni.IPv4Addresses), ds.total, ds.assigned) + delete(ds.eniPool, eniID) // Prometheus gauge - enis.Set(float64(len(ds.eniIPPools))) + enis.Set(float64(len(ds.eniPool))) return nil } // UnassignPodIPv4Address a) find out the IP address based on PodName and PodNameSpace // b) mark IP address as unassigned c) returns IP address, ENI's device number, error -func (ds *DataStore) UnassignPodIPv4Address(k8sPod *k8sapi.K8SPodInfo) (ip string, deviceNumber int, err error) { +func (ds *DataStore) UnassignPodIPv4Address(ipamKey IPAMKey) (ip string, deviceNumber int, err error) { ds.lock.Lock() defer ds.lock.Unlock() - ds.log.Debugf("UnassignPodIPv4Address: IP address pool stats: total:%d, assigned %d, pod(Name: %s, Namespace: %s, Sandbox %s)", - ds.total, ds.assigned, k8sPod.Name, k8sPod.Namespace, k8sPod.Sandbox) + ds.log.Debugf("UnassignPodIPv4Address: IP address pool stats: total:%d, assigned %d, sandbox %s", + ds.total, ds.assigned, ipamKey) + + eni, addr := ds.eniPool.FindAddressForSandbox(ipamKey) - podKey := PodKey{ - name: k8sPod.Name, - namespace: k8sPod.Namespace, - sandbox: k8sPod.Sandbox, + if BackfillMissingCheckpointFromCRI && addr == nil { + ds.log.Debugf("UnassignPodIPv4Address: Failed to find IPAM entry under full key, trying CRI-migrated version") + ipamKey.NetworkName = BackfillNetworkName + ipamKey.IfName = BackfillNetworkIface + eni, addr = ds.eniPool.FindAddressForSandbox(ipamKey) } - ipAddr, ok := ds.podsIP[podKey] - if !ok { - ds.log.Warnf("UnassignPodIPv4Address: Failed to find pod %s namespace %q, sandbox %q", - k8sPod.Name, k8sPod.Namespace, k8sPod.Sandbox) + if addr == nil { + ds.log.Warnf("UnassignPodIPv4Address: Failed to find sandbox %s", + ipamKey) return "", 0, ErrUnknownPod } - for _, eni := range ds.eniIPPools { - ip, ok := eni.IPv4Addresses[ipAddr.IP] - if ok && ip.Assigned { - decrementAssignedCount(ds, eni, ip) - ds.log.Infof("UnassignPodIPv4Address: pod (Name: %s, NameSpace %s Sandbox %s)'s ipAddr %s, DeviceNumber%d", - k8sPod.Name, k8sPod.Namespace, k8sPod.Sandbox, ip.Address, eni.DeviceNumber) - delete(ds.podsIP, podKey) - return ip.Address, eni.DeviceNumber, nil - } + ds.unassignPodIPv4AddressUnsafe(eni, addr) + if err := ds.writeBackingStoreUnsafe(); err != nil { + // Unwind un-assignment + ds.assignPodIPv4AddressUnsafe(ipamKey, eni, addr) + return "", 0, err } + addr.UnassignedTime = time.Now() + + ds.log.Infof("UnassignPodIPv4Address: sandbox %s's ipAddr %s, DeviceNumber %d", + ipamKey, addr.Address, eni.DeviceNumber) + return addr.Address, eni.DeviceNumber, nil +} + +// AllocatedIPs returns a recent snapshot of allocated sandbox<->IPs. +// Note result may already be stale by the time you look at it. +func (ds *DataStore) AllocatedIPs() []PodIPInfo { + ds.lock.Lock() + defer ds.lock.Unlock() - ds.log.Warnf("UnassignPodIPv4Address: Failed to find pod %s namespace %s sandbox %s using IP %s", - k8sPod.Name, k8sPod.Namespace, k8sPod.Sandbox, ipAddr.IP) - return "", 0, ErrUnknownPodIP + ret := make([]PodIPInfo, 0, ds.eniPool.AssignedIPv4Addresses()) + for _, eni := range ds.eniPool { + for _, addr := range eni.IPv4Addresses { + if addr.Assigned() { + info := PodIPInfo{ + IPAMKey: addr.IPAMKey, + IP: addr.Address, + DeviceNumber: eni.DeviceNumber, + } + ret = append(ret, info) + } + } + } + return ret } -// GetPodInfos provides pod IP information to introspection endpoint -func (ds *DataStore) GetPodInfos() *map[string]PodIPInfo { +// FreeableIPs returns a list of unused and potentially freeable IPs. +// Note result may already be stale by the time you look at it. +func (ds *DataStore) FreeableIPs(eniID string) []string { ds.lock.Lock() defer ds.lock.Unlock() - var podInfos = make(map[string]PodIPInfo, len(ds.podsIP)) + eni := ds.eniPool[eniID] + if eni == nil { + // Can't free any IPs from an ENI we don't know about... + return []string{} + } - for podKey, podInfo := range ds.podsIP { - key := podKey.name + "_" + podKey.namespace + "_" + podKey.sandbox - podInfos[key] = podInfo - ds.log.Debugf("GetPodInfos: key %s", key) + freeable := make([]string, 0, len(eni.IPv4Addresses)) + for _, addr := range eni.IPv4Addresses { + if !addr.Assigned() { + freeable = append(freeable, addr.Address) + } } - ds.log.Debugf("GetPodInfos: len %d", len(ds.podsIP)) - return &podInfos + return freeable } // GetENIInfos provides ENI IP information to introspection endpoint @@ -576,11 +764,11 @@ func (ds *DataStore) GetENIInfos() *ENIInfos { var eniInfos = ENIInfos{ TotalIPs: ds.total, AssignedIPs: ds.assigned, - ENIIPPools: make(map[string]ENIIPPool, len(ds.eniIPPools)), + ENIs: make(map[string]ENI, len(ds.eniPool)), } - for eni, eniInfo := range ds.eniIPPools { - eniInfos.ENIIPPools[eni] = *eniInfo + for eni, eniInfo := range ds.eniPool { + eniInfos.ENIs[eni] = *eniInfo } return &eniInfos } @@ -589,27 +777,22 @@ func (ds *DataStore) GetENIInfos() *ENIInfos { func (ds *DataStore) GetENIs() int { ds.lock.Lock() defer ds.lock.Unlock() - return len(ds.eniIPPools) + return len(ds.eniPool) } -// GetENIIPPools returns eni's IP address list -func (ds *DataStore) GetENIIPPools(eni string) (map[string]*AddressInfo, error) { +// GetENIIPs returns the known (allocated & unallocated) ENI IPs. +func (ds *DataStore) GetENIIPs(eniID string) ([]string, error) { ds.lock.Lock() defer ds.lock.Unlock() - eniIPPool, ok := ds.eniIPPools[eni] + eni, ok := ds.eniPool[eniID] if !ok { return nil, errors.New(UnknownENIError) } - var ipPool = make(map[string]*AddressInfo, len(eniIPPool.IPv4Addresses)) - for ip, ipAddr := range eniIPPool.IPv4Addresses { - ipPool[ip] = ipAddr + var ipPool = make([]string, 0, len(eni.IPv4Addresses)) + for ip := range eni.IPv4Addresses { + ipPool = append(ipPool, ip) } return ipPool, nil } - -// InCoolingPeriod checks whether an addr is in addressCoolingPeriod -func (addr AddressInfo) inCoolingPeriod() bool { - return time.Since(addr.UnassignedTime) <= addressCoolingPeriod -} diff --git a/pkg/ipamd/datastore/data_store_test.go b/pkg/ipamd/datastore/data_store_test.go index 2f6ae8c1c7..9ef6ee0847 100644 --- a/pkg/ipamd/datastore/data_store_test.go +++ b/pkg/ipamd/datastore/data_store_test.go @@ -14,12 +14,12 @@ package datastore import ( + "errors" "testing" "time" "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" - "github.com/aws/amazon-vpc-cni-k8s/pkg/k8sapi" "github.com/stretchr/testify/assert" ) @@ -32,7 +32,7 @@ var logConfig = logger.Configuration{ var log = logger.New(&logConfig) func TestAddENI(t *testing.T) { - ds := NewDataStore(log) + ds := NewDataStore(log, NullCheckpoint{}) err := ds.AddENI("eni-1", 1, true) assert.NoError(t, err) @@ -43,14 +43,14 @@ func TestAddENI(t *testing.T) { err = ds.AddENI("eni-2", 2, false) assert.NoError(t, err) - assert.Equal(t, len(ds.eniIPPools), 2) + assert.Equal(t, len(ds.eniPool), 2) eniInfos := ds.GetENIInfos() - assert.Equal(t, len(eniInfos.ENIIPPools), 2) + assert.Equal(t, len(eniInfos.ENIs), 2) } func TestDeleteENI(t *testing.T) { - ds := NewDataStore(log) + ds := NewDataStore(log, NullCheckpoint{}) err := ds.AddENI("eni-1", 1, true) assert.NoError(t, err) @@ -62,29 +62,24 @@ func TestDeleteENI(t *testing.T) { assert.NoError(t, err) eniInfos := ds.GetENIInfos() - assert.Equal(t, len(eniInfos.ENIIPPools), 3) + assert.Equal(t, len(eniInfos.ENIs), 3) err = ds.RemoveENIFromDataStore("eni-2", false) assert.NoError(t, err) eniInfos = ds.GetENIInfos() - assert.Equal(t, len(eniInfos.ENIIPPools), 2) + assert.Equal(t, len(eniInfos.ENIs), 2) err = ds.RemoveENIFromDataStore("unknown-eni", false) assert.Error(t, err) eniInfos = ds.GetENIInfos() - assert.Equal(t, len(eniInfos.ENIIPPools), 2) + assert.Equal(t, len(eniInfos.ENIs), 2) // Add an IP and assign a pod. err = ds.AddIPv4AddressToStore("eni-1", "1.1.1.1") assert.NoError(t, err) - podInfo := &k8sapi.K8SPodInfo{ - Name: "pod-1", - Namespace: "ns-1", - IP: "1.1.1.1", - } - ip, device, err := ds.AssignPodIPv4Address(podInfo) + ip, device, err := ds.AssignPodIPv4Address(IPAMKey{"net1", "sandbox1", "eth0"}) assert.NoError(t, err) assert.Equal(t, "1.1.1.1", ip) assert.Equal(t, 1, device) @@ -98,7 +93,7 @@ func TestDeleteENI(t *testing.T) { } func TestAddENIIPv4Address(t *testing.T) { - ds := NewDataStore(log) + ds := NewDataStore(log, NullCheckpoint{}) err := ds.AddENI("eni-1", 1, true) assert.NoError(t, err) @@ -109,34 +104,34 @@ func TestAddENIIPv4Address(t *testing.T) { err = ds.AddIPv4AddressToStore("eni-1", "1.1.1.1") assert.NoError(t, err) assert.Equal(t, ds.total, 1) - assert.Equal(t, len(ds.eniIPPools["eni-1"].IPv4Addresses), 1) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Addresses), 1) err = ds.AddIPv4AddressToStore("eni-1", "1.1.1.1") assert.Error(t, err) assert.Equal(t, ds.total, 1) - assert.Equal(t, len(ds.eniIPPools["eni-1"].IPv4Addresses), 1) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Addresses), 1) err = ds.AddIPv4AddressToStore("eni-1", "1.1.1.2") assert.NoError(t, err) assert.Equal(t, ds.total, 2) - assert.Equal(t, len(ds.eniIPPools["eni-1"].IPv4Addresses), 2) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Addresses), 2) err = ds.AddIPv4AddressToStore("eni-2", "1.1.2.2") assert.NoError(t, err) assert.Equal(t, ds.total, 3) - assert.Equal(t, len(ds.eniIPPools["eni-1"].IPv4Addresses), 2) - assert.Equal(t, len(ds.eniIPPools["eni-2"].IPv4Addresses), 1) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Addresses), 2) + assert.Equal(t, len(ds.eniPool["eni-2"].IPv4Addresses), 1) err = ds.AddIPv4AddressToStore("dummy-eni", "1.1.2.2") assert.Error(t, err) assert.Equal(t, ds.total, 3) - assert.Equal(t, len(ds.eniIPPools["eni-1"].IPv4Addresses), 2) - assert.Equal(t, len(ds.eniIPPools["eni-2"].IPv4Addresses), 1) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Addresses), 2) + assert.Equal(t, len(ds.eniPool["eni-2"].IPv4Addresses), 1) } -func TestGetENIIPPools(t *testing.T) { - ds := NewDataStore(log) +func TestGetENIIPs(t *testing.T) { + ds := NewDataStore(log, NullCheckpoint{}) err := ds.AddENI("eni-1", 1, true) assert.NoError(t, err) @@ -147,84 +142,81 @@ func TestGetENIIPPools(t *testing.T) { err = ds.AddIPv4AddressToStore("eni-1", "1.1.1.1") assert.NoError(t, err) assert.Equal(t, ds.total, 1) - assert.Equal(t, len(ds.eniIPPools["eni-1"].IPv4Addresses), 1) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Addresses), 1) err = ds.AddIPv4AddressToStore("eni-1", "1.1.1.2") assert.NoError(t, err) assert.Equal(t, ds.total, 2) - assert.Equal(t, len(ds.eniIPPools["eni-1"].IPv4Addresses), 2) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Addresses), 2) err = ds.AddIPv4AddressToStore("eni-2", "1.1.2.2") assert.NoError(t, err) assert.Equal(t, ds.total, 3) - assert.Equal(t, len(ds.eniIPPools["eni-1"].IPv4Addresses), 2) - assert.Equal(t, len(ds.eniIPPools["eni-2"].IPv4Addresses), 1) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Addresses), 2) + assert.Equal(t, len(ds.eniPool["eni-2"].IPv4Addresses), 1) - eniIPPool, err := ds.GetENIIPPools("eni-1") + eniIPPool, err := ds.GetENIIPs("eni-1") assert.NoError(t, err) assert.Equal(t, len(eniIPPool), 2) - _, err = ds.GetENIIPPools("dummy-eni") + _, err = ds.GetENIIPs("dummy-eni") assert.Error(t, err) } func TestDelENIIPv4Address(t *testing.T) { - ds := NewDataStore(log) + ds := NewDataStore(log, NullCheckpoint{}) err := ds.AddENI("eni-1", 1, true) assert.NoError(t, err) err = ds.AddIPv4AddressToStore("eni-1", "1.1.1.1") assert.NoError(t, err) assert.Equal(t, ds.total, 1) - assert.Equal(t, len(ds.eniIPPools["eni-1"].IPv4Addresses), 1) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Addresses), 1) + + // Assign a pod. + key := IPAMKey{"net0", "sandbox-1", "eth0"} + ip, device, err := ds.AssignPodIPv4Address(key) + assert.NoError(t, err) + assert.Equal(t, "1.1.1.1", ip) + assert.Equal(t, 1, device) err = ds.AddIPv4AddressToStore("eni-1", "1.1.1.2") assert.NoError(t, err) assert.Equal(t, ds.total, 2) - assert.Equal(t, len(ds.eniIPPools["eni-1"].IPv4Addresses), 2) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Addresses), 2) err = ds.AddIPv4AddressToStore("eni-1", "1.1.1.3") assert.NoError(t, err) assert.Equal(t, ds.total, 3) - assert.Equal(t, len(ds.eniIPPools["eni-1"].IPv4Addresses), 3) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Addresses), 3) err = ds.DelIPv4AddressFromStore("eni-1", "1.1.1.2", false) assert.NoError(t, err) assert.Equal(t, ds.total, 2) - assert.Equal(t, len(ds.eniIPPools["eni-1"].IPv4Addresses), 2) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Addresses), 2) // delete a unknown IP err = ds.DelIPv4AddressFromStore("eni-1", "10.10.10.10", false) assert.Error(t, err) assert.Equal(t, ds.total, 2) - assert.Equal(t, len(ds.eniIPPools["eni-1"].IPv4Addresses), 2) - - // Assign a pod. - podInfo := &k8sapi.K8SPodInfo{ - Name: "pod-1", - Namespace: "ns-1", - IP: "1.1.1.1", - } - ip, device, err := ds.AssignPodIPv4Address(podInfo) - assert.NoError(t, err) - assert.Equal(t, "1.1.1.1", ip) - assert.Equal(t, 1, device) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Addresses), 2) // Test force removal. The first call fails because the IP has a pod assigned to it, but the // second call force-removes it and succeeds. err = ds.DelIPv4AddressFromStore("eni-1", "1.1.1.1", false) assert.Error(t, err) assert.Equal(t, ds.total, 2) - assert.Equal(t, len(ds.eniIPPools["eni-1"].IPv4Addresses), 2) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Addresses), 2) err = ds.DelIPv4AddressFromStore("eni-1", "1.1.1.1", true) assert.NoError(t, err) assert.Equal(t, ds.total, 1) - assert.Equal(t, len(ds.eniIPPools["eni-1"].IPv4Addresses), 1) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Addresses), 1) } func TestPodIPv4Address(t *testing.T) { - ds := NewDataStore(log) + checkpoint := NewTestCheckpoint(struct{}{}) + ds := NewDataStore(log, checkpoint) ds.AddENI("eni-1", 1, true) @@ -232,113 +224,90 @@ func TestPodIPv4Address(t *testing.T) { ds.AddIPv4AddressToStore("eni-1", "1.1.1.1") - ds.AddIPv4AddressToStore("eni-1", "1.1.1.2") - - ds.AddIPv4AddressToStore("eni-2", "1.1.2.2") - - podInfo := k8sapi.K8SPodInfo{ - Name: "pod-1", - Namespace: "ns-1", - IP: "1.1.1.1", - } - - ip, _, err := ds.AssignPodIPv4Address(&podInfo) + key1 := IPAMKey{"net0", "sandbox-1", "eth0"} + ip, _, err := ds.AssignPodIPv4Address(key1) assert.NoError(t, err) assert.Equal(t, "1.1.1.1", ip) - assert.Equal(t, 3, ds.total) - assert.Equal(t, 2, len(ds.eniIPPools["eni-1"].IPv4Addresses)) - assert.Equal(t, 1, ds.eniIPPools["eni-1"].AssignedIPv4Addresses) + assert.Equal(t, 1, ds.total) + assert.Equal(t, 1, len(ds.eniPool["eni-1"].IPv4Addresses)) + assert.Equal(t, 1, ds.eniPool["eni-1"].AssignedIPv4Addresses()) + assert.Equal(t, checkpoint.Data, &CheckpointData{ + Version: CheckpointFormatVersion, + Allocations: []CheckpointEntry{ + {IPAMKey: IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"}, IPv4: "1.1.1.1"}, + }, + }) + + podsInfos := ds.AllocatedIPs() + assert.Equal(t, len(podsInfos), 1) - ip, _, err = ds.AssignPodIPv4Address(&podInfo) - assert.NoError(t, err) - assert.Equal(t, "1.1.1.1", ip) - - podsInfos := ds.GetPodInfos() - assert.Equal(t, len(*podsInfos), 1) + ds.AddIPv4AddressToStore("eni-2", "1.1.2.2") // duplicate add - ip, _, err = ds.AssignPodIPv4Address(&podInfo) + ip, _, err = ds.AssignPodIPv4Address(key1) // same id assert.NoError(t, err) assert.Equal(t, ip, "1.1.1.1") - assert.Equal(t, ds.total, 3) - assert.Equal(t, len(ds.eniIPPools["eni-1"].IPv4Addresses), 2) - assert.Equal(t, ds.eniIPPools["eni-1"].AssignedIPv4Addresses, 1) - - // wrong ip address - podInfo = k8sapi.K8SPodInfo{ - Name: "pod-1", - Namespace: "ns-1", - IP: "1.1.2.10", - } - - _, _, err = ds.AssignPodIPv4Address(&podInfo) + assert.Equal(t, ds.total, 2) + assert.Equal(t, ds.assigned, 1) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Addresses), 1) + assert.Equal(t, ds.eniPool["eni-1"].AssignedIPv4Addresses(), 1) + assert.Equal(t, len(ds.eniPool["eni-2"].IPv4Addresses), 1) + assert.Equal(t, ds.eniPool["eni-2"].AssignedIPv4Addresses(), 0) + + // Checkpoint error + checkpoint.Error = errors.New("fake checkpoint error") + key2 := IPAMKey{"net0", "sandbox-2", "eth0"} + _, _, err = ds.AssignPodIPv4Address(key2) assert.Error(t, err) + assert.Equal(t, checkpoint.Data, &CheckpointData{ + Version: CheckpointFormatVersion, + Allocations: []CheckpointEntry{ + {IPAMKey: IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"}, IPv4: "1.1.1.1"}, + }, + }) + checkpoint.Error = nil - podInfo = k8sapi.K8SPodInfo{ - Name: "pod-1", - Namespace: "ns-2", - IP: "1.1.2.2", - } - - ip, pod1Ns2Device, err := ds.AssignPodIPv4Address(&podInfo) + ip, pod1Ns2Device, err := ds.AssignPodIPv4Address(key2) assert.NoError(t, err) assert.Equal(t, ip, "1.1.2.2") - assert.Equal(t, ds.total, 3) + assert.Equal(t, ds.total, 2) assert.Equal(t, ds.assigned, 2) - assert.Equal(t, len(ds.eniIPPools["eni-2"].IPv4Addresses), 1) - assert.Equal(t, ds.eniIPPools["eni-2"].AssignedIPv4Addresses, 1) + assert.Equal(t, len(ds.eniPool["eni-2"].IPv4Addresses), 1) + assert.Equal(t, ds.eniPool["eni-2"].AssignedIPv4Addresses(), 1) + assert.Equal(t, len(checkpoint.Data.(*CheckpointData).Allocations), 2) - podsInfos = ds.GetPodInfos() - assert.Equal(t, len(*podsInfos), 2) + podsInfos = ds.AllocatedIPs() + assert.Equal(t, len(podsInfos), 2) - podInfo = k8sapi.K8SPodInfo{ - Name: "pod-1", - Namespace: "ns-3", - Sandbox: "container-1", - } + ds.AddIPv4AddressToStore("eni-1", "1.1.1.2") - ip, _, err = ds.AssignPodIPv4Address(&podInfo) + key3 := IPAMKey{"net0", "sandbox-3", "eth0"} + ip, _, err = ds.AssignPodIPv4Address(key3) assert.NoError(t, err) assert.Equal(t, ip, "1.1.1.2") assert.Equal(t, ds.total, 3) assert.Equal(t, ds.assigned, 3) - assert.Equal(t, len(ds.eniIPPools["eni-1"].IPv4Addresses), 2) - assert.Equal(t, ds.eniIPPools["eni-1"].AssignedIPv4Addresses, 2) + assert.Equal(t, len(ds.eniPool["eni-1"].IPv4Addresses), 2) + assert.Equal(t, ds.eniPool["eni-1"].AssignedIPv4Addresses(), 2) + assert.Equal(t, len(checkpoint.Data.(*CheckpointData).Allocations), 3) // no more IP addresses - podInfo = k8sapi.K8SPodInfo{ - Name: "pod-2", - Namespace: "ns-3", - } - - _, _, err = ds.AssignPodIPv4Address(&podInfo) + key4 := IPAMKey{"net0", "sandbox-4", "eth0"} + _, _, err = ds.AssignPodIPv4Address(key4) assert.Error(t, err) // Unassign unknown Pod - _, _, err = ds.UnassignPodIPv4Address(&podInfo) - assert.Error(t, err) - - // Unassign pod which have same name/namespace, but different container - podInfo = k8sapi.K8SPodInfo{ - Name: "pod-1", - Namespace: "ns-3", - Sandbox: "container-2", - } - _, _, err = ds.UnassignPodIPv4Address(&podInfo) + _, _, err = ds.UnassignPodIPv4Address(key4) assert.Error(t, err) - podInfo = k8sapi.K8SPodInfo{ - Name: "pod-1", - Namespace: "ns-2", - } - - _, deviceNum, err := ds.UnassignPodIPv4Address(&podInfo) + _, deviceNum, err := ds.UnassignPodIPv4Address(key2) assert.NoError(t, err) assert.Equal(t, ds.total, 3) assert.Equal(t, ds.assigned, 2) assert.Equal(t, deviceNum, pod1Ns2Device) - assert.Equal(t, len(ds.eniIPPools["eni-2"].IPv4Addresses), 1) - assert.Equal(t, ds.eniIPPools["eni-2"].AssignedIPv4Addresses, 0) + assert.Equal(t, len(ds.eniPool["eni-2"].IPv4Addresses), 1) + assert.Equal(t, ds.eniPool["eni-2"].AssignedIPv4Addresses(), 0) + assert.Equal(t, len(checkpoint.Data.(*CheckpointData).Allocations), 2) noWarmIPTarget := 0 noMinimumIPTarget := 0 @@ -347,8 +316,8 @@ func TestPodIPv4Address(t *testing.T) { eni := ds.RemoveUnusedENIFromStore(noWarmIPTarget, noMinimumIPTarget) assert.True(t, eni == "") - ds.eniIPPools["eni-2"].createTime = time.Time{} - ds.eniIPPools["eni-2"].lastUnassignedTime = time.Time{} + ds.eniPool["eni-2"].createTime = time.Time{} + ds.eniPool["eni-2"].IPv4Addresses["1.1.2.2"].UnassignedTime = time.Time{} eni = ds.RemoveUnusedENIFromStore(noWarmIPTarget, noMinimumIPTarget) assert.Equal(t, eni, "eni-2") @@ -357,39 +326,30 @@ func TestPodIPv4Address(t *testing.T) { } func TestWarmENIInteractions(t *testing.T) { - ds := NewDataStore(log) + ds := NewDataStore(log, NullCheckpoint{}) ds.AddENI("eni-1", 1, true) ds.AddENI("eni-2", 2, false) ds.AddENI("eni-3", 3, false) + ds.AddIPv4AddressToStore("eni-1", "1.1.1.1") + key1 := IPAMKey{"net0", "sandbox-1", "eth0"} + _, _, err := ds.AssignPodIPv4Address(key1) + assert.NoError(t, err) + ds.AddIPv4AddressToStore("eni-1", "1.1.1.2") + key2 := IPAMKey{"net0", "sandbox-2", "eth0"} + _, _, err = ds.AssignPodIPv4Address(key2) + assert.NoError(t, err) + ds.AddIPv4AddressToStore("eni-2", "1.1.2.1") ds.AddIPv4AddressToStore("eni-2", "1.1.2.2") ds.AddIPv4AddressToStore("eni-3", "1.1.3.1") - podInfo := k8sapi.K8SPodInfo{ - Name: "pod-1", - Namespace: "ns-1", - IP: "1.1.1.1", - } - _, _, err := ds.AssignPodIPv4Address(&podInfo) - assert.NoError(t, err) - - podInfo = k8sapi.K8SPodInfo{ - Name: "pod-2", - Namespace: "ns-2", - IP: "1.1.1.2", - } - _, _, err = ds.AssignPodIPv4Address(&podInfo) - assert.NoError(t, err) - noWarmIPTarget := 0 - ds.eniIPPools["eni-2"].createTime = time.Time{} - ds.eniIPPools["eni-2"].lastUnassignedTime = time.Time{} - ds.eniIPPools["eni-3"].createTime = time.Time{} - ds.eniIPPools["eni-3"].lastUnassignedTime = time.Time{} + ds.eniPool["eni-2"].createTime = time.Time{} + ds.eniPool["eni-3"].createTime = time.Time{} // We have three ENIs, 5 IPs and two pods on ENI 1. Each ENI can handle two pods. // We should not be able to remove any ENIs if either warmIPTarget >= 3 or minimumWarmIPTarget >= 5 diff --git a/pkg/ipamd/introspect.go b/pkg/ipamd/introspect.go index dacef6f40f..17037a2be4 100644 --- a/pkg/ipamd/introspect.go +++ b/pkg/ipamd/introspect.go @@ -84,7 +84,6 @@ func (c *IPAMContext) setupIntrospectionServer() *http.Server { serverFunctions := map[string]func(w http.ResponseWriter, r *http.Request){ "/v1/enis": eniV1RequestHandler(c), "/v1/eni-configs": eniConfigRequestHandler(c), - "/v1/pods": podV1RequestHandler(c), "/v1/networkutils-env-settings": networkEnvV1RequestHandler(), "/v1/ipamd-env-settings": ipamdEnvV1RequestHandler(), } @@ -141,18 +140,6 @@ func eniV1RequestHandler(ipam *IPAMContext) func(http.ResponseWriter, *http.Requ } } -func podV1RequestHandler(ipam *IPAMContext) func(http.ResponseWriter, *http.Request) { - return func(w http.ResponseWriter, r *http.Request) { - responseJSON, err := json.Marshal(ipam.dataStore.GetPodInfos()) - if err != nil { - log.Errorf("Failed to marshal pod data: %v", err) - http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) - return - } - logErr(w.Write(responseJSON)) - } -} - func eniConfigRequestHandler(ipam *IPAMContext) func(http.ResponseWriter, *http.Request) { return func(w http.ResponseWriter, r *http.Request) { responseJSON, err := json.Marshal(ipam.eniConfig.Getter()) diff --git a/pkg/ipamd/ipamd.go b/pkg/ipamd/ipamd.go index 3ddb66c094..b765ced769 100644 --- a/pkg/ipamd/ipamd.go +++ b/pkg/ipamd/ipamd.go @@ -28,13 +28,11 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" - "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/kubernetes" "github.com/aws/amazon-vpc-cni-k8s/pkg/awsutils" - "github.com/aws/amazon-vpc-cni-k8s/pkg/cri" "github.com/aws/amazon-vpc-cni-k8s/pkg/eniconfig" "github.com/aws/amazon-vpc-cni-k8s/pkg/ipamd/datastore" - "github.com/aws/amazon-vpc-cni-k8s/pkg/k8sapi" "github.com/aws/amazon-vpc-cni-k8s/pkg/networkutils" ) @@ -113,6 +111,10 @@ const ( // disableENIProvisioning is used to specify that ENI doesn't need to be synced during initializing a pod. envDisableENIProvisioning = "DISABLE_NETWORK_RESOURCE_PROVISIONING" noDisableENIProvisioning = false + + // Specify where ipam should persist its current IP<->container allocations. + envBackingStorePath = "AWS_VPC_K8S_CNI_BACKING_STORE" + defaultBackingStorePath = "/var/run/aws-node/ipam.json" ) var log = logger.Get() @@ -171,10 +173,9 @@ var ( type IPAMContext struct { awsClient awsutils.APIs dataStore *datastore.DataStore - k8sClient k8sapi.K8SAPIs + k8sClient kubernetes.Interface useCustomNetworking bool eniConfig eniconfig.ENIConfig - criClient cri.APIs networkClient networkutils.NetworkAPIs maxIPsPerENI int maxENI int @@ -288,13 +289,12 @@ func prometheusRegister() { // New retrieves IP address usage information from Instance MetaData service and Kubelet // then initializes IP address pool data store -func New(k8sapiClient k8sapi.K8SAPIs, eniConfig *eniconfig.ENIConfigController) (*IPAMContext, error) { +func New(k8sapiClient kubernetes.Interface, eniConfig *eniconfig.ENIConfigController) (*IPAMContext, error) { prometheusRegister() c := &IPAMContext{} c.k8sClient = k8sapiClient c.networkClient = networkutils.New() - c.criClient = cri.New() c.eniConfig = eniConfig client, err := awsutils.New() @@ -311,6 +311,9 @@ func New(k8sapiClient k8sapi.K8SAPIs, eniConfig *eniconfig.ENIConfigController) c.useCustomNetworking = UseCustomNetworkCfg() c.disableENIProvisioning = disablingENIProvisioning() + checkpointer := datastore.NewJSONFile(dsBackingStorePath()) + c.dataStore = datastore.NewDataStore(log, checkpointer) + err = c.nodeInit() if err != nil { return nil, err @@ -325,14 +328,6 @@ func (c *IPAMContext) nodeInit() error { log.Debugf("Start node init") - eniMetadata, tagMap, err := c.awsClient.DescribeAllENIs() - if err != nil { - return errors.New("ipamd init: failed to retrieve attached ENIs info") - } else { - log.Debugf("DescribeAllENIs success: ENIs: %d, tagged: %d", len(eniMetadata), len(tagMap)) - } - c.setUnmanagedENIs(tagMap) - enis := c.filterUnmanagedENIs(eniMetadata) nodeMaxENI, err := c.getMaxENI() if err != nil { log.Error("Failed to get ENI limit") @@ -362,7 +357,14 @@ func (c *IPAMContext) nodeInit() error { return errors.Wrap(err, "ipamd init: failed to set up host network") } - c.dataStore = datastore.NewDataStore(log) + eniMetadata, tagMap, err := c.awsClient.DescribeAllENIs() + if err != nil { + return errors.New("ipamd init: failed to retrieve attached ENIs info") + } + log.Debugf("DescribeAllENIs success: ENIs: %d, tagged: %d", len(eniMetadata), len(tagMap)) + c.setUnmanagedENIs(tagMap) + enis := c.filterUnmanagedENIs(eniMetadata) + for _, eni := range enis { log.Debugf("Discovered ENI %s, trying to set it up", eni.ENIID) // Retry ENI sync @@ -390,14 +392,10 @@ func (c *IPAMContext) nodeInit() error { time.Sleep(eniAttachTime) } } - localPods, err := c.getLocalPodsWithRetry() - if err != nil { - log.Warnf("During ipamd init, failed to get Pod information from Kubernetes API Server %v", err) - ipamdErrInc("nodeInitK8SGetLocalPodIPsFailed") - // This can happens when L-IPAMD starts before kubelet. - return errors.Wrap(err, "failed to get running pods!") + + if err := c.dataStore.ReadBackingStore(); err != nil { + return err } - log.Debugf("getLocalPodsWithRetry() found %d local pods", len(localPods)) rules, err := c.networkClient.GetRuleList() if err != nil { @@ -405,28 +403,15 @@ func (c *IPAMContext) nodeInit() error { return nil } - for _, ip := range localPods { - if ip.Sandbox == "" { - log.Infof("Skipping Pod %s, Namespace %s, due to no matching sandbox", ip.Name, ip.Namespace) - continue - } - if ip.IP == "" { - log.Infof("Skipping Pod %s, Namespace %s, due to no IP", ip.Name, ip.Namespace) - continue - } - log.Infof("Recovered AddNetwork for Pod %s, Namespace %s, Sandbox %s", ip.Name, ip.Namespace, ip.Sandbox) - _, _, err = c.dataStore.AssignPodIPv4Address(ip) - if err != nil { - ipamdErrInc("nodeInitAssignPodIPv4AddressFailed") - log.Warnf("During ipamd init, failed to use pod IP %s returned from Kubernetes API Server %v", ip.IP, err) - } + for _, info := range c.dataStore.AllocatedIPs() { + // TODO(gus): This should really be done via CNI CHECK calls, rather than in ipam (requires upstream k8s changes). // Update ip rules in case there is a change in VPC CIDRs, AWS_VPC_K8S_CNI_EXTERNALSNAT setting - srcIPNet := net.IPNet{IP: net.ParseIP(ip.IP), Mask: net.IPv4Mask(255, 255, 255, 255)} + srcIPNet := net.IPNet{IP: net.ParseIP(info.IP), Mask: net.IPv4Mask(255, 255, 255, 255)} err = c.networkClient.UpdateRuleListBySrc(rules, srcIPNet, pbVPCcidrs, !c.networkClient.UseExternalSNAT()) if err != nil { - log.Errorf("UpdateRuleListBySrc in nodeInit() failed for IP %s: %v", ip.IP, err) + log.Errorf("UpdateRuleListBySrc in nodeInit() failed for IP %s: %v", info.IP, err) } } // For a new node, attach IPs @@ -442,68 +427,6 @@ func (c *IPAMContext) updateIPStats(unmanaged int) { enisMax.Set(float64(c.maxENI - unmanaged)) } -func (c *IPAMContext) getLocalPodsWithRetry() ([]*k8sapi.K8SPodInfo, error) { - var pods []*k8sapi.K8SPodInfo - var err error - for retry := 1; retry <= maxK8SRetries; retry++ { - pods, err = c.k8sClient.K8SGetLocalPodIPs() - if err == nil { - // Check for pods with no IP since the API server might not have the latest state of the node. - allPodsHaveAnIP := true - for _, pod := range pods { - if pod.IP == "" { - log.Infof("Pod %s, Namespace %s, has no IP", pod.Name, pod.Namespace) - allPodsHaveAnIP = false - } - } - if allPodsHaveAnIP { - break - } - log.Warnf("Not all pods have an IP, trying again in %v seconds.", retryK8SInterval.Seconds()) - } - log.Infof("Not able to get local pods yet (attempt %d/%d): %v", retry, maxK8SRetries, err) - time.Sleep(retryK8SInterval) - } - - if err != nil { - return nil, errors.Wrap(err, "no pods because apiserver not running.") - } - - if pods == nil { - return nil, nil - } - - // Ask the CRI for the set of running pod sandboxes. These sandboxes are - // what the CNI operates on, but the Kubernetes API doesn't expose any - // information about them. If we relied only on the Kubernetes API, we - // could leak IPs or unassign an IP from a still-running pod. - var sandboxes map[string]*cri.SandboxInfo - for retry := 1; retry <= maxK8SRetries; retry++ { - sandboxes, err = c.criClient.GetRunningPodSandboxes(log) - if err == nil { - break - } - log.Infof("Not able to get local pod sandboxes yet (attempt %d/%d): %v", retry, maxK8SRetries, err) - time.Sleep(retryK8SInterval) - } - if err != nil { - return nil, errors.Wrap(err, "Unable to get local pod sandboxes") - } - - // TODO consider using map - for _, pod := range pods { - // Fill in the sandbox ID by matching against the pod's UID - for _, sandbox := range sandboxes { - if sandbox.K8SUID == pod.UID { - log.Debugf("Found pod(%v)'s sandbox ID: %v ", sandbox.Name, sandbox.ID) - pod.Sandbox = sandbox.ID - break - } - } - } - return pods, nil -} - // StartNodeIPPoolManager monitors the IP pool, add or del them when it is required. func (c *IPAMContext) StartNodeIPPoolManager() { sleepDuration := ipPoolMonitorInterval / 2 @@ -578,7 +501,7 @@ func (c *IPAMContext) tryFreeENI() { func (c *IPAMContext) tryUnassignIPsFromAll() { if _, over, warmIPTargetDefined := c.ipTargetState(); warmIPTargetDefined && over > 0 { eniInfos := c.dataStore.GetENIInfos() - for eniID := range eniInfos.ENIIPPools { + for eniID := range eniInfos.ENIs { ips, err := c.findFreeableIPs(eniID) if err != nil { log.Errorf("Error finding unassigned IPs: %s", err) @@ -621,38 +544,14 @@ func (c *IPAMContext) tryUnassignIPsFromAll() { // findFreeableIPs finds and returns IPs that are not assigned to Pods but are attached // to ENIs on the node. func (c *IPAMContext) findFreeableIPs(eni string) ([]string, error) { - podIPInfos := c.dataStore.GetPodInfos() - usedIPs := sets.String{} - // Get IPs that are currently in use by pods - for _, pod := range *podIPInfos { - usedIPs.Insert(pod.IP) - } - - // Get IPs that are currently attached to the instance - eniInfos := c.dataStore.GetENIInfos() - eniIPPools := eniInfos.ENIIPPools - - pool, ok := eniIPPools[eni] - if !ok { - return nil, fmt.Errorf("error finding available IPs: eni %s does not exist", eni) - } - - allocatedIPs := sets.String{} - for _, ip := range pool.IPv4Addresses { - allocatedIPs.Insert(ip.Address) - } - - availableIPs := allocatedIPs.Difference(usedIPs).UnsortedList() - var freeableIPs []string + freeableIPs := c.dataStore.FreeableIPs(eni) // Free the number of IPs `over` the warm IP target, unless `over` is greater than the number of available IPs on // this ENI. In that case we should only free the number of available IPs. _, over, _ := c.ipTargetState() - numFreeable := min(over, len(availableIPs)) + numFreeable := min(over, len(freeableIPs)) + freeableIPs = freeableIPs[:numFreeable] - for _, ip := range availableIPs[:numFreeable] { - freeableIPs = append(freeableIPs, ip) - } return freeableIPs, nil } @@ -974,7 +873,7 @@ func (c *IPAMContext) nodeIPPoolReconcile(interval time.Duration) { return } attachedENIs := c.filterUnmanagedENIs(allENIs) - currentENIIPPools := c.dataStore.GetENIInfos().ENIIPPools + currentENIIPPools := c.dataStore.GetENIInfos().ENIs // Check if a new ENI was added, if so we need to update the tags needToUpdateTags := false @@ -997,7 +896,7 @@ func (c *IPAMContext) nodeIPPoolReconcile(interval time.Duration) { // Mark phase for _, attachedENI := range attachedENIs { - eniIPPool, err := c.dataStore.GetENIIPPools(attachedENI.ENIID) + eniIPPool, err := c.dataStore.GetENIIPs(attachedENI.ENIID) if err == nil { // If the attached ENI is in the data store log.Debugf("Reconcile existing ENI %s IP pool", attachedENI.ENIID) @@ -1039,7 +938,9 @@ func (c *IPAMContext) nodeIPPoolReconcile(interval time.Duration) { c.lastNodeIPPoolAction = curTime } -func (c *IPAMContext) eniIPPoolReconcile(ipPool map[string]*datastore.AddressInfo, attachedENI awsutils.ENIMetadata, eni string) { +func (c *IPAMContext) eniIPPoolReconcile(ipPool []string, attachedENI awsutils.ENIMetadata, eni string) { + seenIPs := make(map[string]bool) + for _, privateIPv4 := range attachedENI.IPv4Addresses { strPrivateIPv4 := aws.StringValue(privateIPv4.PrivateIpAddress) if strPrivateIPv4 == c.primaryIP[eni] { @@ -1082,8 +983,8 @@ func (c *IPAMContext) eniIPPoolReconcile(ipPool map[string]*datastore.AddressInf err := c.dataStore.AddIPv4AddressToStore(eni, strPrivateIPv4) if err != nil && err.Error() == datastore.IPAlreadyInStoreError { - // mark action = remove it from ipPool since the IP should not be deleted - delete(ipPool, strPrivateIPv4) + // mark action + seenIPs[strPrivateIPv4] = true continue } @@ -1097,7 +998,11 @@ func (c *IPAMContext) eniIPPoolReconcile(ipPool map[string]*datastore.AddressInf } // Sweep phase, delete remaining IPs - for existingIP := range ipPool { + for _, existingIP := range ipPool { + if seenIPs[existingIP] { + continue + } + log.Debugf("Reconcile and delete IP %s on ENI %s", existingIP, eni) // Force the delete, since aws local metadata has told us that this ENI is no longer // attached, so any IPs assigned from this ENI will no longer work. @@ -1124,6 +1029,13 @@ func UseCustomNetworkCfg() bool { return false } +func dsBackingStorePath() string { + if value := os.Getenv(envBackingStorePath); value != "" { + return value + } + return defaultBackingStorePath +} + func getWarmIPTarget() int { inputStr, found := os.LookupEnv(envWarmIPTarget) diff --git a/pkg/ipamd/ipamd_test.go b/pkg/ipamd/ipamd_test.go index 5e4f008a3b..21f3a6b34a 100644 --- a/pkg/ipamd/ipamd_test.go +++ b/pkg/ipamd/ipamd_test.go @@ -14,6 +14,7 @@ package ipamd import ( + "fmt" "net" "os" "reflect" @@ -22,18 +23,15 @@ import ( "github.com/aws/amazon-vpc-cni-k8s/pkg/apis/crd/v1alpha1" "github.com/aws/amazon-vpc-cni-k8s/pkg/awsutils" mock_awsutils "github.com/aws/amazon-vpc-cni-k8s/pkg/awsutils/mocks" - "github.com/aws/amazon-vpc-cni-k8s/pkg/cri" - mock_cri "github.com/aws/amazon-vpc-cni-k8s/pkg/cri/mocks" mock_eniconfig "github.com/aws/amazon-vpc-cni-k8s/pkg/eniconfig/mocks" "github.com/aws/amazon-vpc-cni-k8s/pkg/ipamd/datastore" - "github.com/aws/amazon-vpc-cni-k8s/pkg/k8sapi" - mock_k8sapi "github.com/aws/amazon-vpc-cni-k8s/pkg/k8sapi/mocks" mock_networkutils "github.com/aws/amazon-vpc-cni-k8s/pkg/networkutils/mocks" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "github.com/vishvananda/netlink" + k8s_fake "k8s.io/client-go/kubernetes/fake" ) const ( @@ -53,80 +51,84 @@ const ( vpcCIDR = "10.10.0.0/16" ) -func setup(t *testing.T) (*gomock.Controller, - *mock_awsutils.MockAPIs, - *mock_k8sapi.MockK8SAPIs, - *mock_cri.MockAPIs, - *mock_networkutils.MockNetworkAPIs, - *mock_eniconfig.MockENIConfig) { +type testMocks struct { + ctrl *gomock.Controller + awsutils *mock_awsutils.MockAPIs + clientset *k8s_fake.Clientset + network *mock_networkutils.MockNetworkAPIs + eniconfig *mock_eniconfig.MockENIConfig +} + +func setup(t *testing.T) *testMocks { ctrl := gomock.NewController(t) - return ctrl, - mock_awsutils.NewMockAPIs(ctrl), - mock_k8sapi.NewMockK8SAPIs(ctrl), - mock_cri.NewMockAPIs(ctrl), - mock_networkutils.NewMockNetworkAPIs(ctrl), - mock_eniconfig.NewMockENIConfig(ctrl) + return &testMocks{ + ctrl: ctrl, + awsutils: mock_awsutils.NewMockAPIs(ctrl), + clientset: k8s_fake.NewSimpleClientset(), + network: mock_networkutils.NewMockNetworkAPIs(ctrl), + eniconfig: mock_eniconfig.NewMockENIConfig(ctrl), + } } func TestNodeInit(t *testing.T) { - ctrl, mockAWS, mockK8S, mockCRI, mockNetwork, _ := setup(t) - defer ctrl.Finish() + m := setup(t) + defer m.ctrl.Finish() + + fakeCheckpoint := datastore.CheckpointData{ + Version: datastore.CheckpointFormatVersion, + Allocations: []datastore.CheckpointEntry{ + {IPAMKey: datastore.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-id", IfName: "eth0"}, IPv4: ipaddr02}, + }, + } mockContext := &IPAMContext{ - awsClient: mockAWS, - k8sClient: mockK8S, + awsClient: m.awsutils, maxIPsPerENI: 14, maxENI: 4, warmENITarget: 1, warmIPTarget: 3, primaryIP: make(map[string]string), terminating: int32(0), - criClient: mockCRI, - networkClient: mockNetwork} + networkClient: m.network, + dataStore: datastore.NewDataStore(log, datastore.NewTestCheckpoint(fakeCheckpoint)), + } - eni1, eni2 := getDummyENIMetdata() + eni1, eni2 := getDummyENIMetadata() var cidrs []*string - mockAWS.EXPECT().GetENILimit().Return(4, nil) - mockAWS.EXPECT().GetENIipLimit().Return(14, nil) - mockAWS.EXPECT().GetIPv4sFromEC2(eni1.ENIID).Return(eni1.IPv4Addresses, nil) - mockAWS.EXPECT().GetVPCIPv4CIDR().Return(vpcCIDR) + m.awsutils.EXPECT().GetENILimit().Return(4, nil) + m.awsutils.EXPECT().GetENIipLimit().Return(14, nil) + m.awsutils.EXPECT().GetIPv4sFromEC2(eni1.ENIID).AnyTimes().Return(eni1.IPv4Addresses, nil) + m.awsutils.EXPECT().GetIPv4sFromEC2(eni2.ENIID).AnyTimes().Return(eni2.IPv4Addresses, nil) + m.awsutils.EXPECT().GetVPCIPv4CIDR().Return(vpcCIDR) _, parsedVPCCIDR, _ := net.ParseCIDR(vpcCIDR) primaryIP := net.ParseIP(ipaddr01) - mockAWS.EXPECT().GetVPCIPv4CIDRs().Return(cidrs) - mockAWS.EXPECT().GetPrimaryENImac().Return("") - mockNetwork.EXPECT().SetupHostNetwork(parsedVPCCIDR, cidrs, "", &primaryIP).Return(nil) + m.awsutils.EXPECT().GetVPCIPv4CIDRs().Return(cidrs) + m.awsutils.EXPECT().GetPrimaryENImac().Return("") + m.network.EXPECT().SetupHostNetwork(parsedVPCCIDR, cidrs, "", &primaryIP).Return(nil) - mockAWS.EXPECT().GetPrimaryENI().AnyTimes().Return(primaryENIid) + m.awsutils.EXPECT().GetPrimaryENI().AnyTimes().Return(primaryENIid) eniMetadataSlice := []awsutils.ENIMetadata{eni1, eni2} - mockAWS.EXPECT().DescribeAllENIs().Return(eniMetadataSlice, map[string]awsutils.TagMap{}, nil) - mockNetwork.EXPECT().SetupENINetwork(gomock.Any(), secMAC, secDevice, secSubnet) - - mockAWS.EXPECT().GetLocalIPv4().Return(ipaddr01) - k8sName := "/k8s_POD_" + "pod1" + "_" + "default" + "_" + "pod-uid" + "_0" - mockK8S.EXPECT().K8SGetLocalPodIPs().Return([]*k8sapi.K8SPodInfo{{Name: "pod1", - Namespace: "default", UID: "pod-uid", IP: ipaddr02}}, nil) + m.awsutils.EXPECT().DescribeAllENIs().Return(eniMetadataSlice, map[string]awsutils.TagMap{}, nil) + m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, secDevice, secSubnet) - var criList = make(map[string]*cri.SandboxInfo, 0) - criList["pod-uid"] = &cri.SandboxInfo{ID: "sandbox-id", - Name: k8sName, K8SUID: "pod-uid"} - mockCRI.EXPECT().GetRunningPodSandboxes(gomock.Any()).Return(criList, nil) + m.awsutils.EXPECT().GetLocalIPv4().Return(ipaddr01) var rules []netlink.Rule - mockNetwork.EXPECT().GetRuleList().Return(rules, nil) + m.network.EXPECT().GetRuleList().Return(rules, nil) - mockNetwork.EXPECT().UseExternalSNAT().Return(false) - mockNetwork.EXPECT().UpdateRuleListBySrc(gomock.Any(), gomock.Any(), gomock.Any(), true) + m.network.EXPECT().UseExternalSNAT().Return(false) + m.network.EXPECT().UpdateRuleListBySrc(gomock.Any(), gomock.Any(), gomock.Any(), true) // Add IPs - mockAWS.EXPECT().AllocIPAddresses(gomock.Any(), gomock.Any()) + m.awsutils.EXPECT().AllocIPAddresses(gomock.Any(), gomock.Any()) err := mockContext.nodeInit() assert.NoError(t, err) } -func getDummyENIMetdata() (awsutils.ENIMetadata, awsutils.ENIMetadata) { +func getDummyENIMetadata() (awsutils.ENIMetadata, awsutils.ENIMetadata) { primary := true notPrimary := false testAddr1 := ipaddr01 @@ -176,23 +178,22 @@ func TestIncreaseIPPoolCustomENI(t *testing.T) { } func testIncreaseIPPool(t *testing.T, useENIConfig bool) { - ctrl, mockAWS, mockK8S, _, mockNetwork, mockENIConfig := setup(t) - defer ctrl.Finish() + m := setup(t) + defer m.ctrl.Finish() mockContext := &IPAMContext{ - awsClient: mockAWS, - k8sClient: mockK8S, + awsClient: m.awsutils, maxIPsPerENI: 14, maxENI: 4, warmENITarget: 1, - networkClient: mockNetwork, + networkClient: m.network, useCustomNetworking: UseCustomNetworkCfg(), - eniConfig: mockENIConfig, + eniConfig: m.eniconfig, primaryIP: make(map[string]string), terminating: int32(0), } - mockContext.dataStore = datastore.NewDataStore(log) + mockContext.dataStore = testDatastore() primary := true notPrimary := false @@ -213,13 +214,13 @@ func testIncreaseIPPool(t *testing.T, useENIConfig bool) { } if useENIConfig { - mockENIConfig.EXPECT().MyENIConfig().Return(podENIConfig, nil) - mockAWS.EXPECT().AllocENI(true, sg, podENIConfig.Subnet).Return(eni2, nil) + m.eniconfig.EXPECT().MyENIConfig().Return(podENIConfig, nil) + m.awsutils.EXPECT().AllocENI(true, sg, podENIConfig.Subnet).Return(eni2, nil) } else { - mockAWS.EXPECT().AllocENI(false, nil, "").Return(eni2, nil) + m.awsutils.EXPECT().AllocENI(false, nil, "").Return(eni2, nil) } - mockAWS.EXPECT().GetAttachedENIs().Return([]awsutils.ENIMetadata{ + m.awsutils.EXPECT().GetAttachedENIs().Return([]awsutils.ENIMetadata{ { ENIID: primaryENIid, MAC: primaryMAC, @@ -250,19 +251,19 @@ func testIncreaseIPPool(t *testing.T, useENIConfig bool) { }, }, nil) - mockAWS.EXPECT().GetPrimaryENI().Return(primaryENIid) - mockNetwork.EXPECT().SetupENINetwork(gomock.Any(), secMAC, secDevice, secSubnet) + m.awsutils.EXPECT().GetPrimaryENI().Return(primaryENIid) + m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, secDevice, secSubnet) - mockAWS.EXPECT().AllocIPAddresses(eni2, 14) - mockAWS.EXPECT().GetPrimaryENI().Return(primaryENIid) + m.awsutils.EXPECT().AllocIPAddresses(eni2, 14) + m.awsutils.EXPECT().GetPrimaryENI().Return(primaryENIid) mockContext.increaseIPPool() } func TestTryAddIPToENI(t *testing.T) { _ = os.Unsetenv(envCustomNetworkCfg) - ctrl, mockAWS, mockK8S, _, mockNetwork, mockENIConfig := setup(t) - defer ctrl.Finish() + m := setup(t) + defer m.ctrl.Finish() primary := true notPrimary := false @@ -273,19 +274,18 @@ func TestTryAddIPToENI(t *testing.T) { warmIpTarget := 3 mockContext := &IPAMContext{ - awsClient: mockAWS, - k8sClient: mockK8S, + awsClient: m.awsutils, maxIPsPerENI: 14, maxENI: 4, warmENITarget: 1, warmIPTarget: warmIpTarget, - networkClient: mockNetwork, - eniConfig: mockENIConfig, + networkClient: m.network, + eniConfig: m.eniconfig, primaryIP: make(map[string]string), terminating: int32(0), } - mockContext.dataStore = datastore.NewDataStore(log) + mockContext.dataStore = testDatastore() podENIConfig := &v1alpha1.ENIConfigSpec{ SecurityGroups: []string{"sg1-id", "sg2-id"}, @@ -296,9 +296,9 @@ func TestTryAddIPToENI(t *testing.T) { sg = append(sg, aws.String(sgID)) } - mockAWS.EXPECT().AllocENI(false, nil, "").Return(secENIid, nil) - mockAWS.EXPECT().AllocIPAddresses(secENIid, warmIpTarget) - mockAWS.EXPECT().GetAttachedENIs().Return([]awsutils.ENIMetadata{ + m.awsutils.EXPECT().AllocENI(false, nil, "").Return(secENIid, nil) + m.awsutils.EXPECT().AllocIPAddresses(secENIid, warmIpTarget) + m.awsutils.EXPECT().GetAttachedENIs().Return([]awsutils.ENIMetadata{ { ENIID: primaryENIid, MAC: primaryMAC, @@ -328,26 +328,25 @@ func TestTryAddIPToENI(t *testing.T) { }, }, }, nil) - mockAWS.EXPECT().GetPrimaryENI().Return(primaryENIid) - mockNetwork.EXPECT().SetupENINetwork(gomock.Any(), secMAC, secDevice, secSubnet) - mockAWS.EXPECT().GetPrimaryENI().Return(primaryENIid) + m.awsutils.EXPECT().GetPrimaryENI().Return(primaryENIid) + m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, secDevice, secSubnet) + m.awsutils.EXPECT().GetPrimaryENI().Return(primaryENIid) mockContext.increaseIPPool() } func TestNodeIPPoolReconcile(t *testing.T) { - ctrl, mockAWS, mockK8S, _, mockNetwork, _ := setup(t) - defer ctrl.Finish() + m := setup(t) + defer m.ctrl.Finish() mockContext := &IPAMContext{ - awsClient: mockAWS, - k8sClient: mockK8S, - networkClient: mockNetwork, + awsClient: m.awsutils, + networkClient: m.network, primaryIP: make(map[string]string), terminating: int32(0), } - mockContext.dataStore = datastore.NewDataStore(log) + mockContext.dataStore = testDatastore() primary := true notPrimary := false @@ -370,18 +369,18 @@ func TestNodeIPPoolReconcile(t *testing.T) { }, }, } - mockAWS.EXPECT().GetAttachedENIs().Return(eniMetadata, nil) - mockAWS.EXPECT().GetPrimaryENI().Times(2).Return(primaryENIid) - mockAWS.EXPECT().DescribeAllENIs().Return(eniMetadata, map[string]awsutils.TagMap{}, nil) + m.awsutils.EXPECT().GetAttachedENIs().Return(eniMetadata, nil) + m.awsutils.EXPECT().GetPrimaryENI().Times(2).Return(primaryENIid) + m.awsutils.EXPECT().DescribeAllENIs().Return(eniMetadata, map[string]awsutils.TagMap{}, nil) mockContext.nodeIPPoolReconcile(0) curENIs := mockContext.dataStore.GetENIInfos() - assert.Equal(t, len(curENIs.ENIIPPools), 1) + assert.Equal(t, len(curENIs.ENIs), 1) assert.Equal(t, curENIs.TotalIPs, 1) // remove 1 IP - mockAWS.EXPECT().GetAttachedENIs().Return([]awsutils.ENIMetadata{ + m.awsutils.EXPECT().GetAttachedENIs().Return([]awsutils.ENIMetadata{ { ENIID: primaryENIid, MAC: primaryMAC, @@ -397,21 +396,21 @@ func TestNodeIPPoolReconcile(t *testing.T) { mockContext.nodeIPPoolReconcile(0) curENIs = mockContext.dataStore.GetENIInfos() - assert.Equal(t, len(curENIs.ENIIPPools), 1) + assert.Equal(t, len(curENIs.ENIs), 1) assert.Equal(t, curENIs.TotalIPs, 0) // remove eni - mockAWS.EXPECT().GetAttachedENIs().Return(nil, nil) + m.awsutils.EXPECT().GetAttachedENIs().Return(nil, nil) mockContext.nodeIPPoolReconcile(0) curENIs = mockContext.dataStore.GetENIInfos() - assert.Equal(t, len(curENIs.ENIIPPools), 0) + assert.Equal(t, len(curENIs.ENIs), 0) assert.Equal(t, curENIs.TotalIPs, 0) } func TestGetWarmENITarget(t *testing.T) { - ctrl, _, _, _, _, _ := setup(t) - defer ctrl.Finish() + m := setup(t) + defer m.ctrl.Finish() _ = os.Setenv("WARM_IP_TARGET", "5") warmIPTarget := getWarmIPTarget() @@ -427,18 +426,17 @@ func TestGetWarmENITarget(t *testing.T) { } func TestGetWarmIPTargetState(t *testing.T) { - ctrl, mockAWS, mockK8S, _, mockNetwork, _ := setup(t) - defer ctrl.Finish() + m := setup(t) + defer m.ctrl.Finish() mockContext := &IPAMContext{ - awsClient: mockAWS, - k8sClient: mockK8S, - networkClient: mockNetwork, + awsClient: m.awsutils, + networkClient: m.network, primaryIP: make(map[string]string), terminating: int32(0), } - mockContext.dataStore = datastore.NewDataStore(log) + mockContext.dataStore = testDatastore() _, _, warmIPTargetDefined := mockContext.ipTargetState() assert.False(t, warmIPTargetDefined) @@ -471,8 +469,8 @@ func TestGetWarmIPTargetState(t *testing.T) { } func TestIPAMContext_nodeIPPoolTooLow(t *testing.T) { - ctrl, mockAWS, mockK8S, _, mockNetwork, mockENIConfig := setup(t) - defer ctrl.Finish() + m := setup(t) + defer m.ctrl.Finish() type fields struct { maxIPsPerENI int @@ -486,9 +484,9 @@ func TestIPAMContext_nodeIPPoolTooLow(t *testing.T) { fields fields want bool }{ - {"Test new ds, all defaults", fields{14, 1, 0, datastore.NewDataStore(log)}, true}, - {"Test new ds, 0 ENIs", fields{14, 0, 0, datastore.NewDataStore(log)}, true}, - {"Test new ds, 3 warm IPs", fields{14, 0, 3, datastore.NewDataStore(log)}, true}, + {"Test new ds, all defaults", fields{14, 1, 0, testDatastore()}, true}, + {"Test new ds, 0 ENIs", fields{14, 0, 0, testDatastore()}, true}, + {"Test new ds, 3 warm IPs", fields{14, 0, 3, testDatastore()}, true}, {"Test 3 unused IPs, 1 warm", fields{3, 1, 1, datastoreWith3FreeIPs()}, false}, {"Test 1 used, 1 warm ENI", fields{3, 1, 0, datastoreWith1Pod1()}, true}, {"Test 1 used, 0 warm ENI", fields{3, 0, 0, datastoreWith1Pod1()}, false}, @@ -498,12 +496,11 @@ func TestIPAMContext_nodeIPPoolTooLow(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { c := &IPAMContext{ - awsClient: mockAWS, + awsClient: m.awsutils, dataStore: tt.fields.datastore, - k8sClient: mockK8S, useCustomNetworking: false, - eniConfig: mockENIConfig, - networkClient: mockNetwork, + eniConfig: m.eniconfig, + networkClient: m.network, maxIPsPerENI: tt.fields.maxIPsPerENI, maxENI: -1, warmENITarget: tt.fields.warmENITarget, @@ -516,8 +513,12 @@ func TestIPAMContext_nodeIPPoolTooLow(t *testing.T) { } } +func testDatastore() *datastore.DataStore { + return datastore.NewDataStore(log, datastore.NewTestCheckpoint(datastore.CheckpointData{Version: datastore.CheckpointFormatVersion})) +} + func datastoreWith3FreeIPs() *datastore.DataStore { - datastoreWith3FreeIPs := datastore.NewDataStore(log) + datastoreWith3FreeIPs := testDatastore() _ = datastoreWith3FreeIPs.AddENI(primaryENIid, 1, true) _ = datastoreWith3FreeIPs.AddIPv4AddressToStore(primaryENIid, ipaddr01) _ = datastoreWith3FreeIPs.AddIPv4AddressToStore(primaryENIid, ipaddr02) @@ -528,45 +529,32 @@ func datastoreWith3FreeIPs() *datastore.DataStore { func datastoreWith1Pod1() *datastore.DataStore { datastoreWith1Pod1 := datastoreWith3FreeIPs() - podInfo1 := k8sapi.K8SPodInfo{ - Name: "pod-1", - Namespace: "ns-1", - IP: ipaddr01, - } - _, _, _ = datastoreWith1Pod1.AssignPodIPv4Address(&podInfo1) + _, _, _ = datastoreWith1Pod1.AssignPodIPv4Address(datastore.IPAMKey{ + NetworkName: "net0", + ContainerID: "sandbox-1", + IfName: "eth0", + }) return datastoreWith1Pod1 } func datastoreWith3Pods() *datastore.DataStore { datastoreWith3Pods := datastoreWith3FreeIPs() - podInfo1 := k8sapi.K8SPodInfo{ - Name: "pod-1", - Namespace: "ns-1", - IP: ipaddr01, - } - _, _, _ = datastoreWith3Pods.AssignPodIPv4Address(&podInfo1) - - podInfo2 := k8sapi.K8SPodInfo{ - Name: "pod-2", - Namespace: "ns-1", - IP: ipaddr02, - } - _, _, _ = datastoreWith3Pods.AssignPodIPv4Address(&podInfo2) - - podInfo3 := k8sapi.K8SPodInfo{ - Name: "pod-3", - Namespace: "ns-1", - IP: ipaddr03, + for i := 0; i < 3; i++ { + key := datastore.IPAMKey{ + NetworkName: "net0", + ContainerID: fmt.Sprintf("sandbox-%d", i), + IfName: "eth0", + } + _, _, _ = datastoreWith3Pods.AssignPodIPv4Address(key) } - _, _, _ = datastoreWith3Pods.AssignPodIPv4Address(&podInfo3) return datastoreWith3Pods } func TestIPAMContext_filterUnmanagedENIs(t *testing.T) { ctrl := gomock.NewController(t) - eni1, eni2 := getDummyENIMetdata() + eni1, eni2 := getDummyENIMetadata() allENIs := []awsutils.ENIMetadata{eni1, eni2} primaryENIonly := []awsutils.ENIMetadata{eni1} eni1TagMap := map[string]awsutils.TagMap{eni1.ENIID: {"hi": "tag", eniNoManageTagKey: "true"}} @@ -597,8 +585,8 @@ func TestIPAMContext_filterUnmanagedENIs(t *testing.T) { } func TestDisablingENIProvisioning(t *testing.T) { - ctrl, _, _, _, _, _ := setup(t) - defer ctrl.Finish() + m := setup(t) + defer m.ctrl.Finish() _ = os.Setenv(envDisableENIProvisioning, "true") disabled := disablingENIProvisioning() diff --git a/pkg/ipamd/rpc_handler.go b/pkg/ipamd/rpc_handler.go index 9ba52fb300..7882d0089c 100644 --- a/pkg/ipamd/rpc_handler.go +++ b/pkg/ipamd/rpc_handler.go @@ -28,7 +28,6 @@ import ( "google.golang.org/grpc/reflection" "github.com/aws/amazon-vpc-cni-k8s/pkg/ipamd/datastore" - "github.com/aws/amazon-vpc-cni-k8s/pkg/k8sapi" "github.com/aws/amazon-vpc-cni-k8s/rpc" ) @@ -44,13 +43,15 @@ type server struct { // AddNetwork processes CNI add network request and return an IP address for container func (s *server) AddNetwork(ctx context.Context, in *rpc.AddNetworkRequest) (*rpc.AddNetworkReply, error) { - log.Infof("Received AddNetwork for NS %s, Pod %s, NameSpace %s, Sandbox %s, ifname %s", - in.Netns, in.K8S_POD_NAME, in.K8S_POD_NAMESPACE, in.K8S_POD_INFRA_CONTAINER_ID, in.IfName) + log.Infof("Received AddNetwork for NS %s, Sandbox %s, ifname %s", + in.Netns, in.ContainerID, in.IfName) - addr, deviceNumber, err := s.ipamContext.dataStore.AssignPodIPv4Address(&k8sapi.K8SPodInfo{ - Name: in.K8S_POD_NAME, - Namespace: in.K8S_POD_NAMESPACE, - Sandbox: in.K8S_POD_INFRA_CONTAINER_ID}) + ipamKey := datastore.IPAMKey{ + ContainerID: in.ContainerID, + IfName: in.IfName, + NetworkName: in.NetworkName, + } + addr, deviceNumber, err := s.ipamContext.dataStore.AssignPodIPv4Address(ipamKey) var pbVPCcidrs []string for _, cidr := range s.ipamContext.awsClient.GetVPCIPv4CIDRs() { @@ -80,21 +81,16 @@ func (s *server) AddNetwork(ctx context.Context, in *rpc.AddNetworkRequest) (*rp } func (s *server) DelNetwork(ctx context.Context, in *rpc.DelNetworkRequest) (*rpc.DelNetworkReply, error) { - log.Infof("Received DelNetwork for Pod %s, Namespace %s, Sandbox %s", - in.K8S_POD_NAME, in.K8S_POD_NAMESPACE, in.K8S_POD_INFRA_CONTAINER_ID) + log.Infof("Received DelNetwork for Sandbox %s", in.ContainerID) delIPCnt.With(prometheus.Labels{"reason": in.Reason}).Inc() - ip, deviceNumber, err := s.ipamContext.dataStore.UnassignPodIPv4Address(&k8sapi.K8SPodInfo{ - Name: in.K8S_POD_NAME, - Namespace: in.K8S_POD_NAMESPACE, - Sandbox: in.K8S_POD_INFRA_CONTAINER_ID}) - - if err != nil && err == datastore.ErrUnknownPod { - // If L-IPAMD restarts, the pod's IP address are assigned by only pod's name and namespace due to kubelet's introspection. - ip, deviceNumber, err = s.ipamContext.dataStore.UnassignPodIPv4Address(&k8sapi.K8SPodInfo{ - Name: in.K8S_POD_NAME, - Namespace: in.K8S_POD_NAMESPACE}) + ipamKey := datastore.IPAMKey{ + ContainerID: in.ContainerID, + IfName: in.IfName, + NetworkName: in.NetworkName, } + ip, deviceNumber, err := s.ipamContext.dataStore.UnassignPodIPv4Address(ipamKey) + log.Infof("Send DelNetworkReply: IPv4Addr %s, DeviceNumber: %d, err: %v", ip, deviceNumber, err) return &rpc.DelNetworkReply{Success: err == nil, IPv4Addr: ip, DeviceNumber: int32(deviceNumber)}, err @@ -102,7 +98,7 @@ func (s *server) DelNetwork(ctx context.Context, in *rpc.DelNetworkRequest) (*rp // RunRPCHandler handles request from gRPC func (c *IPAMContext) RunRPCHandler() error { - log.Infof("Serving RPC Handler on ", ipamdgRPCaddress) + log.Infof("Serving RPC Handler on %s", ipamdgRPCaddress) listener, err := net.Listen("tcp", ipamdgRPCaddress) if err != nil { log.Errorf("Failed to listen gRPC port: %v", err) diff --git a/pkg/ipamd/rpc_handler_test.go b/pkg/ipamd/rpc_handler_test.go index 8e9057d847..7634062696 100644 --- a/pkg/ipamd/rpc_handler_test.go +++ b/pkg/ipamd/rpc_handler_test.go @@ -26,29 +26,26 @@ import ( ) func TestServer_AddNetwork(t *testing.T) { - ctrl, mockAWS, mockK8S, mockCRI, mockNetwork, _ := setup(t) - defer ctrl.Finish() + m := setup(t) + defer m.ctrl.Finish() mockContext := &IPAMContext{ - awsClient: mockAWS, - k8sClient: mockK8S, + awsClient: m.awsutils, maxIPsPerENI: 14, maxENI: 4, warmENITarget: 1, warmIPTarget: 3, - criClient: mockCRI, - networkClient: mockNetwork, - dataStore: datastore.NewDataStore(log), + networkClient: m.network, + dataStore: datastore.NewDataStore(log, datastore.NullCheckpoint{}), } rpcServer := server{ipamContext: mockContext} addNetworkRequest := &pb.AddNetworkRequest{ - Netns: "netns", - K8S_POD_NAME: "pod", - K8S_POD_NAMESPACE: "ns", - K8S_POD_INFRA_CONTAINER_ID: "cid", - IfName: "eni", + Netns: "netns", + NetworkName: "net0", + ContainerID: "cid", + IfName: "eni", } vpcCIDRs := []*string{aws.String(vpcCIDR)} @@ -72,10 +69,10 @@ func TestServer_AddNetwork(t *testing.T) { }, } for _, tc := range testCases { - mockAWS.EXPECT().GetVPCIPv4CIDRs().Return(tc.vpcCIDRs) - mockNetwork.EXPECT().UseExternalSNAT().Return(tc.useExternalSNAT) + m.awsutils.EXPECT().GetVPCIPv4CIDRs().Return(tc.vpcCIDRs) + m.network.EXPECT().UseExternalSNAT().Return(tc.useExternalSNAT) if !tc.useExternalSNAT { - mockNetwork.EXPECT().GetExcludeSNATCIDRs().Return(tc.snatExclusionCIDRs) + m.network.EXPECT().GetExcludeSNATCIDRs().Return(tc.snatExclusionCIDRs) } addNetworkReply, err := rpcServer.AddNetwork(context.TODO(), addNetworkRequest) diff --git a/rpc/rpc.pb.go b/rpc/rpc.pb.go index 8258713db5..a185656b15 100644 --- a/rpc/rpc.pb.go +++ b/rpc/rpc.pb.go @@ -27,8 +27,10 @@ type AddNetworkRequest struct { K8S_POD_NAME string `protobuf:"bytes,1,opt,name=K8S_POD_NAME,json=k8SPODNAME,proto3" json:"K8S_POD_NAME,omitempty"` K8S_POD_NAMESPACE string `protobuf:"bytes,2,opt,name=K8S_POD_NAMESPACE,json=k8SPODNAMESPACE,proto3" json:"K8S_POD_NAMESPACE,omitempty"` K8S_POD_INFRA_CONTAINER_ID string `protobuf:"bytes,3,opt,name=K8S_POD_INFRA_CONTAINER_ID,json=k8SPODINFRACONTAINERID,proto3" json:"K8S_POD_INFRA_CONTAINER_ID,omitempty"` - Netns string `protobuf:"bytes,4,opt,name=Netns,json=netns,proto3" json:"Netns,omitempty"` + ContainerID string `protobuf:"bytes,7,opt,name=ContainerID,json=containerID,proto3" json:"ContainerID,omitempty"` IfName string `protobuf:"bytes,5,opt,name=IfName,json=ifName,proto3" json:"IfName,omitempty"` + NetworkName string `protobuf:"bytes,6,opt,name=NetworkName,json=networkName,proto3" json:"NetworkName,omitempty"` + Netns string `protobuf:"bytes,4,opt,name=Netns,json=netns,proto3" json:"Netns,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -38,7 +40,7 @@ func (m *AddNetworkRequest) Reset() { *m = AddNetworkRequest{} } func (m *AddNetworkRequest) String() string { return proto.CompactTextString(m) } func (*AddNetworkRequest) ProtoMessage() {} func (*AddNetworkRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_rpc_052972c68ac5c392, []int{0} + return fileDescriptor_rpc_884830e012bd6176, []int{0} } func (m *AddNetworkRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_AddNetworkRequest.Unmarshal(m, b) @@ -79,9 +81,9 @@ func (m *AddNetworkRequest) GetK8S_POD_INFRA_CONTAINER_ID() string { return "" } -func (m *AddNetworkRequest) GetNetns() string { +func (m *AddNetworkRequest) GetContainerID() string { if m != nil { - return m.Netns + return m.ContainerID } return "" } @@ -93,6 +95,20 @@ func (m *AddNetworkRequest) GetIfName() string { return "" } +func (m *AddNetworkRequest) GetNetworkName() string { + if m != nil { + return m.NetworkName + } + return "" +} + +func (m *AddNetworkRequest) GetNetns() string { + if m != nil { + return m.Netns + } + return "" +} + type AddNetworkReply struct { Success bool `protobuf:"varint,1,opt,name=Success,json=success,proto3" json:"Success,omitempty"` IPv4Addr string `protobuf:"bytes,2,opt,name=IPv4Addr,json=iPv4Addr,proto3" json:"IPv4Addr,omitempty"` @@ -108,7 +124,7 @@ func (m *AddNetworkReply) Reset() { *m = AddNetworkReply{} } func (m *AddNetworkReply) String() string { return proto.CompactTextString(m) } func (*AddNetworkReply) ProtoMessage() {} func (*AddNetworkReply) Descriptor() ([]byte, []int) { - return fileDescriptor_rpc_052972c68ac5c392, []int{1} + return fileDescriptor_rpc_884830e012bd6176, []int{1} } func (m *AddNetworkReply) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_AddNetworkReply.Unmarshal(m, b) @@ -168,6 +184,9 @@ type DelNetworkRequest struct { K8S_POD_NAMESPACE string `protobuf:"bytes,2,opt,name=K8S_POD_NAMESPACE,json=k8SPODNAMESPACE,proto3" json:"K8S_POD_NAMESPACE,omitempty"` K8S_POD_INFRA_CONTAINER_ID string `protobuf:"bytes,3,opt,name=K8S_POD_INFRA_CONTAINER_ID,json=k8SPODINFRACONTAINERID,proto3" json:"K8S_POD_INFRA_CONTAINER_ID,omitempty"` Reason string `protobuf:"bytes,5,opt,name=Reason,json=reason,proto3" json:"Reason,omitempty"` + ContainerID string `protobuf:"bytes,8,opt,name=ContainerID,json=containerID,proto3" json:"ContainerID,omitempty"` + IfName string `protobuf:"bytes,6,opt,name=IfName,json=ifName,proto3" json:"IfName,omitempty"` + NetworkName string `protobuf:"bytes,7,opt,name=NetworkName,json=networkName,proto3" json:"NetworkName,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -177,7 +196,7 @@ func (m *DelNetworkRequest) Reset() { *m = DelNetworkRequest{} } func (m *DelNetworkRequest) String() string { return proto.CompactTextString(m) } func (*DelNetworkRequest) ProtoMessage() {} func (*DelNetworkRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_rpc_052972c68ac5c392, []int{2} + return fileDescriptor_rpc_884830e012bd6176, []int{2} } func (m *DelNetworkRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DelNetworkRequest.Unmarshal(m, b) @@ -225,6 +244,27 @@ func (m *DelNetworkRequest) GetReason() string { return "" } +func (m *DelNetworkRequest) GetContainerID() string { + if m != nil { + return m.ContainerID + } + return "" +} + +func (m *DelNetworkRequest) GetIfName() string { + if m != nil { + return m.IfName + } + return "" +} + +func (m *DelNetworkRequest) GetNetworkName() string { + if m != nil { + return m.NetworkName + } + return "" +} + type DelNetworkReply struct { Success bool `protobuf:"varint,1,opt,name=Success,json=success,proto3" json:"Success,omitempty"` IPv4Addr string `protobuf:"bytes,2,opt,name=IPv4Addr,json=iPv4Addr,proto3" json:"IPv4Addr,omitempty"` @@ -238,7 +278,7 @@ func (m *DelNetworkReply) Reset() { *m = DelNetworkReply{} } func (m *DelNetworkReply) String() string { return proto.CompactTextString(m) } func (*DelNetworkReply) ProtoMessage() {} func (*DelNetworkReply) Descriptor() ([]byte, []int) { - return fileDescriptor_rpc_052972c68ac5c392, []int{3} + return fileDescriptor_rpc_884830e012bd6176, []int{3} } func (m *DelNetworkReply) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DelNetworkReply.Unmarshal(m, b) @@ -391,32 +431,35 @@ var _CNIBackend_serviceDesc = grpc.ServiceDesc{ Metadata: "rpc.proto", } -func init() { proto.RegisterFile("rpc.proto", fileDescriptor_rpc_052972c68ac5c392) } - -var fileDescriptor_rpc_052972c68ac5c392 = []byte{ - // 383 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x53, 0xcd, 0xae, 0x93, 0x40, - 0x18, 0x2d, 0x22, 0x94, 0x7e, 0x69, 0x42, 0x3a, 0x69, 0x08, 0xe9, 0xaa, 0x61, 0xd5, 0xb8, 0xe8, - 0x42, 0x5d, 0x34, 0xc6, 0x0d, 0x02, 0x26, 0xc4, 0x38, 0x25, 0x43, 0x75, 0x4b, 0x28, 0x8c, 0x09, - 0x81, 0x02, 0xce, 0x40, 0xb5, 0x6f, 0xe0, 0xeb, 0xb8, 0xf0, 0x29, 0x7c, 0x29, 0xc3, 0x94, 0x5a, - 0xee, 0x65, 0x7b, 0x17, 0x77, 0x79, 0xce, 0x9c, 0x93, 0xef, 0x7c, 0x3f, 0x03, 0x33, 0x56, 0x27, - 0xdb, 0x9a, 0x55, 0x4d, 0x85, 0x64, 0x56, 0x27, 0xd6, 0x5f, 0x09, 0x16, 0x76, 0x9a, 0x62, 0xda, - 0xfc, 0xa8, 0x58, 0x4e, 0xe8, 0xf7, 0x96, 0xf2, 0x06, 0xad, 0x61, 0xfe, 0x69, 0x17, 0x46, 0xc1, - 0xde, 0x8d, 0xb0, 0xfd, 0xd9, 0x33, 0xa5, 0xb5, 0xb4, 0x99, 0x11, 0xc8, 0x77, 0x61, 0xb0, 0x77, - 0x3b, 0x06, 0xbd, 0x82, 0xc5, 0x50, 0x11, 0x06, 0xb6, 0xe3, 0x99, 0x2f, 0x84, 0x4c, 0xbf, 0xcb, - 0x04, 0x8d, 0xde, 0xc1, 0xea, 0xa6, 0xf5, 0xf1, 0x47, 0x62, 0x47, 0xce, 0x1e, 0x1f, 0x6c, 0x1f, - 0x7b, 0x24, 0xf2, 0x5d, 0x53, 0x16, 0x26, 0xe3, 0x6a, 0x12, 0xef, 0xff, 0x9f, 0x7d, 0x17, 0x2d, - 0x41, 0xc1, 0xb4, 0x29, 0xb9, 0xf9, 0x52, 0xc8, 0x94, 0xb2, 0x03, 0xc8, 0x00, 0xd5, 0xff, 0x86, - 0xe3, 0x13, 0x35, 0x15, 0x41, 0xab, 0x99, 0x40, 0xd6, 0x6f, 0x09, 0xf4, 0x61, 0x37, 0x75, 0x71, - 0x41, 0x26, 0x4c, 0xc3, 0x36, 0x49, 0x28, 0xe7, 0xa2, 0x0d, 0x8d, 0x4c, 0xf9, 0x15, 0xa2, 0x15, - 0x68, 0x7e, 0x70, 0x7e, 0x6b, 0xa7, 0x29, 0xeb, 0xa3, 0x6b, 0x59, 0x8f, 0x91, 0x05, 0x73, 0x97, - 0x9e, 0xb3, 0x84, 0xe2, 0xf6, 0x74, 0xa4, 0x4c, 0x94, 0x57, 0xc8, 0x3c, 0x1d, 0x70, 0x68, 0x03, - 0xfa, 0x17, 0x4e, 0xbd, 0x9f, 0x0d, 0x65, 0x65, 0x5c, 0x84, 0xd8, 0x3e, 0x88, 0x38, 0x1a, 0xd1, - 0xdb, 0x87, 0x74, 0x57, 0xe9, 0x6b, 0xe0, 0x24, 0x59, 0xca, 0xb8, 0xa9, 0xae, 0xe5, 0xae, 0xd2, - 0xb9, 0xc7, 0xd6, 0x1f, 0x09, 0x16, 0x2e, 0x2d, 0x9e, 0xed, 0x06, 0x0c, 0x50, 0x09, 0x8d, 0x79, - 0x55, 0xde, 0x66, 0xcd, 0x04, 0xb2, 0x72, 0xd0, 0x87, 0xb1, 0x9f, 0x6e, 0xd4, 0xf2, 0x78, 0xd4, - 0xaf, 0x7f, 0x49, 0x00, 0x0e, 0xf6, 0x3f, 0xc4, 0x49, 0x4e, 0xcb, 0x14, 0xbd, 0x07, 0xb8, 0xaf, - 0x19, 0x19, 0xdb, 0xee, 0xa8, 0x47, 0x57, 0xbc, 0x5a, 0x8e, 0xf8, 0xba, 0xb8, 0x58, 0x93, 0xce, - 0x7d, 0x4f, 0xde, 0xbb, 0x47, 0x1b, 0xe8, 0xdd, 0x8f, 0x5a, 0xb4, 0x26, 0x47, 0x55, 0xfc, 0x9e, - 0x37, 0xff, 0x02, 0x00, 0x00, 0xff, 0xff, 0x36, 0x31, 0xee, 0x44, 0x4a, 0x03, 0x00, 0x00, +func init() { proto.RegisterFile("rpc.proto", fileDescriptor_rpc_884830e012bd6176) } + +var fileDescriptor_rpc_884830e012bd6176 = []byte{ + // 425 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x53, 0x4f, 0x8f, 0x93, 0x40, + 0x14, 0xdf, 0xb6, 0x96, 0xb2, 0x6f, 0x9b, 0x90, 0x4e, 0x36, 0xcd, 0xa4, 0xa7, 0x86, 0xd3, 0xc6, + 0xc3, 0x1e, 0xd4, 0xc3, 0xc6, 0x78, 0x41, 0xa8, 0xc9, 0xc4, 0x38, 0x25, 0xc3, 0xea, 0x95, 0xb0, + 0xc3, 0x98, 0x10, 0xd8, 0x01, 0x67, 0xa0, 0xba, 0xdf, 0xc0, 0x9b, 0xf1, 0xa3, 0xf8, 0x0d, 0x0d, + 0x53, 0x56, 0x70, 0x49, 0x7a, 0xf2, 0xe0, 0xf1, 0xf7, 0x2f, 0xc3, 0x7b, 0x3f, 0x1e, 0x9c, 0xab, + 0x8a, 0x5f, 0x57, 0xaa, 0xac, 0x4b, 0x34, 0x53, 0x15, 0x77, 0x7f, 0x4c, 0x61, 0xe5, 0xa5, 0x29, + 0x15, 0xf5, 0xd7, 0x52, 0xe5, 0x4c, 0x7c, 0x69, 0x84, 0xae, 0xd1, 0x16, 0x96, 0xef, 0x6f, 0xa2, + 0x38, 0xdc, 0x07, 0x31, 0xf5, 0x3e, 0xec, 0xf0, 0x64, 0x3b, 0xb9, 0x3a, 0x67, 0x90, 0xdf, 0x44, + 0xe1, 0x3e, 0x68, 0x19, 0xf4, 0x1c, 0x56, 0x43, 0x47, 0x14, 0x7a, 0xfe, 0x0e, 0x4f, 0x8d, 0xcd, + 0xe9, 0x6d, 0x86, 0x46, 0xaf, 0x61, 0xf3, 0xe8, 0x25, 0xf4, 0x1d, 0xf3, 0x62, 0x7f, 0x4f, 0x6f, + 0x3d, 0x42, 0x77, 0x2c, 0x26, 0x01, 0x9e, 0x99, 0xd0, 0xfa, 0x18, 0x32, 0xfa, 0x1f, 0x99, 0x04, + 0x68, 0x0b, 0x17, 0x7e, 0x29, 0xeb, 0x24, 0x93, 0x42, 0x91, 0x00, 0x2f, 0x8c, 0xf9, 0x82, 0xf7, + 0x14, 0x5a, 0x83, 0x45, 0x3e, 0xd3, 0xe4, 0x5e, 0xe0, 0xb9, 0x11, 0xad, 0xcc, 0xa0, 0x36, 0xd9, + 0x4d, 0x65, 0x44, 0xeb, 0x98, 0x94, 0x3d, 0x85, 0x2e, 0x61, 0x4e, 0x45, 0x2d, 0x35, 0x7e, 0x66, + 0xb4, 0xb9, 0x6c, 0x81, 0xfb, 0x6b, 0x02, 0xce, 0x70, 0x23, 0x55, 0xf1, 0x80, 0x30, 0x2c, 0xa2, + 0x86, 0x73, 0xa1, 0xb5, 0x59, 0x85, 0xcd, 0x16, 0xfa, 0x08, 0xd1, 0x06, 0x6c, 0x12, 0x1e, 0x5e, + 0x79, 0x69, 0xaa, 0xba, 0xf1, 0xed, 0xac, 0xc3, 0xc8, 0x85, 0x65, 0x20, 0x0e, 0x19, 0x17, 0xb4, + 0xb9, 0xbf, 0x13, 0xca, 0x3c, 0x33, 0x67, 0xcb, 0x74, 0xc0, 0xa1, 0x2b, 0x70, 0x3e, 0x6a, 0xb1, + 0xfb, 0x56, 0x0b, 0x25, 0x93, 0x22, 0xa2, 0xde, 0xad, 0x19, 0xc3, 0x66, 0x4e, 0xf3, 0x37, 0xdd, + 0xbe, 0xf4, 0x29, 0xf4, 0x79, 0x96, 0x2a, 0x8d, 0xad, 0xed, 0xac, 0x7d, 0xe9, 0xd0, 0x61, 0xf7, + 0xe7, 0x14, 0x56, 0x81, 0x28, 0xfe, 0xdb, 0x16, 0xd7, 0x60, 0x31, 0x91, 0xe8, 0x52, 0x3e, 0x76, + 0xa4, 0x0c, 0x7a, 0xda, 0xae, 0x7d, 0xaa, 0x5d, 0xeb, 0x54, 0xbb, 0x8b, 0x51, 0xbb, 0x6e, 0x0e, + 0xce, 0x70, 0x25, 0xff, 0xae, 0xc6, 0xd9, 0xb8, 0xc6, 0x17, 0xdf, 0x27, 0x00, 0x3e, 0x25, 0x6f, + 0x13, 0x9e, 0x0b, 0x99, 0xa2, 0x37, 0x00, 0xfd, 0x2f, 0x84, 0xd6, 0xd7, 0xed, 0xd1, 0x8d, 0xae, + 0x6c, 0x73, 0x39, 0xe2, 0xab, 0xe2, 0xc1, 0x3d, 0x6b, 0xd3, 0xfd, 0x97, 0x77, 0xe9, 0x51, 0xbb, + 0x5d, 0xfa, 0xc9, 0x88, 0xee, 0xd9, 0x9d, 0x65, 0xae, 0xfb, 0xe5, 0xef, 0x00, 0x00, 0x00, 0xff, + 0xff, 0xbf, 0x77, 0x76, 0x6d, 0xea, 0x03, 0x00, 0x00, } diff --git a/rpc/rpc.proto b/rpc/rpc.proto index bb3281f1ce..e5bf4cc782 100644 --- a/rpc/rpc.proto +++ b/rpc/rpc.proto @@ -12,9 +12,11 @@ message AddNetworkRequest { string K8S_POD_NAME = 1; string K8S_POD_NAMESPACE = 2; string K8S_POD_INFRA_CONTAINER_ID = 3; - string Netns = 4; + string ContainerID = 7; string IfName = 5; - // next field: 6 + string NetworkName = 6; + string Netns = 4; + // next field: 8 } message AddNetworkReply { @@ -31,7 +33,10 @@ message DelNetworkRequest { string K8S_POD_NAMESPACE = 2; string K8S_POD_INFRA_CONTAINER_ID = 3; string Reason = 5; - // next field: 6 + string ContainerID = 8; + string IfName = 6; + string NetworkName = 7; + // next field: 9 } message DelNetworkReply {