diff --git a/pkg/hns/endpoint_windows.go b/pkg/hns/endpoint_windows.go index b0c462854..35fece7ab 100644 --- a/pkg/hns/endpoint_windows.go +++ b/pkg/hns/endpoint_windows.go @@ -39,7 +39,7 @@ type EndpointInfo struct { NetworkId string Gateway net.IP IpAddress net.IP - MacAddress string + MacAddress string } // GetSandboxContainerID returns the sandbox ID of this pod. @@ -205,7 +205,8 @@ func ConstructHnsResult(hnsNetwork *hcsshim.HNSNetwork, hnsEndpoint *hcsshim.HNS resultIPConfig := ¤t.IPConfig{ Address: net.IPNet{ IP: hnsEndpoint.IPAddress, - Mask: ipSubnet.Mask}, + Mask: ipSubnet.Mask, + }, Gateway: net.ParseIP(hnsEndpoint.GatewayAddress), } result := ¤t.Result{ @@ -249,7 +250,7 @@ func GenerateHcnEndpoint(epInfo *EndpointInfo, n *NetConf) (*hcn.HostComputeEndp Minor: 0, }, Name: epInfo.EndpointName, - MacAddress: epInfo.MacAddress, + MacAddress: epInfo.MacAddress, HostComputeNetwork: epInfo.NetworkId, Dns: hcn.Dns{ Domain: epInfo.DNS.Domain, @@ -289,7 +290,7 @@ func RemoveHcnEndpoint(epName string) error { if epNamespace != nil { err = hcn.RemoveNamespaceEndpoint(hcnEndpoint.HostComputeNamespace, hcnEndpoint.Id) if err != nil && !hcn.IsNotFoundError(err) { - return errors.Annotatef(err,"error removing endpoint: %s from namespace", epName) + return errors.Annotatef(err, "error removing endpoint: %s from namespace", epName) } } @@ -361,7 +362,8 @@ func ConstructHcnResult(hcnNetwork *hcn.HostComputeNetwork, hcnEndpoint *hcn.Hos resultIPConfig := ¤t.IPConfig{ Address: net.IPNet{ IP: ipAddress, - Mask: ipSubnet.Mask}, + Mask: ipSubnet.Mask, + }, Gateway: net.ParseIP(hcnEndpoint.Routes[0].NextHop), } result := ¤t.Result{ diff --git a/pkg/hns/netconf_suite_windows_test.go b/pkg/hns/netconf_suite_windows_test.go index 0877bed10..f08660d57 100644 --- a/pkg/hns/netconf_suite_windows_test.go +++ b/pkg/hns/netconf_suite_windows_test.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -14,10 +14,10 @@ package hns import ( + "testing" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - - "testing" ) func TestNetConf(t *testing.T) { diff --git a/pkg/hns/netconf_windows.go b/pkg/hns/netconf_windows.go index cae0589f7..837cc1d6d 100644 --- a/pkg/hns/netconf_windows.go +++ b/pkg/hns/netconf_windows.go @@ -66,9 +66,9 @@ var protocolEnums = map[string]uint32{ } func (p *PortMapEntry) GetProtocolEnum() (uint32, error) { - var u, err = strconv.ParseUint(p.Protocol, 0, 10) + u, err := strconv.ParseUint(p.Protocol, 0, 10) if err != nil { - var pe, exist = protocolEnums[strings.ToLower(p.Protocol)] + pe, exist := protocolEnums[strings.ToLower(p.Protocol)] if !exist { return 0, errors.New("invalid protocol supplied to port mapping policy") } @@ -323,7 +323,7 @@ func (n *NetConf) ApplyPortMappingPolicy(portMappings []PortMapEntry) { toPolicyValue := func(p *PortMapEntry) json.RawMessage { if n.ApiVersion == 2 { - var protocolEnum, _ = p.GetProtocolEnum() + protocolEnum, _ := p.GetProtocolEnum() return bprintf(`{"Type": "PortMapping", "Settings": {"InternalPort": %d, "ExternalPort": %d, "Protocol": %d, "VIP": "%s"}}`, p.ContainerPort, p.HostPort, protocolEnum, p.HostIP) } return bprintf(`{"Type": "NAT", "InternalPort": %d, "ExternalPort": %d, "Protocol": "%s"}`, p.ContainerPort, p.HostPort, p.Protocol) diff --git a/pkg/hns/netconf_windows_test.go b/pkg/hns/netconf_windows_test.go index 9cd7c4346..211387051 100644 --- a/pkg/hns/netconf_windows_test.go +++ b/pkg/hns/netconf_windows_test.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, diff --git a/plugins/ipam/host-local/backend/allocator/allocator.go b/plugins/ipam/host-local/backend/allocator/allocator.go index c060ed740..aad6d5d6f 100644 --- a/plugins/ipam/host-local/backend/allocator/allocator.go +++ b/plugins/ipam/host-local/backend/allocator/allocator.go @@ -40,11 +40,50 @@ func NewIPAllocator(s *RangeSet, store backend.Store, id int) *IPAllocator { } } -// Get allocates an IP -func (a *IPAllocator) Get(id string, ifname string, requestedIP net.IP) (*current.IPConfig, error) { +// GetByPodNsAndName allocates an IP or used reserved IP for specified pod +func (a *IPAllocator) GetByPodNsAndName(id string, ifname string, requestedIP net.IP, podNs, podName string) (*current.IPConfig, error) { a.store.Lock() defer a.store.Unlock() + if len(podName) != 0 { + podIPIsExist, knownIP := a.store.HasReservedIP(podNs, podName) + + if podIPIsExist { + // podName file is exist, update ip file with new container id. + _, err := a.store.ReservePodInfo(id, knownIP, podNs, podName, podIPIsExist) + if err != nil { + return nil, err + } + + reservedIP, gw := a.GetGWofKnowIP(knownIP) + if reservedIP == nil { + return nil, fmt.Errorf("no IP addresses available in range set: %s", a.rangeset.String()) + } + + return ¤t.IPConfig{ + Address: *reservedIP, + Gateway: gw, + }, nil + } + // reserve ip for new pod + ipCfg, err := a.Get(id, ifname, requestedIP) + if err != nil { + return ipCfg, err + } + if ipCfg != nil { + _, err := a.store.ReservePodInfo(id, ipCfg.Address.IP, podNs, podName, podIPIsExist) + if err != nil { + return ipCfg, err + } + } + return ipCfg, nil + } + + return a.Get(id, ifname, requestedIP) +} + +// Get allocates an IP +func (a *IPAllocator) Get(id string, ifname string, requestedIP net.IP) (*current.IPConfig, error) { var reservedIP *net.IPNet var gw net.IP @@ -123,6 +162,18 @@ func (a *IPAllocator) Release(id string, ifname string) error { return a.store.ReleaseByID(id, ifname) } +// GetGWofKnowIP returns the known IP, its mask, and its gateway +func (a *IPAllocator) GetGWofKnowIP(ip net.IP) (*net.IPNet, net.IP) { + rg := Range{} + for i, r := range *a.rangeset { + if r.Contains(ip) { + rg = (*a.rangeset)[i] + break + } + } + return &net.IPNet{IP: ip, Mask: rg.Subnet.Mask}, rg.Gateway +} + type RangeIter struct { rangeset *RangeSet diff --git a/plugins/ipam/host-local/backend/disk/backend.go b/plugins/ipam/host-local/backend/disk/backend.go index 3ad19d99d..1e0ea1c7f 100644 --- a/plugins/ipam/host-local/backend/disk/backend.go +++ b/plugins/ipam/host-local/backend/disk/backend.go @@ -15,6 +15,7 @@ package disk import ( + "fmt" "net" "os" "path/filepath" @@ -200,3 +201,117 @@ func GetEscapedPath(dataDir string, fname string) string { } return filepath.Join(dataDir, fname) } + +// HasReservedIP verify the pod already had reserved ip or not. +// and return the reserved ip on the other hand. +func (s *Store) HasReservedIP(podNs, podName string) (bool, net.IP) { + ip := net.IP{} + if len(podName) == 0 { + return false, ip + } + + // Pod, ip mapping info are recorded with file name: PodIP_PodNs_PodName + podIPNsNameFileName, err := s.findPodFileName("", podNs, podName) + if err != nil { + return false, ip + } + + if len(podIPNsNameFileName) != 0 { + ipStr, ns, name := resolvePodFileName(podIPNsNameFileName) + if ns == podNs && name == podName { + ip = net.ParseIP(ipStr) + if ip != nil { + return true, ip + } + } + } + + return false, ip +} + +// ReservePodInfo create podName file for storing ip or update ip file with container id +// in terms of podIPIsExist +func (s *Store) ReservePodInfo(id string, ip net.IP, podNs, podName string, podIPIsExist bool) (bool, error) { + if podIPIsExist { + // pod Ns/Name file is exist, update ip file with new container id. + fname := GetEscapedPath(s.dataDir, ip.String()) + err := os.WriteFile(fname, []byte(strings.TrimSpace(id)), 0o644) + if err != nil { + return false, err + } + } else if len(podName) != 0 { + // for new pod, create a new file named "PodIP_PodNs_PodName", + // if there is already file named with prefix "ip_", rename the old file with new PodNs and PodName. + podIPNsNameFile := GetEscapedPath(s.dataDir, podFileName(ip.String(), podNs, podName)) + podIPNsNameFileName, err := s.findPodFileName(ip.String(), "", "") + if err != nil { + return false, err + } + + if len(podIPNsNameFileName) != 0 { + oldPodIPNsNameFile := GetEscapedPath(s.dataDir, podIPNsNameFileName) + err = os.Rename(oldPodIPNsNameFile, podIPNsNameFile) + if err != nil { + return false, err + } + return true, nil + } + + err = os.WriteFile(podIPNsNameFile, []byte{}, 0o644) + if err != nil { + return false, err + } + } + + return true, nil +} + +func podFileName(ip, ns, name string) string { + if len(ip) != 0 && len(ns) != 0 { + return fmt.Sprintf("%s_%s_%s", ip, ns, name) + } + + return name +} + +func resolvePodFileName(fName string) (string, string, string) { + parts := strings.Split(fName, "_") + ip := "" + ns := "" + name := "" + if len(parts) == 3 { + ip = parts[0] + ns = parts[1] + name = parts[2] + } + + return ip, ns, name +} + +func (s *Store) findPodFileName(ip, ns, name string) (string, error) { + var pattern string + + switch { + case len(ip) != 0: + pattern = fmt.Sprintf("%s_*", ip) + case len(ns) != 0 && len(name) != 0: + pattern = fmt.Sprintf("*_%s_%s", ns, name) + default: + return "", nil + } + pattern = GetEscapedPath(s.dataDir, pattern) + + podFiles, err := filepath.Glob(pattern) + if err != nil { + return "", err + } + + if len(podFiles) == 1 { + _, fName := filepath.Split(podFiles[0]) + if strings.Count(fName, "_") == 2 { + return fName, nil + } + } + + return "", nil +} diff --git a/plugins/ipam/host-local/backend/store.go b/plugins/ipam/host-local/backend/store.go index afd2af6e8..073b3d5ff 100644 --- a/plugins/ipam/host-local/backend/store.go +++ b/plugins/ipam/host-local/backend/store.go @@ -24,4 +24,6 @@ type Store interface { LastReservedIP(rangeID string) (net.IP, error) ReleaseByID(id string, ifname string) error GetByID(id string, ifname string) []net.IP + HasReservedIP(podNs, podName string) (bool, net.IP) + ReservePodInfo(id string, ip net.IP, podNs, podName string, podIPIsExist bool) (bool, error) } diff --git a/plugins/ipam/host-local/backend/testing/fake_store.go b/plugins/ipam/host-local/backend/testing/fake_store.go index 954044351..834939642 100644 --- a/plugins/ipam/host-local/backend/testing/fake_store.go +++ b/plugins/ipam/host-local/backend/testing/fake_store.go @@ -89,3 +89,19 @@ func (s *FakeStore) GetByID(id string, _ string) []net.IP { func (s *FakeStore) SetIPMap(m map[string]string) { s.ipMap = m } + +func (s *FakeStore) ReleaseByPodName(_ string) error { + return nil +} + +func (s *FakeStore) HasReservedIP(_, podName string) (bool, net.IP) { + ip := net.IP{} + if podName == "" { + return false, ip + } + return false, ip +} + +func (s *FakeStore) ReservePodInfo(_ string, _ net.IP, _, _ string, _ bool) (bool, error) { + return true, nil +} diff --git a/plugins/ipam/host-local/main.go b/plugins/ipam/host-local/main.go index 0f53574ea..a6875544c 100644 --- a/plugins/ipam/host-local/main.go +++ b/plugins/ipam/host-local/main.go @@ -54,6 +54,38 @@ func cmdCheck(args *skel.CmdArgs) error { return nil } +// Args: [][2]string{ +// {"IgnoreUnknown", "1"}, +// {"K8S_POD_NAMESPACE", podNs}, +// {"K8S_POD_NAME", podName}, +// {"K8S_POD_INFRA_CONTAINER_ID", podSandboxID.ID}, +// }, +func resolvePodNsAndNameFromEnvArgs(envArgs string) (string, string, error) { + var ns, name string + if envArgs == "" { + return ns, name, nil + } + + pairs := strings.Split(envArgs, ";") + for _, pair := range pairs { + kv := strings.Split(pair, "=") + if len(kv) != 2 { + return ns, name, fmt.Errorf("ARGS: invalid pair %q", pair) + } + + if kv[0] == "K8S_POD_NAMESPACE" { + ns = kv[1] + } else if kv[0] == "K8S_POD_NAME" { + name = kv[1] + } + } + + if len(ns)+len(name) > 230 { + return "", "", fmt.Errorf("ARGS: length of pod ns and name exceed the length limit") + } + return ns, name, nil +} + func cmdAdd(args *skel.CmdArgs) error { ipamConf, confVersion, err := allocator.LoadIPAMConfig(args.StdinData, args.Args) if err != nil { @@ -101,7 +133,13 @@ func cmdAdd(args *skel.CmdArgs) error { } } - ipConf, err := allocator.Get(args.ContainerID, args.IfName, requestedIP) + // get pod namespace and pod name + podNs, podName, err := resolvePodNsAndNameFromEnvArgs(args.Args) + if err != nil { + return fmt.Errorf("failed to get pod ns/name from env args: %s", err) + } + + ipConf, err := allocator.GetByPodNsAndName(args.ContainerID, args.IfName, requestedIP, podNs, podName) if err != nil { // Deallocate all already allocated IPs for _, alloc := range allocs { diff --git a/plugins/main/windows/win-overlay/win-overlay_windows.go b/plugins/main/windows/win-overlay/win-overlay_windows.go index 2df1b74b8..74f58802a 100644 --- a/plugins/main/windows/win-overlay/win-overlay_windows.go +++ b/plugins/main/windows/win-overlay/win-overlay_windows.go @@ -106,13 +106,13 @@ func cmdHcnAdd(args *skel.CmdArgs, n *NetConf) (*current.Result, error) { return nil, errors.Annotatef(err, "error while hcn.GetNetworkByName(%s)", networkName) } if hcnNetwork == nil { - return nil, fmt.Errorf("network %v is not found", networkName) + return nil, fmt.Errorf("network %v is not found", networkName) } if hnsNetwork == nil { return nil, fmt.Errorf("network %v not found", networkName) } - if !strings.EqualFold(string (hcnNetwork.Type), "Overlay") { + if !strings.EqualFold(string(hcnNetwork.Type), "Overlay") { return nil, fmt.Errorf("network %v is of an unexpected type: %v", networkName, hcnNetwork.Type) } @@ -131,7 +131,6 @@ func cmdHcnAdd(args *skel.CmdArgs, n *NetConf) (*current.Result, error) { n.ApplyOutboundNatPolicy(hnsNetwork.Subnets[0].AddressPrefix) } hcnEndpoint, err := hns.GenerateHcnEndpoint(epInfo, &n.NetConf) - if err != nil { return nil, errors.Annotate(err, "error while generating HostComputeEndpoint") } @@ -142,15 +141,14 @@ func cmdHcnAdd(args *skel.CmdArgs, n *NetConf) (*current.Result, error) { } result, err := hns.ConstructHcnResult(hcnNetwork, hcnEndpoint) - if err != nil { ipam.ExecDel(n.IPAM.Type, args.StdinData) return nil, errors.Annotate(err, "error while constructing HostComputeEndpoint addition result") } return result, nil - } + func cmdHnsAdd(args *skel.CmdArgs, n *NetConf) (*current.Result, error) { success := false @@ -243,6 +241,7 @@ func cmdHnsAdd(args *skel.CmdArgs, n *NetConf) (*current.Result, error) { success = true return result, nil } + func cmdAdd(args *skel.CmdArgs) error { n, cniVersion, err := loadNetConf(args.StdinData) if err != nil {