Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

host-local: keep Pod IP #984

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 7 additions & 5 deletions pkg/hns/endpoint_windows.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ type EndpointInfo struct {
NetworkId string
Gateway net.IP
IpAddress net.IP
MacAddress string
MacAddress string
}

// GetSandboxContainerID returns the sandbox ID of this pod.
Expand Down Expand Up @@ -205,7 +205,8 @@ func ConstructHnsResult(hnsNetwork *hcsshim.HNSNetwork, hnsEndpoint *hcsshim.HNS
resultIPConfig := &current.IPConfig{
Address: net.IPNet{
IP: hnsEndpoint.IPAddress,
Mask: ipSubnet.Mask},
Mask: ipSubnet.Mask,
},
Gateway: net.ParseIP(hnsEndpoint.GatewayAddress),
}
result := &current.Result{
Expand Down Expand Up @@ -249,7 +250,7 @@ func GenerateHcnEndpoint(epInfo *EndpointInfo, n *NetConf) (*hcn.HostComputeEndp
Minor: 0,
},
Name: epInfo.EndpointName,
MacAddress: epInfo.MacAddress,
MacAddress: epInfo.MacAddress,
HostComputeNetwork: epInfo.NetworkId,
Dns: hcn.Dns{
Domain: epInfo.DNS.Domain,
Expand Down Expand Up @@ -289,7 +290,7 @@ func RemoveHcnEndpoint(epName string) error {
if epNamespace != nil {
err = hcn.RemoveNamespaceEndpoint(hcnEndpoint.HostComputeNamespace, hcnEndpoint.Id)
if err != nil && !hcn.IsNotFoundError(err) {
return errors.Annotatef(err,"error removing endpoint: %s from namespace", epName)
return errors.Annotatef(err, "error removing endpoint: %s from namespace", epName)
}
}

Expand Down Expand Up @@ -361,7 +362,8 @@ func ConstructHcnResult(hcnNetwork *hcn.HostComputeNetwork, hcnEndpoint *hcn.Hos
resultIPConfig := &current.IPConfig{
Address: net.IPNet{
IP: ipAddress,
Mask: ipSubnet.Mask},
Mask: ipSubnet.Mask,
},
Gateway: net.ParseIP(hcnEndpoint.Routes[0].NextHop),
}
result := &current.Result{
Expand Down
6 changes: 3 additions & 3 deletions pkg/hns/netconf_suite_windows_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
Expand All @@ -14,10 +14,10 @@
package hns

import (
"testing"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"

"testing"
)

func TestNetConf(t *testing.T) {
Expand Down
6 changes: 3 additions & 3 deletions pkg/hns/netconf_windows.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,9 +66,9 @@ var protocolEnums = map[string]uint32{
}

func (p *PortMapEntry) GetProtocolEnum() (uint32, error) {
var u, err = strconv.ParseUint(p.Protocol, 0, 10)
u, err := strconv.ParseUint(p.Protocol, 0, 10)
if err != nil {
var pe, exist = protocolEnums[strings.ToLower(p.Protocol)]
pe, exist := protocolEnums[strings.ToLower(p.Protocol)]
if !exist {
return 0, errors.New("invalid protocol supplied to port mapping policy")
}
Expand Down Expand Up @@ -323,7 +323,7 @@ func (n *NetConf) ApplyPortMappingPolicy(portMappings []PortMapEntry) {

toPolicyValue := func(p *PortMapEntry) json.RawMessage {
if n.ApiVersion == 2 {
var protocolEnum, _ = p.GetProtocolEnum()
protocolEnum, _ := p.GetProtocolEnum()
return bprintf(`{"Type": "PortMapping", "Settings": {"InternalPort": %d, "ExternalPort": %d, "Protocol": %d, "VIP": "%s"}}`, p.ContainerPort, p.HostPort, protocolEnum, p.HostIP)
}
return bprintf(`{"Type": "NAT", "InternalPort": %d, "ExternalPort": %d, "Protocol": "%s"}`, p.ContainerPort, p.HostPort, p.Protocol)
Expand Down
2 changes: 1 addition & 1 deletion pkg/hns/netconf_windows_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
Expand Down
55 changes: 53 additions & 2 deletions plugins/ipam/host-local/backend/allocator/allocator.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,11 +40,50 @@ func NewIPAllocator(s *RangeSet, store backend.Store, id int) *IPAllocator {
}
}

// Get allocates an IP
func (a *IPAllocator) Get(id string, ifname string, requestedIP net.IP) (*current.IPConfig, error) {
// GetByPodNsAndName allocates an IP or used reserved IP for specified pod
func (a *IPAllocator) GetByPodNsAndName(id string, ifname string, requestedIP net.IP, podNs, podName string) (*current.IPConfig, error) {
a.store.Lock()
defer a.store.Unlock()
if len(podName) != 0 {
podIPIsExist, knownIP := a.store.HasReservedIP(podNs, podName)

if podIPIsExist {
// podName file is exist, update ip file with new container id.
_, err := a.store.ReservePodInfo(id, knownIP, podNs, podName, podIPIsExist)
if err != nil {
return nil, err
}

reservedIP, gw := a.GetGWofKnowIP(knownIP)
if reservedIP == nil {
return nil, fmt.Errorf("no IP addresses available in range set: %s", a.rangeset.String())
}

return &current.IPConfig{
Address: *reservedIP,
Gateway: gw,
}, nil
}
// reserve ip for new pod
ipCfg, err := a.Get(id, ifname, requestedIP)
if err != nil {
return ipCfg, err
}

if ipCfg != nil {
_, err := a.store.ReservePodInfo(id, ipCfg.Address.IP, podNs, podName, podIPIsExist)
if err != nil {
return ipCfg, err
}
}
return ipCfg, nil
}

return a.Get(id, ifname, requestedIP)
}

// Get allocates an IP
func (a *IPAllocator) Get(id string, ifname string, requestedIP net.IP) (*current.IPConfig, error) {
var reservedIP *net.IPNet
var gw net.IP

Expand Down Expand Up @@ -123,6 +162,18 @@ func (a *IPAllocator) Release(id string, ifname string) error {
return a.store.ReleaseByID(id, ifname)
}

// GetGWofKnowIP returns the known IP, its mask, and its gateway
func (a *IPAllocator) GetGWofKnowIP(ip net.IP) (*net.IPNet, net.IP) {
rg := Range{}
for i, r := range *a.rangeset {
if r.Contains(ip) {
rg = (*a.rangeset)[i]
break
}
}
return &net.IPNet{IP: ip, Mask: rg.Subnet.Mask}, rg.Gateway
}

type RangeIter struct {
rangeset *RangeSet

Expand Down
115 changes: 115 additions & 0 deletions plugins/ipam/host-local/backend/disk/backend.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
package disk

import (
"fmt"
"net"
"os"
"path/filepath"
Expand Down Expand Up @@ -200,3 +201,117 @@ func GetEscapedPath(dataDir string, fname string) string {
}
return filepath.Join(dataDir, fname)
}

// HasReservedIP verify the pod already had reserved ip or not.
// and return the reserved ip on the other hand.
func (s *Store) HasReservedIP(podNs, podName string) (bool, net.IP) {
ip := net.IP{}
if len(podName) == 0 {
return false, ip
}

// Pod, ip mapping info are recorded with file name: PodIP_PodNs_PodName
podIPNsNameFileName, err := s.findPodFileName("", podNs, podName)
if err != nil {
return false, ip
}

if len(podIPNsNameFileName) != 0 {
ipStr, ns, name := resolvePodFileName(podIPNsNameFileName)
if ns == podNs && name == podName {
ip = net.ParseIP(ipStr)
if ip != nil {
return true, ip
}
}
}

return false, ip
}

// ReservePodInfo create podName file for storing ip or update ip file with container id
// in terms of podIPIsExist
func (s *Store) ReservePodInfo(id string, ip net.IP, podNs, podName string, podIPIsExist bool) (bool, error) {
if podIPIsExist {
// pod Ns/Name file is exist, update ip file with new container id.
fname := GetEscapedPath(s.dataDir, ip.String())
err := os.WriteFile(fname, []byte(strings.TrimSpace(id)), 0o644)
if err != nil {
return false, err
}
} else if len(podName) != 0 {
// for new pod, create a new file named "PodIP_PodNs_PodName",
// if there is already file named with prefix "ip_", rename the old file with new PodNs and PodName.
podIPNsNameFile := GetEscapedPath(s.dataDir, podFileName(ip.String(), podNs, podName))
podIPNsNameFileName, err := s.findPodFileName(ip.String(), "", "")
if err != nil {
return false, err
}

if len(podIPNsNameFileName) != 0 {
oldPodIPNsNameFile := GetEscapedPath(s.dataDir, podIPNsNameFileName)
err = os.Rename(oldPodIPNsNameFile, podIPNsNameFile)
if err != nil {
return false, err
}
return true, nil
}

err = os.WriteFile(podIPNsNameFile, []byte{}, 0o644)
if err != nil {
return false, err
}
}

return true, nil
}

func podFileName(ip, ns, name string) string {
if len(ip) != 0 && len(ns) != 0 {
return fmt.Sprintf("%s_%s_%s", ip, ns, name)
}

return name
}

func resolvePodFileName(fName string) (string, string, string) {
parts := strings.Split(fName, "_")
ip := ""
ns := ""
name := ""
if len(parts) == 3 {
ip = parts[0]
ns = parts[1]
name = parts[2]
}

return ip, ns, name
}

func (s *Store) findPodFileName(ip, ns, name string) (string, error) {
var pattern string

switch {
case len(ip) != 0:
pattern = fmt.Sprintf("%s_*", ip)
case len(ns) != 0 && len(name) != 0:
pattern = fmt.Sprintf("*_%s_%s", ns, name)
default:
return "", nil
}
pattern = GetEscapedPath(s.dataDir, pattern)

podFiles, err := filepath.Glob(pattern)
if err != nil {
return "", err
}

if len(podFiles) == 1 {
_, fName := filepath.Split(podFiles[0])
if strings.Count(fName, "_") == 2 {
return fName, nil
}
}

return "", nil
}
2 changes: 2 additions & 0 deletions plugins/ipam/host-local/backend/store.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,4 +24,6 @@ type Store interface {
LastReservedIP(rangeID string) (net.IP, error)
ReleaseByID(id string, ifname string) error
GetByID(id string, ifname string) []net.IP
HasReservedIP(podNs, podName string) (bool, net.IP)
ReservePodInfo(id string, ip net.IP, podNs, podName string, podIPIsExist bool) (bool, error)
}
16 changes: 16 additions & 0 deletions plugins/ipam/host-local/backend/testing/fake_store.go
Original file line number Diff line number Diff line change
Expand Up @@ -89,3 +89,19 @@ func (s *FakeStore) GetByID(id string, _ string) []net.IP {
func (s *FakeStore) SetIPMap(m map[string]string) {
s.ipMap = m
}

func (s *FakeStore) ReleaseByPodName(_ string) error {
return nil
}

func (s *FakeStore) HasReservedIP(_, podName string) (bool, net.IP) {
ip := net.IP{}
if podName == "" {
return false, ip
}
return false, ip
}

func (s *FakeStore) ReservePodInfo(_ string, _ net.IP, _, _ string, _ bool) (bool, error) {
return true, nil
}
40 changes: 39 additions & 1 deletion plugins/ipam/host-local/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,38 @@ func cmdCheck(args *skel.CmdArgs) error {
return nil
}

// Args: [][2]string{
// {"IgnoreUnknown", "1"},
// {"K8S_POD_NAMESPACE", podNs},
// {"K8S_POD_NAME", podName},
// {"K8S_POD_INFRA_CONTAINER_ID", podSandboxID.ID},
// },
func resolvePodNsAndNameFromEnvArgs(envArgs string) (string, string, error) {
var ns, name string
if envArgs == "" {
return ns, name, nil
}

pairs := strings.Split(envArgs, ";")
for _, pair := range pairs {
kv := strings.Split(pair, "=")
if len(kv) != 2 {
return ns, name, fmt.Errorf("ARGS: invalid pair %q", pair)
}

if kv[0] == "K8S_POD_NAMESPACE" {
ns = kv[1]
} else if kv[0] == "K8S_POD_NAME" {
name = kv[1]
}
}

if len(ns)+len(name) > 230 {
return "", "", fmt.Errorf("ARGS: length of pod ns and name exceed the length limit")
}
return ns, name, nil
}

func cmdAdd(args *skel.CmdArgs) error {
ipamConf, confVersion, err := allocator.LoadIPAMConfig(args.StdinData, args.Args)
if err != nil {
Expand Down Expand Up @@ -101,7 +133,13 @@ func cmdAdd(args *skel.CmdArgs) error {
}
}

ipConf, err := allocator.Get(args.ContainerID, args.IfName, requestedIP)
// get pod namespace and pod name
podNs, podName, err := resolvePodNsAndNameFromEnvArgs(args.Args)
if err != nil {
return fmt.Errorf("failed to get pod ns/name from env args: %s", err)
}

ipConf, err := allocator.GetByPodNsAndName(args.ContainerID, args.IfName, requestedIP, podNs, podName)
if err != nil {
// Deallocate all already allocated IPs
for _, alloc := range allocs {
Expand Down
Loading