nftables support for ipmasq and portmap

go.mod

@@ -21,6 +21,7 @@ require (
 	github.com/safchain/ethtool v0.4.1
 	github.com/vishvananda/netlink v1.3.0
 	golang.org/x/sys v0.23.0
+	sigs.k8s.io/knftables v0.0.17
 )
 
 require (

go.sum

@@ -68,6 +68,8 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN
 github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 h1:FKHo8hFI3A+7w0aUQuYXQ+6EN5stWmeY/AZqtM8xk9k=
 github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY=
+github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc=
 github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk=
 github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
 github.com/networkplumbing/go-nft v0.4.0 h1:kExVMwXW48DOAukkBwyI16h4uhE5lN9iMvQd52lpTyU=
@@ -194,3 +196,5 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+sigs.k8s.io/knftables v0.0.17 h1:wGchTyRF/iGTIjd+vRaR1m676HM7jB8soFtyr/148ic=
+sigs.k8s.io/knftables v0.0.17/go.mod h1:f/5ZLKYEUPUhVjUCg6l80ACdL7CIIyeL0DxfgojGRTk=

pkg/ip/ipmasq_iptables_linux.go

@@ -0,0 +1,161 @@
+// Copyright 2015 CNI authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ip
+
+import (
+	"fmt"
+	"net"
+
+	"github.com/coreos/go-iptables/iptables"
+
+	"github.com/containernetworking/cni/pkg/types"
+	"github.com/containernetworking/plugins/pkg/utils"
+)
+
+// setupIPMasqIPTables is the iptables-based implementation of SetupIPMasqForNetwork
+func setupIPMasqIPTables(ipn *net.IPNet, network, _, containerID string) error {
+	// Note: for historical reasons, the iptables implementation ignores ifname.
+	chain := utils.FormatChainName(network, containerID)
+	comment := utils.FormatComment(network, containerID)
+	return SetupIPMasq(ipn, chain, comment)
+}
+
+// SetupIPMasq installs iptables rules to masquerade traffic
+// coming from ip of ipn and going outside of ipn.
+// Deprecated: This function only supports iptables. Use SetupIPMasqForNetwork, which
+// supports both iptables and nftables.
+func SetupIPMasq(ipn *net.IPNet, chain string, comment string) error {
+	isV6 := ipn.IP.To4() == nil
+
+	var ipt *iptables.IPTables
+	var err error
+	var multicastNet string
+
+	if isV6 {
+		ipt, err = iptables.NewWithProtocol(iptables.ProtocolIPv6)
+		multicastNet = "ff00::/8"
+	} else {
+		ipt, err = iptables.NewWithProtocol(iptables.ProtocolIPv4)
+		multicastNet = "224.0.0.0/4"
+	}
+	if err != nil {
+		return fmt.Errorf("failed to locate iptables: %v", err)
+	}
+
+	// Create chain if doesn't exist
+	exists := false
+	chains, err := ipt.ListChains("nat")
+	if err != nil {
+		return fmt.Errorf("failed to list chains: %v", err)
+	}
+	for _, ch := range chains {
+		if ch == chain {
+			exists = true
+			break
+		}
+	}
+	if !exists {
+		if err = ipt.NewChain("nat", chain); err != nil {
+			return err
+		}
+	}
+
+	// Packets to this network should not be touched
+	if err := ipt.AppendUnique("nat", chain, "-d", ipn.String(), "-j", "ACCEPT", "-m", "comment", "--comment", comment); err != nil {
+		return err
+	}
+
+	// Don't masquerade multicast - pods should be able to talk to other pods
+	// on the local network via multicast.
+	if err := ipt.AppendUnique("nat", chain, "!", "-d", multicastNet, "-j", "MASQUERADE", "-m", "comment", "--comment", comment); err != nil {
+		return err
+	}
+
+	// Packets from the specific IP of this network will hit the chain
+	return ipt.AppendUnique("nat", "POSTROUTING", "-s", ipn.IP.String(), "-j", chain, "-m", "comment", "--comment", comment)
+}
+
+// teardownIPMasqIPTables is the iptables-based implementation of TeardownIPMasqForNetwork
+func teardownIPMasqIPTables(ipn *net.IPNet, network, _, containerID string) error {
+	// Note: for historical reasons, the iptables implementation ignores ifname.
+	chain := utils.FormatChainName(network, containerID)
+	comment := utils.FormatComment(network, containerID)
+	return TeardownIPMasq(ipn, chain, comment)
+}
+
+// TeardownIPMasq undoes the effects of SetupIPMasq.
+// Deprecated: This function only supports iptables. Use TeardownIPMasqForNetwork, which
+// supports both iptables and nftables.
+func TeardownIPMasq(ipn *net.IPNet, chain string, comment string) error {
+	isV6 := ipn.IP.To4() == nil
+
+	var ipt *iptables.IPTables
+	var err error
+
+	if isV6 {
+		ipt, err = iptables.NewWithProtocol(iptables.ProtocolIPv6)
+	} else {
+		ipt, err = iptables.NewWithProtocol(iptables.ProtocolIPv4)
+	}
+	if err != nil {
+		return fmt.Errorf("failed to locate iptables: %v", err)
+	}
+
+	err = ipt.Delete("nat", "POSTROUTING", "-s", ipn.IP.String(), "-j", chain, "-m", "comment", "--comment", comment)
+	if err != nil && !isNotExist(err) {
+		return err
+	}
+
+	// for downward compatibility
+	err = ipt.Delete("nat", "POSTROUTING", "-s", ipn.String(), "-j", chain, "-m", "comment", "--comment", comment)
+	if err != nil && !isNotExist(err) {
+		return err
+	}
+
+	err = ipt.ClearChain("nat", chain)
+	if err != nil && !isNotExist(err) {
+		return err
+	}
+
+	err = ipt.DeleteChain("nat", chain)
+	if err != nil && !isNotExist(err) {
+		return err
+	}
+
+	return nil
+}
+
+// gcIPMasqIPTables is the iptables-based implementation of GCIPMasqForNetwork
+func gcIPMasqIPTables(_ string, _ []types.GCAttachment) error {
+	// FIXME: The iptables implementation does not support GC.
+	//
+	// (In theory, it _could_ backward-compatibly support it, by adding a no-op rule
+	// with a comment indicating the network to each chain it creates, so that it
+	// could later figure out which chains corresponded to which networks; older
+	// implementations would ignore the extra rule but would still correctly delete
+	// the chain on teardown (because they ClearChain() before doing DeleteChain()).
+
+	return nil
+}
+
+// isNotExist returnst true if the error is from iptables indicating
+// that the target does not exist.
+func isNotExist(err error) bool {
+	e, ok := err.(*iptables.Error)
+	if !ok {
+		return false
+	}
+	return e.IsNotExist()
+}

pkg/ip/ipmasq_linux.go

@@ -15,111 +15,78 @@
 package ip
 
 import (
+	"errors"
 	"fmt"
 	"net"
+	"strings"
 
-	"github.com/coreos/go-iptables/iptables"
+	"github.com/containernetworking/cni/pkg/types"
+	"github.com/containernetworking/plugins/pkg/utils"
 )
 
-// SetupIPMasq installs iptables rules to masquerade traffic
-// coming from ip of ipn and going outside of ipn
-func SetupIPMasq(ipn *net.IPNet, chain string, comment string) error {
-	isV6 := ipn.IP.To4() == nil
-
-	var ipt *iptables.IPTables
-	var err error
-	var multicastNet string
-
-	if isV6 {
-		ipt, err = iptables.NewWithProtocol(iptables.ProtocolIPv6)
-		multicastNet = "ff00::/8"
-	} else {
-		ipt, err = iptables.NewWithProtocol(iptables.ProtocolIPv4)
-		multicastNet = "224.0.0.0/4"
-	}
-	if err != nil {
-		return fmt.Errorf("failed to locate iptables: %v", err)
-	}
-
-	// Create chain if doesn't exist
-	exists := false
-	chains, err := ipt.ListChains("nat")
-	if err != nil {
-		return fmt.Errorf("failed to list chains: %v", err)
-	}
-	for _, ch := range chains {
-		if ch == chain {
-			exists = true
-			break
-		}
-	}
-	if !exists {
-		if err = ipt.NewChain("nat", chain); err != nil {
-			return err
+// SetupIPMasqForNetwork installs rules to masquerade traffic coming from ip of ipn and
+// going outside of ipn, using a chain name based on network, ifname, and containerID. The
+// backend can be either "iptables" or "nftables"; if it is nil, then a suitable default
+// implementation will be used.
+func SetupIPMasqForNetwork(backend *string, ipn *net.IPNet, network, ifname, containerID string) error {
+	if backend == nil {
+		// Prefer iptables, unless only nftables is available
+		defaultBackend := "iptables"
+		if !utils.SupportsIPTables() && utils.SupportsNFTables() {
+			defaultBackend = "nftables"
 		}
+		backend = &defaultBackend
 	}
 
-	// Packets to this network should not be touched
-	if err := ipt.AppendUnique("nat", chain, "-d", ipn.String(), "-j", "ACCEPT", "-m", "comment", "--comment", comment); err != nil {
-		return err
+	switch *backend {
+	case "iptables":
+		return setupIPMasqIPTables(ipn, network, ifname, containerID)
+	case "nftables":
+		return setupIPMasqNFTables(ipn, network, ifname, containerID)
+	default:
+		return fmt.Errorf("unknown ipmasq backend %q", *backend)
 	}
-
-	// Don't masquerade multicast - pods should be able to talk to other pods
-	// on the local network via multicast.
-	if err := ipt.AppendUnique("nat", chain, "!", "-d", multicastNet, "-j", "MASQUERADE", "-m", "comment", "--comment", comment); err != nil {
-		return err
-	}
-
-	// Packets from the specific IP of this network will hit the chain
-	return ipt.AppendUnique("nat", "POSTROUTING", "-s", ipn.IP.String(), "-j", chain, "-m", "comment", "--comment", comment)
 }
 
-// TeardownIPMasq undoes the effects of SetupIPMasq
-func TeardownIPMasq(ipn *net.IPNet, chain string, comment string) error {
-	isV6 := ipn.IP.To4() == nil
+// TeardownIPMasqForNetwork undoes the effects of SetupIPMasqForNetwork
+func TeardownIPMasqForNetwork(ipn *net.IPNet, network, ifname, containerID string) error {
+	var errs []string
 
-	var ipt *iptables.IPTables
-	var err error
+	// Do both the iptables and the nftables cleanup, since the pod may have been
+	// created with a different version of this plugin or a different configuration.
 
-	if isV6 {
-		ipt, err = iptables.NewWithProtocol(iptables.ProtocolIPv6)
-	} else {
-		ipt, err = iptables.NewWithProtocol(iptables.ProtocolIPv4)
-	}
-	if err != nil {
-		return fmt.Errorf("failed to locate iptables: %v", err)
+	err := teardownIPMasqIPTables(ipn, network, ifname, containerID)
+	if err != nil && utils.SupportsIPTables() {
+		errs = append(errs, err.Error())
 	}
 
-	err = ipt.Delete("nat", "POSTROUTING", "-s", ipn.IP.String(), "-j", chain, "-m", "comment", "--comment", comment)
-	if err != nil && !isNotExist(err) {
-		return err
+	err = teardownIPMasqNFTables(ipn, network, ifname, containerID)
+	if err != nil && utils.SupportsNFTables() {
+		errs = append(errs, err.Error())
 	}
 
-	// for downward compatibility
-	err = ipt.Delete("nat", "POSTROUTING", "-s", ipn.String(), "-j", chain, "-m", "comment", "--comment", comment)
-	if err != nil && !isNotExist(err) {
-		return err
+	if errs == nil {
+		return nil
 	}
+	return errors.New(strings.Join(errs, "\n"))
+}
 
-	err = ipt.ClearChain("nat", chain)
-	if err != nil && !isNotExist(err) {
-		return err
-	}
+// GCIPMasqForNetwork garbage collects stale IPMasq entries for network
+func GCIPMasqForNetwork(network string, attachments []types.GCAttachment) error {
+	var errs []string
 
-	err = ipt.DeleteChain("nat", chain)
-	if err != nil && !isNotExist(err) {
-		return err
+	err := gcIPMasqIPTables(network, attachments)
+	if err != nil && utils.SupportsIPTables() {
+		errs = append(errs, err.Error())
 	}
 
-	return nil
-}
+	err = gcIPMasqNFTables(network, attachments)
+	if err != nil && utils.SupportsNFTables() {
+		errs = append(errs, err.Error())
+	}
 
-// isNotExist returnst true if the error is from iptables indicating
-// that the target does not exist.
-func isNotExist(err error) bool {
-	e, ok := err.(*iptables.Error)
-	if !ok {
-		return false
+	if errs == nil {
+		return nil
 	}
-	return e.IsNotExist()
+	return errors.New(strings.Join(errs, "\n"))
 }