From 94256afc067665d14f93f96fd5aa02558389d461 Mon Sep 17 00:00:00 2001 From: Alejandro Pedraza Date: Thu, 28 Mar 2024 10:11:16 -0500 Subject: [PATCH] Add IPv6/dual-stack support (#350) ## Flags Changes This adds the proxy-init flag `--iptables-mode` (with possible values `legacy` and `nft`), which supersedes `--firewal-bin-path` and `firewall-save-bin-path` (which still remain supported). Also the `--ipv6` flag has been added (default `true`). After the set of rules run via iptables are processed, if `--ipv6` is true (which is the default), the same set of rules will be run via ip6tables. Analog changes were applied to linkerd-cni as well. ## Backwards-Compatibility This is backwards-compatible with older control planes and upcoming control planes. If `--ipv6` is not passed (and thus defaults to true), this doesn't impact operation even if the cluster doesn't support IPv6; the ip6tables rules are applied but they're innocuous. OTOH if there's no kernel support for IPv6 (which is the case for github runners*) then the ip6tables command will fail but we'll just log the failure and not fail the linkerd-init container (nor the `add` command for linkerd-cni). This avoids having to explicitly set `--ipv6=false`, but it can be set if the user is aware of such limitations and wants to get rid of the errors. ## Testing Improvements The cni-plugin-integration workflow has been simplified by using a matrix strategy, and enhanced by parameterizing the iptables-mode config. ## Linkerd IPv6 Support This allows routing IPv6 traffic to the proxy, but is just the first step towards IPv6/dual-stack support. Control plane and proxy changes will come up next. ## (*) Github Runners IPv6 Support Even though `modinfo` signals support for IPv6, `ip6tables` commands throw modprobe errors. Indeed, according to actions/runner-images#668 support is not there yet. Also, according to actions/runner#3138 there are issues with hosted runners as well, but that might not affect us if we still expose an IPv4 interface to interact with github. Something to take into account when we get to IPv6 integration testing. --- .github/workflows/cni-plugin-integration.yml | 29 ++---- .../manifests/calico/linkerd-cni.yaml | 4 +- .../manifests/cilium/linkerd-cni.yaml | 4 +- .../manifests/flannel/linkerd-cni.yaml | 4 +- cni-plugin/integration/run.sh | 8 +- cni-plugin/integration/testutil/test_util.go | 3 + cni-plugin/main.go | 45 +++++++-- justfile | 1 + proxy-init/cmd/root.go | 97 +++++++++++++++++-- proxy-init/cmd/root_test.go | 7 ++ 10 files changed, 157 insertions(+), 45 deletions(-) diff --git a/.github/workflows/cni-plugin-integration.yml b/.github/workflows/cni-plugin-integration.yml index 4e80b8e4..d08c7826 100644 --- a/.github/workflows/cni-plugin-integration.yml +++ b/.github/workflows/cni-plugin-integration.yml @@ -12,33 +12,20 @@ on: - justfile* jobs: - cni-flannel-test: - continue-on-error: true - timeout-minutes: 15 - runs-on: ubuntu-latest - steps: - - uses: linkerd/dev/actions/setup-tools@v43 - - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 - - name: Run CNI integration tests - run: just cni-plugin-test-integration-flannel - cni-calico-test: - continue-on-error: true - timeout-minutes: 15 - runs-on: ubuntu-latest - steps: - - uses: linkerd/dev/actions/setup-tools@v43 - - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 - - name: Run CNI integration tests - run: just cni-plugin-test-integration-calico - cni-cilium-test: - continue-on-error: true + cni-test: + strategy: + matrix: + cni: [flannel, calico, cilium] + iptables-mode: [legacy, nft] timeout-minutes: 15 runs-on: ubuntu-latest steps: - uses: linkerd/dev/actions/setup-tools@v43 - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 - name: Run CNI integration tests - run: just cni-plugin-test-integration-cilium + env: + IPTABLES_MODE: ${{ matrix.iptables-mode }} + run: just cni-plugin-test-integration-${{ matrix.cni }} ordering-test: continue-on-error: true timeout-minutes: 15 diff --git a/cni-plugin/integration/manifests/calico/linkerd-cni.yaml b/cni-plugin/integration/manifests/calico/linkerd-cni.yaml index 05f110bd..ec009421 100644 --- a/cni-plugin/integration/manifests/calico/linkerd-cni.yaml +++ b/cni-plugin/integration/manifests/calico/linkerd-cni.yaml @@ -80,7 +80,9 @@ data: "ports-to-redirect": [], "inbound-ports-to-ignore": ["4191","4190"], "simulate": false, - "use-wait-flag": false + "use-wait-flag": false, + "iptables-mode": "$IPTABLES_MODE", + "ipv6": true } } --- diff --git a/cni-plugin/integration/manifests/cilium/linkerd-cni.yaml b/cni-plugin/integration/manifests/cilium/linkerd-cni.yaml index 05f110bd..ec009421 100644 --- a/cni-plugin/integration/manifests/cilium/linkerd-cni.yaml +++ b/cni-plugin/integration/manifests/cilium/linkerd-cni.yaml @@ -80,7 +80,9 @@ data: "ports-to-redirect": [], "inbound-ports-to-ignore": ["4191","4190"], "simulate": false, - "use-wait-flag": false + "use-wait-flag": false, + "iptables-mode": "$IPTABLES_MODE", + "ipv6": true } } --- diff --git a/cni-plugin/integration/manifests/flannel/linkerd-cni.yaml b/cni-plugin/integration/manifests/flannel/linkerd-cni.yaml index 5b9166ad..3774ca7a 100644 --- a/cni-plugin/integration/manifests/flannel/linkerd-cni.yaml +++ b/cni-plugin/integration/manifests/flannel/linkerd-cni.yaml @@ -84,7 +84,9 @@ data: "ports-to-redirect": [], "inbound-ports-to-ignore": ["4191","4190"], "simulate": false, - "use-wait-flag": false + "use-wait-flag": false, + "iptables-mode": "$IPTABLES_MODE", + "ipv6": true } } --- diff --git a/cni-plugin/integration/run.sh b/cni-plugin/integration/run.sh index 29414462..3b51339a 100755 --- a/cni-plugin/integration/run.sh +++ b/cni-plugin/integration/run.sh @@ -4,9 +4,8 @@ set -euxo pipefail cd "${BASH_SOURCE[0]%/*}" -# Integration tests to run. Scenario is passed in as an environment variable. -# Default is 'flannel' SCENARIO=${CNI_TEST_SCENARIO:-flannel} +IPTABLES_MODE=${IPTABLES_MODE:-legacy} # Run kubectl with the correct context. function k() { @@ -25,7 +24,10 @@ function create_test_lab() { # can enable a testing matrix? # Apply all files in scenario directory. For non-flannel CNIs, this will # include the CNI manifest itself. - k apply -f "manifests/$SCENARIO/" + for f in ./manifests/"$SCENARIO"/*.yaml + do + envsubst < "$f" | k apply -f - + done } function cleanup() { diff --git a/cni-plugin/integration/testutil/test_util.go b/cni-plugin/integration/testutil/test_util.go index 3d2429e9..1691e11d 100644 --- a/cni-plugin/integration/testutil/test_util.go +++ b/cni-plugin/integration/testutil/test_util.go @@ -49,8 +49,11 @@ type ProxyInit struct { PortsToRedirect []int `json:"ports-to-redirect"` InboundPortsToIgnore []string `json:"inbound-ports-to-ignore"` OutboundPortsToIgnore []string `json:"outbound-ports-to-ignore"` + SubnetsToIgnore []string `json:"subnets-to-ignore"` Simulate bool `json:"simulate"` UseWaitFlag bool `json:"use-wait-flag"` + IPTablesMode string `json:"iptables-mode"` + IPv6 bool `json:"ipv6"` } // LinkerdPlugin is what we use for CNI configuration in the plugins section diff --git a/cni-plugin/main.go b/cni-plugin/main.go index 9eb68980..ab441e1d 100644 --- a/cni-plugin/main.go +++ b/cni-plugin/main.go @@ -52,6 +52,8 @@ type ProxyInit struct { SubnetsToIgnore []string `json:"subnets-to-ignore"` Simulate bool `json:"simulate"` UseWaitFlag bool `json:"use-wait-flag"` + IPTablesMode string `json:"iptables-mode"` + IPv6 bool `json:"ipv6"` } // Kubernetes a K8s specific struct to hold config @@ -219,8 +221,8 @@ func cmdAdd(args *skel.CmdArgs) error { SimulateOnly: conf.ProxyInit.Simulate, NetNs: args.Netns, UseWaitFlag: conf.ProxyInit.UseWaitFlag, - FirewallBinPath: "iptables-legacy", - FirewallSaveBinPath: "iptables-legacy-save", + IPTablesMode: conf.ProxyInit.IPTablesMode, + IPv6: conf.ProxyInit.IPv6, } // Check if there are any overridden ports to be skipped @@ -292,17 +294,24 @@ func cmdAdd(args *skel.CmdArgs) error { options.OutboundPortsToIgnore = append(options.OutboundPortsToIgnore, skippedPorts...) } - firewallConfiguration, err := cmd.BuildFirewallConfiguration(&options) - if err != nil { - logEntry.Errorf("linkerd-cni: could not create a Firewall Configuration from the options: %v", options) - return err + // This ensures BC against linkerd2-cni older versions not yet passing this flag + if options.IPTablesMode == "" { + options.IPTablesMode = cmd.IPTablesModeLegacy } - err = iptables.ConfigureFirewall(*firewallConfiguration) - if err != nil { - logEntry.Errorf("linkerd-cni: could not configure firewall: %s", err) + // always trigger the IPv4 rules + optIPv4 := options + optIPv4.IPv6 = false + if err := buildAndConfigure(logEntry, &optIPv4); err != nil { return err } + + // trigger the IPv6 rules + if options.IPv6 { + if err := buildAndConfigure(logEntry, &options); err != nil { + return err + } + } } else { if containsInitContainer { logEntry.Debug("linkerd-cni: linkerd-init initContainer is present, skipping.") @@ -353,6 +362,24 @@ func getAPIServerPorts(ctx context.Context, api *kubernetes.Clientset) ([]string return ports, nil } +func buildAndConfigure(logEntry *logrus.Entry, options *cmd.RootOptions) error { + firewallConfiguration, err := cmd.BuildFirewallConfiguration(options) + if err != nil { + logEntry.Errorf("linkerd-cni: could not create a Firewall Configuration from the options: %v", options) + return err + } + + err = iptables.ConfigureFirewall(*firewallConfiguration) + // We couldn't find a robust way of checking IPv6 support besides trying to just call ip6tables-save. + // If IPv4 rules worked but not IPv6, let's not fail the container (the actual problem will get logged). + if !options.IPv6 && err != nil { + logEntry.Errorf("linkerd-cni: could not configure firewall: %s", err) + return err + } + + return nil +} + func getAnnotationOverride(ctx context.Context, api *kubernetes.Clientset, pod *v1.Pod, key string) (string, error) { // Check if the annotation is present on the pod if override := pod.GetObjectMeta().GetAnnotations()[key]; override != "" { diff --git a/justfile b/justfile index ba3cb506..b279fd34 100644 --- a/justfile +++ b/justfile @@ -224,6 +224,7 @@ _cni-plugin-setup-cilium: echo "Mounted /sys/fs/bpf to cilium-test-server cluster" helm repo add cilium https://helm.cilium.io/ helm install cilium cilium/cilium --version 1.13.0 \ + --kube-context k3d-l5d-cilium-test \ --namespace kube-system \ --set kubeProxyReplacement=partial \ --set hostServices.enabled=false \ diff --git a/proxy-init/cmd/root.go b/proxy-init/cmd/root.go index 3c8fa8f7..6df5852b 100644 --- a/proxy-init/cmd/root.go +++ b/proxy-init/cmd/root.go @@ -13,6 +13,22 @@ import ( "github.com/linkerd/linkerd2-proxy-init/internal/util" ) +const ( + // IPTablesModeLegacy signals the usage of the iptables-legacy commands + IPTablesModeLegacy = "legacy" + // IPTablesModeNFT signals the usage of the iptables-nft commands + IPTablesModeNFT = "nft" + + cmdLegacy = "iptables-legacy" + cmdLegacySave = "iptables-legacy-save" + cmdLegacyIPv6 = "ip6tables-legacy" + cmdLegacyIPv6Save = "ip6tables-legacy-save" + cmdNFT = "iptables-nft" + cmdNFTSave = "iptables-nft-save" + cmdNFTIPv6 = "ip6tables-nft" + cmdNFTIPv6Save = "ip6tables-nft-save" +) + // RootOptions provides the information that will be used to build a firewall configuration. type RootOptions struct { IncomingProxyPort int @@ -30,6 +46,8 @@ type RootOptions struct { LogLevel string FirewallBinPath string FirewallSaveBinPath string + IPTablesMode string + IPv6 bool } func newRootOptions() *RootOptions { @@ -47,8 +65,10 @@ func newRootOptions() *RootOptions { TimeoutCloseWaitSecs: 0, LogFormat: "plain", LogLevel: "info", - FirewallBinPath: "iptables-legacy", - FirewallSaveBinPath: "iptables-legacy-save", + FirewallBinPath: "", + FirewallSaveBinPath: "", + IPTablesMode: "", + IPv6: true, } } @@ -61,7 +81,7 @@ func NewRootCmd() *cobra.Command { Use: "proxy-init", Short: "proxy-init adds a Kubernetes pod to the Linkerd service mesh", Long: "proxy-init adds a Kubernetes pod to the Linkerd service mesh.", - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(_ *cobra.Command, _ []string) error { if options.TimeoutCloseWaitSecs != 0 { sysctl := exec.Command("sysctl", "-w", @@ -75,16 +95,39 @@ func NewRootCmd() *cobra.Command { log.Info(string(out)) } - config, err := BuildFirewallConfiguration(options) + log.SetFormatter(getFormatter(options.LogFormat)) + err := setLogLevel(options.LogLevel) if err != nil { return err } - log.SetFormatter(getFormatter(options.LogFormat)) - err = setLogLevel(options.LogLevel) + + // always trigger the IPv4 rules + optIPv4 := *options + optIPv4.IPv6 = false + config, err := BuildFirewallConfiguration(&optIPv4) if err != nil { return err } - return iptables.ConfigureFirewall(*config) + + if err = iptables.ConfigureFirewall(*config); err != nil { + return err + } + + if !options.IPv6 { + return nil + } + + // trigger the IPv6 rules + config, err = BuildFirewallConfiguration(options) + if err != nil { + return err + } + + // We couldn't find a robust way of checking IPv6 support besides trying to just call ip6tables-save. + // If IPv4 rules worked but not IPv6, let's not fail the container (the actual problem will get logged). + _ = iptables.ConfigureFirewall(*config) + + return nil }, } @@ -101,6 +144,10 @@ func NewRootCmd() *cobra.Command { cmd.PersistentFlags().IntVar(&options.TimeoutCloseWaitSecs, "timeout-close-wait-secs", options.TimeoutCloseWaitSecs, "Sets nf_conntrack_tcp_timeout_close_wait") cmd.PersistentFlags().StringVar(&options.LogFormat, "log-format", options.LogFormat, "Configure log format ('plain' or 'json')") cmd.PersistentFlags().StringVar(&options.LogLevel, "log-level", options.LogLevel, "Configure log level") + cmd.PersistentFlags().StringVar(&options.IPTablesMode, "iptables-mode", options.IPTablesMode, "Variant of iptables command to use (\"legacy\" or \"nft\"); overrides --firewall-bin-path and --firewall-save-bin-path") + cmd.PersistentFlags().BoolVar(&options.IPv6, "ipv6", options.IPv6, "Set rules both via iptables and ip6tables to support dual-stack networking") + + // these two flags are kept for backwards-compatibility, but --iptables-mode is preferred cmd.PersistentFlags().StringVar(&options.FirewallBinPath, "firewall-bin-path", options.FirewallBinPath, "Path to iptables binary") cmd.PersistentFlags().StringVar(&options.FirewallSaveBinPath, "firewall-save-bin-path", options.FirewallSaveBinPath, "Path to iptables-save binary") return cmd @@ -108,6 +155,21 @@ func NewRootCmd() *cobra.Command { // BuildFirewallConfiguration returns an iptables FirewallConfiguration suitable to use to configure iptables. func BuildFirewallConfiguration(options *RootOptions) (*iptables.FirewallConfiguration, error) { + if options.IPTablesMode != "" && options.IPTablesMode != IPTablesModeLegacy && options.IPTablesMode != IPTablesModeNFT { + return nil, fmt.Errorf("--iptables-mode valid values are only \"%s\" and \"%s\"", IPTablesModeLegacy, IPTablesModeNFT) + } + + if options.IPTablesMode == "" { + switch options.FirewallBinPath { + case "", cmdLegacy: + options.IPTablesMode = IPTablesModeLegacy + case cmdNFT: + options.IPTablesMode = IPTablesModeNFT + default: + return nil, fmt.Errorf("--firewall-bin-path valid values are only \"%s\" and \"%s\"", cmdLegacy, cmdNFT) + } + } + if !util.IsValidPort(options.IncomingProxyPort) { return nil, fmt.Errorf("--incoming-proxy-port must be a valid TCP port number") } @@ -116,6 +178,8 @@ func BuildFirewallConfiguration(options *RootOptions) (*iptables.FirewallConfigu return nil, fmt.Errorf("--outgoing-proxy-port must be a valid TCP port number") } + cmd, cmdSave := getCommands(options) + sanitizedSubnets := []string{} for _, subnet := range options.SubnetsToIgnore { subnet := strings.TrimSpace(subnet) @@ -138,8 +202,8 @@ func BuildFirewallConfiguration(options *RootOptions) (*iptables.FirewallConfigu SimulateOnly: options.SimulateOnly, NetNs: options.NetNs, UseWaitFlag: options.UseWaitFlag, - BinPath: options.FirewallBinPath, - SaveBinPath: options.FirewallSaveBinPath, + BinPath: cmd, + SaveBinPath: cmdSave, } if len(options.PortsToRedirect) > 0 { @@ -160,6 +224,21 @@ func getFormatter(format string) log.Formatter { } } +func getCommands(options *RootOptions) (string, string) { + if options.IPTablesMode == IPTablesModeLegacy { + if options.IPv6 { + return cmdLegacyIPv6, cmdLegacyIPv6Save + } + return cmdLegacy, cmdLegacySave + } + + if options.IPv6 { + return cmdNFTIPv6, cmdNFTIPv6Save + } + + return cmdNFT, cmdNFTSave +} + func setLogLevel(logLevel string) error { level, err := log.ParseLevel(logLevel) if err != nil { diff --git a/proxy-init/cmd/root_test.go b/proxy-init/cmd/root_test.go index d1b31b28..5a2ca97b 100644 --- a/proxy-init/cmd/root_test.go +++ b/proxy-init/cmd/root_test.go @@ -31,6 +31,7 @@ func TestBuildFirewallConfiguration(t *testing.T) { options.IncomingProxyPort = expectedIncomingProxyPort options.OutgoingProxyPort = expectedOutgoingProxyPort options.ProxyUserID = expectedProxyUserID + options.IPv6 = false config, err := BuildFirewallConfiguration(options) if err != nil { @@ -51,6 +52,7 @@ func TestBuildFirewallConfiguration(t *testing.T) { options: &RootOptions{ IncomingProxyPort: -1, OutgoingProxyPort: 1234, + IPTablesMode: IPTablesModeLegacy, }, errorMessage: "--incoming-proxy-port must be a valid TCP port number", }, @@ -58,6 +60,7 @@ func TestBuildFirewallConfiguration(t *testing.T) { options: &RootOptions{ IncomingProxyPort: 100000, OutgoingProxyPort: 1234, + IPTablesMode: IPTablesModeLegacy, }, errorMessage: "--incoming-proxy-port must be a valid TCP port number", }, @@ -65,6 +68,7 @@ func TestBuildFirewallConfiguration(t *testing.T) { options: &RootOptions{ IncomingProxyPort: 1234, OutgoingProxyPort: -1, + IPTablesMode: IPTablesModeLegacy, }, errorMessage: "--outgoing-proxy-port must be a valid TCP port number", }, @@ -72,12 +76,14 @@ func TestBuildFirewallConfiguration(t *testing.T) { options: &RootOptions{ IncomingProxyPort: 1234, OutgoingProxyPort: 100000, + IPTablesMode: IPTablesModeLegacy, }, errorMessage: "--outgoing-proxy-port must be a valid TCP port number", }, { options: &RootOptions{ SubnetsToIgnore: []string{"1.1.1.1/24", "0.0.0.0"}, + IPTablesMode: IPTablesModeLegacy, }, errorMessage: "0.0.0.0 is not a valid CIDR address", }, @@ -102,6 +108,7 @@ func TestBuildFirewallConfiguration(t *testing.T) { // Tests that subnets are parsed properly and trimmed of excess whitespace options: &RootOptions{ SubnetsToIgnore: []string{"1.1.1.1/24 "}, + IPTablesMode: IPTablesModeLegacy, }, errorMessage: "", },