From 32f70cef8006227836a8f5a9edf24108809d1beb Mon Sep 17 00:00:00 2001
From: Rob Brockbank <rob@tigera.io>
Date: Wed, 23 May 2018 11:00:05 -0700
Subject: [PATCH] Add environment option to disable on-node SNAT

---
 pkg/networkutils/network.go | 57 ++++++++++++++++++++++++++++++-------
 1 file changed, 47 insertions(+), 10 deletions(-)

diff --git a/pkg/networkutils/network.go b/pkg/networkutils/network.go
index 17bb33e0404..09680c29a16 100644
--- a/pkg/networkutils/network.go
+++ b/pkg/networkutils/network.go
@@ -15,6 +15,8 @@ package networkutils
 
 import (
 	"net"
+	"os"
+	"strconv"
 	"strings"
 	"syscall"
 
@@ -42,6 +44,11 @@ const (
 	fromPodRulePriority = 1536
 
 	mainRoutingTable = 254
+
+	// This environment is used to specify whether an external NAT gateway will be used to provide SNAT of
+	// secondary ENI IP addresses.  If set to "true", the SNAT iptables rule and off-VPC ip rule will not
+	// be installed and will be removed if they are already installed.
+	envExternalSNAT = "AWS_VPC_K8S_CNI_EXTERNALSNAT"
 )
 
 // NetworkAPIs defines the host level and the eni level network related operations
@@ -67,28 +74,36 @@ func isDuplicateRuleAdd(err error) bool {
 	return strings.Contains(err.Error(), "File exists")
 }
 
-// SetupNodeNetwork performs node level network configuration
+// SetupHostNetwork performs node level network configuration
 // TODO : implement ip rule not to 10.0.0.0/16(vpc'subnet) table main priority  1024
 func (os *linuxNetwork) SetupHostNetwork(vpcCIDR *net.IPNet, primaryAddr *net.IP) error {
 
+	externalSNAT, err := useExternalSNAT()
+	if err != nil {
+		return err
+	}
+
 	hostRule := os.netLink.NewRule()
 	hostRule.Dst = vpcCIDR
 	hostRule.Table = mainRoutingTable
 	hostRule.Priority = hostRulePriority
 	hostRule.Invert = true
 
-	// if this is a restart, cleanup previous rule first
-	err := os.netLink.RuleDel(hostRule)
+	// If this is a restart, cleanup previous rule first
+	err = os.netLink.RuleDel(hostRule)
 	if err != nil && !containsNoSuchRule(err) {
 		log.Errorf("Failed to cleanup old host IP rule: %v", err)
 		return errors.Wrapf(err, "host network setup: failed to delete old host rule")
 	}
 
-	err = os.netLink.RuleAdd(hostRule)
-
-	if err != nil {
-		log.Errorf("Failed to add host IP rule: %v", err)
-		return errors.Wrapf(err, "host network setup: failed to add host rule")
+	// Only include the rule if SNAT is not being handled by an external NAT gateway and needs to be
+	// handled on-node.
+	if !externalSNAT {
+		err = os.netLink.RuleAdd(hostRule)
+		if err != nil {
+			log.Errorf("Failed to add host IP rule: %v", err)
+			return errors.Wrapf(err, "host network setup: failed to add host rule")
+		}
 	}
 
 	ipt, err := iptables.New()
@@ -105,11 +120,19 @@ func (os *linuxNetwork) SetupHostNetwork(vpcCIDR *net.IPNet, primaryAddr *net.IP
 		return errors.Wrapf(err, "host network setup: failed to add POSTROUTING rule for primary address %s", primaryAddr)
 	}
 
-	if !exists {
+	if !exists && !externalSNAT {
+		// We are handling SNAT on-node, so include the iptables SNAT POSTROUTING rule.
 		err = ipt.Append("nat", "POSTROUTING", natCmd...)
 
 		if err != nil {
-			return errors.Wrapf(err, "host network setup: failed to append POSTROUTING rule for primary adderss %s", primaryAddr)
+			return errors.Wrapf(err, "host network setup: failed to append POSTROUTING rule for primary address %s", primaryAddr)
+		}
+	} else if exists && externalSNAT {
+		// We are not handling SNAT on-node, so delete the existing iptables SNAT POSTROUTING rule.
+		err = ipt.Delete("nat", "POSTROUTING", natCmd...)
+
+		if err != nil {
+			return errors.Wrapf(err, "host network setup: failed to delete POSTROUTING rule for primary address %s", primaryAddr)
 		}
 	}
 
@@ -123,6 +146,20 @@ func containsNoSuchRule(err error) bool {
 	return false
 }
 
+// useExternalSNAT returns whether SNAT of secondary ENI IPs should be handled with an external
+// NAT gateway rather than on node.
+func useExternalSNAT() (bool, error) {
+	if externalSNATStr := os.Getenv(envExternalSNAT); externalSNATStr != "" {
+		externalSNAT, err := strconv.ParseBool(externalSNATStr)
+		if err != nil {
+			log.Error("Failed to parse "+envExternalSNAT, err.Error())
+			return false, errors.Wrap(err, "ipamd init: failed to parse "+envExternalSNAT)
+		}
+		return externalSNAT, nil
+	}
+	return false, nil
+}
+
 // LinkByMac returns linux netlink based on interface MAC
 func LinkByMac(mac string, netLink netlinkwrapper.NetLink) (netlink.Link, error) {
 	links, err := netLink.LinkList()