Connect transparent proxy support

Add support for Consul Connect transparent proxies

Fixes: #10628

api/consul.go

@@ -158,12 +158,17 @@ func (st *SidecarTask) Canonicalize() {
 
 // ConsulProxy represents a Consul Connect sidecar proxy jobspec block.
 type ConsulProxy struct {
-	LocalServiceAddress string                 `mapstructure:"local_service_address" hcl:"local_service_address,optional"`
-	LocalServicePort    int                    `mapstructure:"local_service_port" hcl:"local_service_port,optional"`
-	Expose              *ConsulExposeConfig    `mapstructure:"expose" hcl:"expose,block"`
-	ExposeConfig        *ConsulExposeConfig    // Deprecated: only to maintain backwards compatibility. Use Expose instead.
-	Upstreams           []*ConsulUpstream      `hcl:"upstreams,block"`
-	Config              map[string]interface{} `hcl:"config,block"`
+	LocalServiceAddress string              `mapstructure:"local_service_address" hcl:"local_service_address,optional"`
+	LocalServicePort    int                 `mapstructure:"local_service_port" hcl:"local_service_port,optional"`
+	Expose              *ConsulExposeConfig `mapstructure:"expose" hcl:"expose,block"`
+	ExposeConfig        *ConsulExposeConfig // Deprecated: only to maintain backwards compatibility. Use Expose instead.
+	Upstreams           []*ConsulUpstream   `hcl:"upstreams,block"`
+
+	// TransparentProxy configures the Envoy sidecar to use "transparent
+	// proxying", which creates IP tables rules inside the network namespace to
+	// ensure traffic flows thru the Envoy proxy
+	TransparentProxy *ConsulTransparentProxy `mapstructure:"transparent_proxy" hcl:"transparent_proxy,block"`
+	Config           map[string]interface{}  `hcl:"config,block"`
 }
 
 func (cp *ConsulProxy) Canonicalize() {
@@ -177,6 +182,8 @@ func (cp *ConsulProxy) Canonicalize() {
 		cp.Upstreams = nil
 	}
 
+	cp.TransparentProxy.Canonicalize()
+
 	for _, upstream := range cp.Upstreams {
 		upstream.Canonicalize()
 	}
@@ -257,6 +264,61 @@ func (cu *ConsulUpstream) Canonicalize() {
 	}
 }
 
+// ConsulTransparentProxy is used to configure the Envoy sidecar for
+// "transparent proxying", which creates IP tables rules inside the network
+// namespace to ensure traffic flows thru the Envoy proxy
+type ConsulTransparentProxy struct {
+	// UID of the Envoy proxy. Defaults to the default Envoy proxy container
+	// image user.
+	UID string `mapstructure:"uid" hcl:"uid,optional"`
+
+	// OutboundPort is the Envoy proxy's outbound listener port. Inbound TCP
+	// traffic hitting the PROXY_IN_REDIRECT chain will be redirected here.
+	// Defaults to 15001.
+	OutboundPort uint16 `mapstructure:"outbound_port" hcl:"outbound_port,optional"`
+
+	// ExcludeInboundPorts is an additional set of ports will be excluded from
+	// redirection to the Envoy proxy. Can be Port.Label or Port.Value. This set
+	// will be added to the ports automatically excluded for the Expose.Port and
+	// Check.Expose fields.
+	ExcludeInboundPorts []string `mapstructure:"exclude_inbound_ports" hcl:"exclude_inbound_ports,optional"`
+
+	// ExcludeOutboundPorts is a set of outbound ports that will not be
+	// redirected to the Envoy proxy, specified as port numbers.
+	ExcludeOutboundPorts []uint16 `mapstructure:"exclude_outbound_ports" hcl:"exclude_outbound_ports,optional"`
+
+	// ExcludeOutboundCIDRs is a set of outbound CIDR blocks that will not be
+	// redirected to the Envoy proxy.
+	ExcludeOutboundCIDRs []string `mapstructure:"exclude_outbound_cidrs" hcl:"exclude_outbound_cidrs,optional"`
+
+	// ExcludeUIDs is a set of user IDs whose network traffic will not be
+	// redirected through the Envoy proxy.
+	ExcludeUIDs []string `mapstructure:"exclude_uids" hcl:"exclude_uids,optional"`
+
+	// NoDNS disables redirection of DNS traffic to Consul DNS. By default NoDNS
+	// is false and transparent proxy will direct DNS traffic to Consul DNS if
+	// available on the client.
+	NoDNS bool `mapstructure:"no_dns" hcl:"no_dns,optional"`
+}
+
+func (tp *ConsulTransparentProxy) Canonicalize() {
+	if tp == nil {
+		return
+	}
+	if len(tp.ExcludeInboundPorts) == 0 {
+		tp.ExcludeInboundPorts = nil
+	}
+	if len(tp.ExcludeOutboundCIDRs) == 0 {
+		tp.ExcludeOutboundCIDRs = nil
+	}
+	if len(tp.ExcludeOutboundPorts) == 0 {
+		tp.ExcludeOutboundPorts = nil
+	}
+	if len(tp.ExcludeUIDs) == 0 {
+		tp.ExcludeUIDs = nil
+	}
+}
+
 type ConsulExposeConfig struct {
 	Paths []*ConsulExposePath `mapstructure:"path" hcl:"path,block"`
 	Path  []*ConsulExposePath // Deprecated: only to maintain backwards compatibility. Use Paths instead.

client/allocrunner/network_manager_linux.go

@@ -190,13 +190,13 @@ func newNetworkConfigurator(log hclog.Logger, alloc *structs.Allocation, config
 
 	switch {
 	case netMode == "bridge":
-		c, err := newBridgeNetworkConfigurator(log, config.BridgeNetworkName, config.BridgeNetworkAllocSubnet, config.BridgeNetworkHairpinMode, config.CNIPath, ignorePortMappingHostIP)
+		c, err := newBridgeNetworkConfigurator(log, alloc, config.BridgeNetworkName, config.BridgeNetworkAllocSubnet, config.BridgeNetworkHairpinMode, config.CNIPath, ignorePortMappingHostIP, config.Node)
 		if err != nil {
 			return nil, err
 		}
 		return &synchronizedNetworkConfigurator{c}, nil
 	case strings.HasPrefix(netMode, "cni/"):
-		c, err := newCNINetworkConfigurator(log, config.CNIPath, config.CNIInterfacePrefix, config.CNIConfigDir, netMode[4:], ignorePortMappingHostIP)
+		c, err := newCNINetworkConfigurator(log, config.CNIPath, config.CNIInterfacePrefix, config.CNIConfigDir, netMode[4:], ignorePortMappingHostIP, config.Node)
 		if err != nil {
 			return nil, err
 		}

client/allocrunner/networking_bridge_linux.go

@@ -43,7 +43,7 @@ type bridgeNetworkConfigurator struct {
 	logger hclog.Logger
 }
 
-func newBridgeNetworkConfigurator(log hclog.Logger, bridgeName, ipRange string, hairpinMode bool, cniPath string, ignorePortMappingHostIP bool) (*bridgeNetworkConfigurator, error) {
+func newBridgeNetworkConfigurator(log hclog.Logger, alloc *structs.Allocation, bridgeName, ipRange string, hairpinMode bool, cniPath string, ignorePortMappingHostIP bool, node *structs.Node) (*bridgeNetworkConfigurator, error) {
 	b := &bridgeNetworkConfigurator{
 		bridgeName:  bridgeName,
 		allocSubnet: ipRange,
@@ -59,7 +59,20 @@ func newBridgeNetworkConfigurator(log hclog.Logger, bridgeName, ipRange string,
 		b.allocSubnet = defaultNomadAllocSubnet
 	}
 
-	c, err := newCNINetworkConfiguratorWithConf(log, cniPath, bridgeNetworkAllocIfPrefix, ignorePortMappingHostIP, buildNomadBridgeNetConfig(*b))
+	var netCfg []byte
+
+	tg := alloc.Job.LookupTaskGroup(alloc.TaskGroup)
+	for _, svc := range tg.Services {
+		if svc.Connect.HasTransparentProxy() {
+			netCfg = buildNomadBridgeNetConfig(*b, true)
+			break
+		}
+	}
+	if netCfg == nil {
+		netCfg = buildNomadBridgeNetConfig(*b, false)
+	}
+
+	c, err := newCNINetworkConfiguratorWithConf(log, cniPath, bridgeNetworkAllocIfPrefix, ignorePortMappingHostIP, netCfg, node)
 	if err != nil {
 		return nil, err
 	}
@@ -139,12 +152,19 @@ func (b *bridgeNetworkConfigurator) Teardown(ctx context.Context, alloc *structs
 	return b.cni.Teardown(ctx, alloc, spec)
 }
 
-func buildNomadBridgeNetConfig(b bridgeNetworkConfigurator) []byte {
+func buildNomadBridgeNetConfig(b bridgeNetworkConfigurator, withConsulCNI bool) []byte {
+	var consulCNI string
+	if withConsulCNI {
+		consulCNI = consulCNIBlock
+	}
+
 	return []byte(fmt.Sprintf(nomadCNIConfigTemplate,
 		b.bridgeName,
 		b.hairpinMode,
 		b.allocSubnet,
-		cniAdminChainName))
+		cniAdminChainName,
+		consulCNI,
+	))
 }
 
 // Update website/content/docs/networking/cni.mdx when the bridge configuration
@@ -187,7 +207,14 @@ const nomadCNIConfigTemplate = `{
 			"type": "portmap",
 			"capabilities": {"portMappings": true},
 			"snat": true
-		}
+		}%s
 	]
 }
 `
+
+const consulCNIBlock = `,
+		{
+			"type": "consul-cni",
+			"log_level": "debug"
+		}
+`

client/allocrunner/networking_bridge_linux_test.go

@@ -14,8 +14,9 @@ import (
 func Test_buildNomadBridgeNetConfig(t *testing.T) {
 	ci.Parallel(t)
 	testCases := []struct {
-		name string
-		b    *bridgeNetworkConfigurator
+		name          string
+		withConsulCNI bool
+		b             *bridgeNetworkConfigurator
 	}{
 		{
 			name: "empty",
@@ -38,14 +39,28 @@ func Test_buildNomadBridgeNetConfig(t *testing.T) {
 				hairpinMode: true,
 			},
 		},
+		{
+			name:          "consul-cni",
+			withConsulCNI: true,
+			b: &bridgeNetworkConfigurator{
+				bridgeName:  defaultNomadBridgeName,
+				allocSubnet: defaultNomadAllocSubnet,
+				hairpinMode: true,
+			},
+		},
 	}
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
 			tc := tc
 			ci.Parallel(t)
-			bCfg := buildNomadBridgeNetConfig(*tc.b)
+			bCfg := buildNomadBridgeNetConfig(*tc.b, tc.withConsulCNI)
 			// Validate that the JSON created is rational
 			must.True(t, json.Valid(bCfg))
+			if tc.withConsulCNI {
+				must.StrContains(t, string(bCfg), "consul-cni")
+			} else {
+				must.StrNotContains(t, string(bCfg), "consul-cni")
+			}
 		})
 	}
 }