Merge branch 'main' into docs/annotations-page-structure-fixes

.github/workflows/test-integrations.yml

@@ -374,6 +374,7 @@ jobs:
               | jq --raw-input --argjson runnercount "$NUM_RUNNERS" "$JQ_SLICER" \
               | jq --compact-output 'map(join(" "))'
           } >> "$GITHUB_OUTPUT"
+
   compatibility-integration-test:
     runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }}
     needs:
@@ -387,6 +388,8 @@ jobs:
       fail-fast: false
       matrix:
         test-cases: ${{ fromJSON(needs.generate-compatibility-job-matrices.outputs.compatibility-matrix) }}
+    env:
+      ENVOY_VERSION: "1.25.4"
     steps:
       - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
       - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0
@@ -405,6 +408,8 @@ jobs:
 
       - name: Build consul:local image
         run: docker build -t ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local -f ./build-support/docker/Consul-Dev.dockerfile .
+      - name: Build consul-envoy:target-version image
+        run: docker build -t consul-envoy:target-version --build-arg CONSUL_IMAGE=${{ env.CONSUL_LATEST_IMAGE_NAME }}:local --build-arg ENVOY_VERSION=${{ env.ENVOY_VERSION }} -f ./test/integration/consul-container/assets/Dockerfile-consul-envoy ./test/integration/consul-container/assets
       - name: Configure GH workaround for ipv6 loopback
         if: ${{ !endsWith(github.repository, '-enterprise') }}
         run: |
@@ -520,7 +525,8 @@ jobs:
         consul-version: [ "1.14", "1.15"]
         test-cases: ${{ fromJSON(needs.generate-upgrade-job-matrices.outputs.upgrade-matrix) }}
     env:
-      CONSUL_VERSION: ${{ matrix.consul-version }}
+      CONSUL_LATEST_VERSION: ${{ matrix.consul-version }}
+      ENVOY_VERSION: "1.24.6"
     steps:
       - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
       - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0
@@ -538,6 +544,10 @@ jobs:
         run: chmod +x consul
       - name: Build consul:local image
         run: docker build -t ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local -f ./build-support/docker/Consul-Dev.dockerfile .
+      - name: Build consul-envoy:latest-version image
+        run: docker build -t consul-envoy:latest-version --build-arg CONSUL_IMAGE=docker.mirror.hashicorp.services/${{ env.CONSUL_LATEST_IMAGE_NAME }}:${{ env.CONSUL_LATEST_VERSION }} --build-arg ENVOY_VERSION=${{ env.ENVOY_VERSION }} -f ./test/integration/consul-container/assets/Dockerfile-consul-envoy ./test/integration/consul-container/assets
+      - name: Build consul-envoy:target-version image
+        run: docker build -t consul-envoy:target-version --build-arg CONSUL_IMAGE=${{ env.CONSUL_LATEST_IMAGE_NAME }}:local --build-arg ENVOY_VERSION=${{ env.ENVOY_VERSION }} -f ./test/integration/consul-container/assets/Dockerfile-consul-envoy ./test/integration/consul-container/assets
       - name: Configure GH workaround for ipv6 loopback
         if: ${{ !endsWith(github.repository, '-enterprise') }}
         run: |
@@ -567,8 +577,8 @@ jobs:
             -run "${{ matrix.test-cases }}" \
             --target-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \
             --target-version local \
-            --latest-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \
-            --latest-version "${{ env.CONSUL_VERSION }}"
+            --latest-image docker.mirror.hashicorp.services/${{ env.CONSUL_LATEST_IMAGE_NAME }} \
+            --latest-version "${{ env.CONSUL_LATEST_VERSION }}"
           ls -lrt
         env:
           # this is needed because of incompatibility between RYUK container and GHA

GNUmakefile

@@ -61,6 +61,7 @@ GO_BUILD_TAG?=consul-build-go
 UI_BUILD_TAG?=consul-build-ui
 BUILD_CONTAINER_NAME?=consul-builder
 CONSUL_IMAGE_VERSION?=latest
+ENVOY_VERSION?='1.25.4'
 
 ################
 # CI Variables #
@@ -459,10 +460,13 @@ else
 		--latest-version latest
 endif
 
+# NOTE: Use DOCKER_BUILDKIT=0, if docker build fails to resolve consul:local base image
 .PHONY: test-compat-integ-setup
 test-compat-integ-setup: dev-docker
 	@docker tag consul-dev:latest $(CONSUL_COMPAT_TEST_IMAGE):local
 	@docker run --rm -t $(CONSUL_COMPAT_TEST_IMAGE):local consul version
+	@#  'consul-envoy:target-version' is needed by compatibility integ test
+	@docker build -t consul-envoy:target-version --build-arg CONSUL_IMAGE=$(CONSUL_COMPAT_TEST_IMAGE):local --build-arg ENVOY_VERSION=${ENVOY_VERSION} -f ./test/integration/consul-container/assets/Dockerfile-consul-envoy ./test/integration/consul-container/assets
 
 .PHONY: test-metrics-integ
 test-metrics-integ: test-compat-integ-setup

agent/catalog_endpoint.go

@@ -573,3 +573,34 @@ RETRY_ONCE:
 		s.nodeMetricsLabels())
 	return out.Services, nil
 }
+
+func (s *HTTPHandlers) AssignManualServiceVIPs(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
+	metrics.IncrCounterWithLabels([]string{"client", "api", "service_virtual_ips"}, 1,
+		s.nodeMetricsLabels())
+
+	var args structs.AssignServiceManualVIPsRequest
+	if err := s.parseEntMetaNoWildcard(req, &args.EnterpriseMeta); err != nil {
+		return nil, err
+	}
+
+	if err := s.rewordUnknownEnterpriseFieldError(decodeBody(req.Body, &args)); err != nil {
+		return nil, HTTPError{StatusCode: http.StatusBadRequest, Reason: fmt.Sprintf("Request decode failed: %v", err)}
+	}
+
+	// Setup the default DC if not provided
+	if args.Datacenter == "" {
+		args.Datacenter = s.agent.config.Datacenter
+	}
+	s.parseToken(req, &args.Token)
+
+	// Forward to the servers
+	var out structs.AssignServiceManualVIPsResponse
+	if err := s.agent.RPC(req.Context(), "Internal.AssignManualServiceVIPs", &args, &out); err != nil {
+		metrics.IncrCounterWithLabels([]string{"client", "rpc", "error", "service_virtual_ips"}, 1,
+			s.nodeMetricsLabels())
+		return nil, err
+	}
+	metrics.IncrCounterWithLabels([]string{"client", "api", "success", "service_virtual_ips"}, 1,
+		s.nodeMetricsLabels())
+	return out, nil
+}

agent/catalog_endpoint_test.go

@@ -2028,3 +2028,67 @@ func TestCatalog_GatewayServices_Ingress(t *testing.T) {
 		require.Equal(r, expect, gatewayServices)
 	})
 }
+
+func TestCatalogRegister_AssignManualServiceVIPs(t *testing.T) {
+	if testing.Short() {
+		t.Skip("too slow for testing.Short")
+	}
+
+	t.Parallel()
+	a := NewTestAgent(t, "")
+	defer a.Shutdown()
+
+	testrpc.WaitForTestAgent(t, a.RPC, "dc1")
+
+	for _, service := range []string{"api", "web"} {
+		req := structs.ConfigEntryRequest{
+			Datacenter: "dc1",
+			Entry: &structs.ServiceResolverConfigEntry{
+				Kind: structs.ServiceResolver,
+				Name: service,
+			},
+		}
+		var out bool
+		require.NoError(t, a.RPC(context.Background(), "ConfigEntry.Apply", &req, &out))
+	}
+
+	assignVIPs := func(req structs.AssignServiceManualVIPsRequest, expect structs.AssignServiceManualVIPsResponse) {
+		httpReq, _ := http.NewRequest("PUT", "/v1/internal/service-virtual-ip", jsonReader(req))
+		resp := httptest.NewRecorder()
+		obj, err := a.srv.AssignManualServiceVIPs(resp, httpReq)
+		require.NoError(t, err)
+
+		result, ok := obj.(structs.AssignServiceManualVIPsResponse)
+		require.True(t, ok)
+		require.Equal(t, expect, result)
+	}
+
+	// Assign some manual IPs to the service
+	assignVIPs(structs.AssignServiceManualVIPsRequest{
+		Service:    "api",
+		ManualVIPs: []string{"1.1.1.1", "2.2.2.2", "3.3.3.3"},
+	}, structs.AssignServiceManualVIPsResponse{
+		Found: true,
+	})
+
+	// Assign some manual IPs to the new service, reassigning one from the existing service.
+	assignVIPs(structs.AssignServiceManualVIPsRequest{
+		Service:    "web",
+		ManualVIPs: []string{"2.2.2.2", "4.4.4.4"},
+	}, structs.AssignServiceManualVIPsResponse{
+		Found: true,
+		UnassignedFrom: []structs.PeeredServiceName{
+			{
+				ServiceName: structs.ServiceName{Name: "api", EnterpriseMeta: *acl.DefaultEnterpriseMeta()},
+			},
+		},
+	})
+
+	// Assign some manual IPs a non-existent service, should be a no-op.
+	assignVIPs(structs.AssignServiceManualVIPsRequest{
+		Service:    "nope",
+		ManualVIPs: []string{"1.1.1.1", "2.2.2.2", "3.3.3.3", "4.4.4.4"},
+	}, structs.AssignServiceManualVIPsResponse{
+		Found: false,
+	})
+}

agent/consul/discovery_chain_endpoint_test.go

@@ -261,6 +261,8 @@ func TestDiscoveryChainEndpoint_Get(t *testing.T) {
 						33*time.Second,
 					),
 				},
+				AutoVirtualIPs:   []string{"240.0.0.1"},
+				ManualVirtualIPs: []string{},
 			},
 		}
 		require.Equal(t, expect, resp)

agent/consul/discoverychain/compile.go

@@ -44,6 +44,11 @@ type CompileRequest struct {
 	OverrideConnectTimeout time.Duration
 
 	Entries *configentry.DiscoveryChainSet
+
+	// AutoVirtualIPs and ManualVirtualIPs are lists of IPs associated with
+	// the service.
+	AutoVirtualIPs   []string
+	ManualVirtualIPs []string
 }
 
 // Compile assembles a discovery chain in the form of a graph of nodes using
@@ -98,6 +103,8 @@ func Compile(req CompileRequest) (*structs.CompiledDiscoveryChain, error) {
 		overrideProtocol:       req.OverrideProtocol,
 		overrideConnectTimeout: req.OverrideConnectTimeout,
 		entries:                entries,
+		autoVirtualIPs:         req.AutoVirtualIPs,
+		manualVirtualIPs:       req.ManualVirtualIPs,
 
 		resolvers:     make(map[structs.ServiceID]*structs.ServiceResolverConfigEntry),
 		splitterNodes: make(map[string]*structs.DiscoveryGraphNode),
@@ -139,6 +146,11 @@ type compiler struct {
 	// This is an INPUT field.
 	entries *configentry.DiscoveryChainSet
 
+	// autoVirtualIPs and manualVirtualIPs are lists of IPs associated with
+	// the service.
+	autoVirtualIPs   []string
+	manualVirtualIPs []string
+
 	// resolvers is initially seeded by copying the provided entries.Resolvers
 	// map and default resolvers are added as they are needed.
 	resolvers map[structs.ServiceID]*structs.ServiceResolverConfigEntry
@@ -352,6 +364,8 @@ func (c *compiler) compile() (*structs.CompiledDiscoveryChain, error) {
 		StartNode:         c.startNode,
 		Nodes:             c.nodes,
 		Targets:           c.loadedTargets,
+		AutoVirtualIPs:    c.autoVirtualIPs,
+		ManualVirtualIPs:  c.manualVirtualIPs,
 	}, nil
 }
 

agent/consul/fsm/commands_oss.go

@@ -794,9 +794,13 @@ func (c *FSM) applyManualVirtualIPs(buf []byte, index uint64) interface{} {
 		panic(fmt.Errorf("failed to decode request: %v", err))
 	}
 
-	if err := c.state.AssignManualVirtualIPs(index, req.Service, req.ManualIPs); err != nil {
+	found, unassignedFrom, err := c.state.AssignManualServiceVIPs(index, req.Service, req.ManualIPs)
+	if err != nil {
 		c.logger.Warn("AssignManualVirtualIPs failed", "error", err)
 		return err
 	}
-	return nil
+	return structs.AssignServiceManualVIPsResponse{
+		Found:          found,
+		UnassignedFrom: unassignedFrom,
+	}
 }

agent/consul/internal_endpoint.go

@@ -5,6 +5,7 @@ package consul
 
 import (
 	"fmt"
+	"net"
 
 	"github.com/hashicorp/go-bexpr"
 	"github.com/hashicorp/go-hclog"
@@ -17,6 +18,8 @@ import (
 	"github.com/hashicorp/consul/agent/structs"
 )
 
+const MaximumManualVIPsPerService = 8
+
 // Internal endpoint is used to query the miscellaneous info that
 // does not necessarily fit into the other systems. It is also
 // used to hold undocumented APIs that users should not rely on.
@@ -741,6 +744,55 @@ func (m *Internal) PeeredUpstreams(args *structs.PartitionSpecificRequest, reply
 		})
 }
 
+// AssignManualServiceVIPs allows for assigning virtual IPs to a service manually, so that they can
+// be returned along with discovery chain information for use by transparent proxies.
+func (m *Internal) AssignManualServiceVIPs(args *structs.AssignServiceManualVIPsRequest, reply *structs.AssignServiceManualVIPsResponse) error {
+	if done, err := m.srv.ForwardRPC("Internal.AssignManualServiceVIPs", args, reply); done {
+		return err
+	}
+
+	var authzCtx acl.AuthorizerContext
+	authz, err := m.srv.ResolveTokenAndDefaultMeta(args.Token, &args.EnterpriseMeta, &authzCtx)
+	if err != nil {
+		return err
+	}
+	if err := authz.ToAllowAuthorizer().MeshWriteAllowed(&authzCtx); err != nil {
+		return err
+	}
+
+	if err := m.srv.validateEnterpriseRequest(&args.EnterpriseMeta, true); err != nil {
+		return err
+	}
+
+	if len(args.ManualVIPs) > MaximumManualVIPsPerService {
+		return fmt.Errorf("cannot associate more than %d manual virtual IPs with the same service", MaximumManualVIPsPerService)
+	}
+
+	for _, ip := range args.ManualVIPs {
+		parsedIP := net.ParseIP(ip)
+		if parsedIP == nil || parsedIP.To4() == nil {
+			return fmt.Errorf("%q is not a valid IPv4 address", parsedIP.String())
+		}
+	}
+
+	req := state.ServiceVirtualIP{
+		Service: structs.PeeredServiceName{
+			ServiceName: structs.NewServiceName(args.Service, &args.EnterpriseMeta),
+		},
+		ManualIPs: args.ManualVIPs,
+	}
+	resp, err := m.srv.raftApplyMsgpack(structs.UpdateVirtualIPRequestType, req)
+	if err != nil {
+		return err
+	}
+	typedResp, ok := resp.(structs.AssignServiceManualVIPsResponse)
+	if !ok {
+		return fmt.Errorf("unexpected type %T for AssignManualServiceVIPs", resp)
+	}
+	*reply = typedResp
+	return nil
+}
+
 // EventFire is a bit of an odd endpoint, but it allows for a cross-DC RPC
 // call to fire an event. The primary use case is to enable user events being
 // triggered in a remote DC.