fix(kuma-cp) allocate a new VIP for ExternalService host (#2302)

(cherry picked from commit 90b1c93)

pkg/dns/outbound.go

@@ -20,14 +20,15 @@ func VIPOutbounds(
 	resourceKey model.ResourceKey,
 	dataplanes []*core_mesh.DataplaneResource,
 	zoneIngresses []*core_mesh.ZoneIngressResource,
-	vips vips.List,
+	vipList vips.List,
 	externalServices []*core_mesh.ExternalServiceResource,
 ) []*mesh_proto.Dataplane_Networking_Outbound {
 	type vipEntry struct {
-		ip   string
-		port uint32
+		ip        string
+		port      uint32
+		entryType vips.EntryType
 	}
-	serviceVIPMap := map[string]vipEntry{}
+	serviceVIPMap := map[string][]vipEntry{}
 	services := []string{}
 	for _, dataplane := range dataplanes {
 		// backwards compatibility
@@ -37,9 +38,9 @@ func VIPOutbounds(
 					// Only add outbounds for services in the same mesh
 					inService := service.Tags[mesh_proto.ServiceTag]
 					if _, found := serviceVIPMap[inService]; !found {
-						vip, err := ForwardLookup(vips, inService)
+						vip, err := ForwardLookup(vipList, vips.NewServiceEntry(inService))
 						if err == nil {
-							serviceVIPMap[inService] = vipEntry{vip, VIPListenPort}
+							serviceVIPMap[inService] = append(serviceVIPMap[inService], vipEntry{vip, VIPListenPort, vips.Service})
 							services = append(services, inService)
 						}
 					}
@@ -49,9 +50,9 @@ func VIPOutbounds(
 			for _, inbound := range dataplane.Spec.GetNetworking().GetInbound() {
 				inService := inbound.GetTags()[mesh_proto.ServiceTag]
 				if _, found := serviceVIPMap[inService]; !found {
-					vip, err := ForwardLookup(vips, inService)
+					vip, err := ForwardLookup(vipList, vips.NewServiceEntry(inService))
 					if err == nil {
-						serviceVIPMap[inService] = vipEntry{vip, VIPListenPort}
+						serviceVIPMap[inService] = append(serviceVIPMap[inService], vipEntry{vip, VIPListenPort, vips.Service})
 						services = append(services, inService)
 					}
 				}
@@ -65,9 +66,9 @@ func VIPOutbounds(
 				// Only add outbounds for services in the same mesh
 				inService := service.Tags[mesh_proto.ServiceTag]
 				if _, found := serviceVIPMap[inService]; !found {
-					vip, err := ForwardLookup(vips, inService)
+					vip, err := ForwardLookup(vipList, vips.NewServiceEntry(inService))
 					if err == nil {
-						serviceVIPMap[inService] = vipEntry{vip, VIPListenPort}
+						serviceVIPMap[inService] = append(serviceVIPMap[inService], vipEntry{vip, VIPListenPort, vips.Service})
 						services = append(services, inService)
 					}
 				}
@@ -77,8 +78,9 @@ func VIPOutbounds(
 
 	for _, externalService := range externalServices {
 		inService := externalService.Spec.Tags[mesh_proto.ServiceTag]
+		host := externalService.Spec.GetHost()
 		if _, found := serviceVIPMap[inService]; !found {
-			vip, err := ForwardLookup(vips, inService)
+			vip1, err := ForwardLookup(vipList, vips.NewHostEntry(host))
 			if err == nil {
 				port := externalService.Spec.GetPort()
 				var p32 uint32
@@ -87,7 +89,19 @@ func VIPOutbounds(
 				} else {
 					p32 = uint32(p64)
 				}
-				serviceVIPMap[inService] = vipEntry{vip, p32}
+				serviceVIPMap[inService] = append(serviceVIPMap[inService], vipEntry{vip1, p32, vips.Host})
+				services = append(services, inService)
+			}
+			vip2, err := ForwardLookup(vipList, vips.NewServiceEntry(inService))
+			if err == nil {
+				port := externalService.Spec.GetPort()
+				var p32 uint32
+				if p64, err := strconv.ParseUint(port, 10, 32); err != nil {
+					p32 = VIPListenPort
+				} else {
+					p32 = uint32(p64)
+				}
+				serviceVIPMap[inService] = append(serviceVIPMap[inService], vipEntry{vip2, p32, vips.Service})
 				services = append(services, inService)
 			}
 		}
@@ -96,30 +110,34 @@ func VIPOutbounds(
 	sort.Strings(services)
 	outbounds := []*mesh_proto.Dataplane_Networking_Outbound{}
 	for _, service := range services {
-		entry := serviceVIPMap[service]
-		outbounds = append(outbounds, &mesh_proto.Dataplane_Networking_Outbound{
-			Address: entry.ip,
-			Port:    entry.port,
-			Tags:    map[string]string{mesh_proto.ServiceTag: service},
-		})
-
-		// todo (lobkovilya): backwards compatibility, could be deleted in the next major release Kuma 1.2.x
-		if entry.port != VIPListenPort {
+		entries := serviceVIPMap[service]
+		for _, entry := range entries {
 			outbounds = append(outbounds, &mesh_proto.Dataplane_Networking_Outbound{
 				Address: entry.ip,
-				Port:    VIPListenPort,
+				Port:    entry.port,
 				Tags:    map[string]string{mesh_proto.ServiceTag: service},
 			})
+
+			if entry.entryType != vips.Host {
+				// todo (lobkovilya): backwards compatibility, could be deleted in the next major release Kuma 1.2.x
+				if entry.port != VIPListenPort {
+					outbounds = append(outbounds, &mesh_proto.Dataplane_Networking_Outbound{
+						Address: entry.ip,
+						Port:    VIPListenPort,
+						Tags:    map[string]string{mesh_proto.ServiceTag: service},
+					})
+				}
+			}
 		}
 	}
 
 	return outbounds
 }
 
-func ForwardLookup(vips vips.List, service string) (string, error) {
-	ip, found := vips[service]
+func ForwardLookup(vips vips.List, entry vips.Entry) (string, error) {
+	ip, found := vips[entry]
 	if !found {
-		return "", errors.Errorf("service [%s] not found", service)
+		return "", errors.Errorf("entry name [%s] not found", entry.Name)
 	}
 	return ip, nil
 }

pkg/dns/outbound_test.go

@@ -48,7 +48,7 @@ var _ = Describe("VIPOutbounds", func() {
 		for i := 1; i <= 5; i++ {
 			service := "service-" + strconv.Itoa(i)
 			vip := fmt.Sprintf("240.0.0.%d", i)
-			vipList[service] = vip
+			vipList[vips.NewServiceEntry(service)] = vip
 
 			dataplanes.Items = append(dataplanes.Items, &core_mesh.DataplaneResource{
 				Meta: &test_model.ResourceMeta{
@@ -104,8 +104,8 @@ var _ = Describe("VIPOutbounds", func() {
 
 		// given
 		vipList := vips.List{
-			"service-a": "240.0.0.1",
-			"service-b": "240.0.0.2",
+			vips.NewServiceEntry("service-a"): "240.0.0.1",
+			vips.NewServiceEntry("service-b"): "240.0.0.2",
 		}
 		services := []*mesh_proto.Dataplane_Networking_Ingress_AvailableService{
 			{
@@ -171,7 +171,7 @@ var _ = Describe("VIPOutbounds", func() {
 		for i := 1; i <= 5; i++ {
 			service := "service-" + strconv.Itoa(i)
 			vip := fmt.Sprintf("240.0.0.%d", i)
-			vipList[service] = vip
+			vipList[vips.NewServiceEntry(service)] = vip
 
 			otherDataplanes = append(otherDataplanes, &core_mesh.DataplaneResource{
 				Meta: &test_model.ResourceMeta{
@@ -229,9 +229,9 @@ var _ = Describe("VIPOutbounds", func() {
 				},
 			},
 		}
-		vipList["first-external-service"] = "240.0.0.6"
-		vipList["second-external-service"] = "240.0.0.7"
-		vipList["third-external-service"] = "240.0.0.8"
+		vipList[vips.NewServiceEntry("first-external-service")] = "240.0.0.6"
+		vipList[vips.NewServiceEntry("second-external-service")] = "240.0.0.7"
+		vipList[vips.NewServiceEntry("third-external-service")] = "240.0.0.8"
 
 		actual := &mesh_proto.Dataplane_Networking{}
 		actual.Outbound = dns.VIPOutbounds(model.MetaToResourceKey(dataplane.Meta), otherDataplanes, nil, vipList, externalServices)
@@ -299,10 +299,10 @@ var _ = Describe("VIPOutbounds", func() {
 		}
 
 		vipList := vips.List{
-			"old-ingress-svc-1": "240.0.0.0",
-			"old-ingress-svc-2": "240.0.0.1",
-			"new-ingress-svc-1": "240.0.0.2",
-			"new-ingress-svc-2": "240.0.0.3",
+			vips.NewServiceEntry("old-ingress-svc-1"): "240.0.0.0",
+			vips.NewServiceEntry("old-ingress-svc-2"): "240.0.0.1",
+			vips.NewServiceEntry("new-ingress-svc-1"): "240.0.0.2",
+			vips.NewServiceEntry("new-ingress-svc-2"): "240.0.0.3",
 		}
 
 		otherDataplanes := []*core_mesh.DataplaneResource{{

pkg/dns/resolver/resolver.go

@@ -14,10 +14,7 @@ type DNSResolver interface {
 	GetDomain() string
 	SetVIPs(list vips.List)
 	GetVIPs() vips.List
-
-	ForwardLookup(service string) (string, error)
 	ForwardLookupFQDN(name string) (string, error)
-	ReverseLookup(ip string) (string, error)
 }
 
 type dnsResolver struct {
@@ -50,18 +47,6 @@ func (s *dnsResolver) GetVIPs() vips.List {
 	return s.viplist
 }
 
-func (s *dnsResolver) ForwardLookup(service string) (string, error) {
-	s.RLock()
-	defer s.RUnlock()
-
-	ip, found := s.viplist[service]
-
-	if !found {
-		return "", errors.Errorf("service [%s] not found in domain [%s].", service, s.domain)
-	}
-	return ip, nil
-}
-
 func (s *dnsResolver) ForwardLookupFQDN(name string) (string, error) {
 	s.RLock()
 	defer s.RUnlock()
@@ -79,27 +64,14 @@ func (s *dnsResolver) ForwardLookupFQDN(name string) (string, error) {
 		return "", err
 	}
 
-	ip, found := s.viplist[service]
+	ip, found := s.viplist[vips.NewServiceEntry(service)]
 	if !found {
 		return "", errors.Errorf("service [%s] not found in domain [%s].", service, domain)
 	}
 
 	return ip, nil
 }
 
-func (s *dnsResolver) ReverseLookup(ip string) (string, error) {
-	s.RLock()
-	defer s.RUnlock()
-
-	for service, serviceIP := range s.viplist {
-		if serviceIP == ip {
-			return service + "." + s.domain, nil
-		}
-	}
-
-	return "", errors.Errorf("IP [%s] not found", ip)
-}
-
 func (s *dnsResolver) domainFromName(name string) (string, error) {
 	split := dns.SplitDomainName(name)
 	if len(split) < 1 {

pkg/dns/server_test.go

@@ -60,8 +60,8 @@ var _ = Describe("DNS server", func() {
 		It("should resolve", func() {
 			// given
 			var err error
-			dnsResolver.SetVIPs(map[string]string{
-				"service": "240.0.0.1",
+			dnsResolver.SetVIPs(vips.List{
+				vips.NewServiceEntry("service"): "240.0.0.1",
 			})
 			ip, err = dnsResolver.ForwardLookupFQDN("service.mesh")
 			Expect(err).ToNot(HaveOccurred())
@@ -86,8 +86,8 @@ var _ = Describe("DNS server", func() {
 
 		It("should resolve concurrent", func() {
 			// given
-			dnsResolver.SetVIPs(map[string]string{
-				"service": "240.0.0.1",
+			dnsResolver.SetVIPs(vips.List{
+				vips.NewServiceEntry("service"): "240.0.0.1",
 			})
 			ip, err := dnsResolver.ForwardLookupFQDN("service.mesh")
 			Expect(err).ToNot(HaveOccurred())
@@ -118,8 +118,8 @@ var _ = Describe("DNS server", func() {
 
 		It("should resolve IPv6 concurrent", func() {
 			// given
-			dnsResolver.SetVIPs(map[string]string{
-				"service": "fd00::1",
+			dnsResolver.SetVIPs(vips.List{
+				vips.NewServiceEntry("service"): "fd00::1",
 			})
 			ip, err := dnsResolver.ForwardLookupFQDN("service.mesh")
 			Expect(err).ToNot(HaveOccurred())
@@ -151,8 +151,8 @@ var _ = Describe("DNS server", func() {
 		It("should not resolve", func() {
 			// given
 			var err error
-			dnsResolver.SetVIPs(map[string]string{
-				"service": "240.0.0.1",
+			dnsResolver.SetVIPs(vips.List{
+				vips.NewServiceEntry("service"): "240.0.0.1",
 			})
 			ip, err = dnsResolver.ForwardLookupFQDN("service.mesh")
 			Expect(err).ToNot(HaveOccurred())
@@ -177,7 +177,7 @@ var _ = Describe("DNS server", func() {
 
 		It("should not resolve when no vips", func() {
 			// given
-			dnsResolver.SetVIPs(map[string]string{})
+			dnsResolver.SetVIPs(vips.List{})
 
 			// when
 			client := new(dns.Client)
@@ -202,7 +202,7 @@ var _ = Describe("DNS server", func() {
 			// given
 			var err error
 			dnsResolver.SetVIPs(vips.List{
-				"my.service": "240.0.0.1",
+				vips.NewServiceEntry("my.service"): "240.0.0.1",
 			})
 			ip, err = dnsResolver.ForwardLookupFQDN("my.service.mesh")
 			Expect(err).ToNot(HaveOccurred())
@@ -229,7 +229,7 @@ var _ = Describe("DNS server", func() {
 			// given
 			var err error
 			dnsResolver.SetVIPs(vips.List{
-				"my-service_test-namespace_svc_80": "240.0.0.1",
+				vips.NewServiceEntry("my-service_test-namespace_svc_80"): "240.0.0.1",
 			})
 			ip, err = dnsResolver.ForwardLookupFQDN("my-service_test-namespace_svc_80.mesh")
 			Expect(err).ToNot(HaveOccurred())

pkg/dns/vips/interfaces.go

@@ -1,15 +1,65 @@
 package vips
 
-type List map[string]string
+import (
+	"fmt"
+	"sort"
+)
+
+type EntryType int
+
+const (
+	Service EntryType = iota
+	Host
+)
+
+type Entry struct {
+	Type EntryType `json:"type"`
+	Name string    `json:"name"`
+}
+
+func (e Entry) String() string {
+	return fmt.Sprintf("%v:%s", e.Type, e.Name)
+}
+
+func (e Entry) MarshalText() (text []byte, err error) {
+	return []byte(e.String()), nil
+}
+
+func (e *Entry) UnmarshalText(text []byte) error {
+	_, err := fmt.Sscanf(string(text), "%v:%s", &e.Type, &e.Name)
+	return err
+}
+
+func NewHostEntry(host string) Entry {
+	return Entry{Host, host}
+}
+
+func NewServiceEntry(name string) Entry {
+	return Entry{Service, name}
+}
+
+type EntrySet map[Entry]bool
+
+func (s EntrySet) ToArray() (entries []Entry) {
+	for entry := range s {
+		entries = append(entries, entry)
+	}
+	sort.SliceStable(entries, func(i, j int) bool {
+		return entries[i].String() < entries[j].String()
+	})
+	return
+}
+
+type List map[Entry]string
 
 func (vips List) Append(other List) {
 	for k, v := range other {
 		vips[k] = v
 	}
 }
 
-func (vips List) FQDNsByIPs() map[string]string {
-	ipToDomain := map[string]string{}
+func (vips List) FQDNsByIPs() map[string]Entry {
+	ipToDomain := map[string]Entry{}
 	for domain, ip := range vips {
 		ipToDomain[ip] = domain
 	}