Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(kuma-cp) allocate a new VIP for ExternalService host #2302

Merged
merged 2 commits into from
Jul 8, 2021
Merged

fix(kuma-cp) allocate a new VIP for ExternalService host #2302

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3007f87
fix(kuma-cp) allocate new VIP for ExternalService host
lobkovilya Jul 5, 2021
c87e631
fix(kuma-cp) review
lobkovilya Jul 7, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 41 additions & 23 deletions pkg/dns/outbound.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,15 @@ func VIPOutbounds(
resourceKey model.ResourceKey,
dataplanes []*core_mesh.DataplaneResource,
zoneIngresses []*core_mesh.ZoneIngressResource,
vips vips.List,
vipList vips.List,
externalServices []*core_mesh.ExternalServiceResource,
) []*mesh_proto.Dataplane_Networking_Outbound {
type vipEntry struct {
ip string
port uint32
host bool
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

how about EntryType?

}
serviceVIPMap := map[string]vipEntry{}
serviceVIPMap := map[string][]vipEntry{}
services := []string{}
for _, dataplane := range dataplanes {
// backwards compatibility
Expand All @@ -37,9 +38,9 @@ func VIPOutbounds(
// Only add outbounds for services in the same mesh
inService := service.Tags[mesh_proto.ServiceTag]
if _, found := serviceVIPMap[inService]; !found {
vip, err := ForwardLookup(vips, inService)
vip, err := ForwardLookup(vipList, vips.NewServiceEntry(inService))
if err == nil {
serviceVIPMap[inService] = vipEntry{vip, VIPListenPort}
serviceVIPMap[inService] = append(serviceVIPMap[inService], vipEntry{vip, VIPListenPort, false})
services = append(services, inService)
}
}
Expand All @@ -49,9 +50,9 @@ func VIPOutbounds(
for _, inbound := range dataplane.Spec.GetNetworking().GetInbound() {
inService := inbound.GetTags()[mesh_proto.ServiceTag]
if _, found := serviceVIPMap[inService]; !found {
vip, err := ForwardLookup(vips, inService)
vip, err := ForwardLookup(vipList, vips.NewServiceEntry(inService))
if err == nil {
serviceVIPMap[inService] = vipEntry{vip, VIPListenPort}
serviceVIPMap[inService] = append(serviceVIPMap[inService], vipEntry{vip, VIPListenPort, false})
services = append(services, inService)
}
}
Expand All @@ -65,9 +66,9 @@ func VIPOutbounds(
// Only add outbounds for services in the same mesh
inService := service.Tags[mesh_proto.ServiceTag]
if _, found := serviceVIPMap[inService]; !found {
vip, err := ForwardLookup(vips, inService)
vip, err := ForwardLookup(vipList, vips.NewServiceEntry(inService))
if err == nil {
serviceVIPMap[inService] = vipEntry{vip, VIPListenPort}
serviceVIPMap[inService] = append(serviceVIPMap[inService], vipEntry{vip, VIPListenPort, false})
services = append(services, inService)
}
}
Expand All @@ -77,8 +78,9 @@ func VIPOutbounds(

for _, externalService := range externalServices {
inService := externalService.Spec.Tags[mesh_proto.ServiceTag]
host := externalService.Spec.GetHost()
if _, found := serviceVIPMap[inService]; !found {
vip, err := ForwardLookup(vips, inService)
vip1, err := ForwardLookup(vipList, vips.NewHostEntry(host))
if err == nil {
port := externalService.Spec.GetPort()
var p32 uint32
Expand All @@ -87,7 +89,19 @@ func VIPOutbounds(
} else {
p32 = uint32(p64)
}
serviceVIPMap[inService] = vipEntry{vip, p32}
serviceVIPMap[inService] = append(serviceVIPMap[inService], vipEntry{vip1, p32, true})
services = append(services, inService)
}
vip2, err := ForwardLookup(vipList, vips.NewServiceEntry(inService))
if err == nil {
port := externalService.Spec.GetPort()
var p32 uint32
if p64, err := strconv.ParseUint(port, 10, 32); err != nil {
p32 = VIPListenPort
} else {
p32 = uint32(p64)
}
serviceVIPMap[inService] = append(serviceVIPMap[inService], vipEntry{vip2, p32, false})
services = append(services, inService)
}
}
Expand All @@ -96,30 +110,34 @@ func VIPOutbounds(
sort.Strings(services)
outbounds := []*mesh_proto.Dataplane_Networking_Outbound{}
for _, service := range services {
entry := serviceVIPMap[service]
outbounds = append(outbounds, &mesh_proto.Dataplane_Networking_Outbound{
Address: entry.ip,
Port: entry.port,
Tags: map[string]string{mesh_proto.ServiceTag: service},
})

// todo (lobkovilya): backwards compatibility, could be deleted in the next major release Kuma 1.2.x
if entry.port != VIPListenPort {
entries := serviceVIPMap[service]
for _, entry := range entries {
outbounds = append(outbounds, &mesh_proto.Dataplane_Networking_Outbound{
Address: entry.ip,
Port: VIPListenPort,
Port: entry.port,
Tags: map[string]string{mesh_proto.ServiceTag: service},
})

if !entry.host {
// todo (lobkovilya): backwards compatibility, could be deleted in the next major release Kuma 1.2.x
if entry.port != VIPListenPort {
outbounds = append(outbounds, &mesh_proto.Dataplane_Networking_Outbound{
Address: entry.ip,
Port: VIPListenPort,
Tags: map[string]string{mesh_proto.ServiceTag: service},
})
}
}
}
}

return outbounds
}

func ForwardLookup(vips vips.List, service string) (string, error) {
ip, found := vips[service]
func ForwardLookup(vips vips.List, entry vips.Entry) (string, error) {
ip, found := vips[entry]
if !found {
return "", errors.Errorf("service [%s] not found", service)
return "", errors.Errorf("entry name [%s] not found", entry.Name)
}
return ip, nil
}
22 changes: 11 additions & 11 deletions pkg/dns/outbound_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ var _ = Describe("VIPOutbounds", func() {
for i := 1; i <= 5; i++ {
service := "service-" + strconv.Itoa(i)
vip := fmt.Sprintf("240.0.0.%d", i)
vipList[service] = vip
vipList[vips.NewServiceEntry(service)] = vip

dataplanes.Items = append(dataplanes.Items, &core_mesh.DataplaneResource{
Meta: &test_model.ResourceMeta{
Expand Down Expand Up @@ -104,8 +104,8 @@ var _ = Describe("VIPOutbounds", func() {

// given
vipList := vips.List{
"service-a": "240.0.0.1",
"service-b": "240.0.0.2",
vips.NewServiceEntry("service-a"): "240.0.0.1",
vips.NewServiceEntry("service-b"): "240.0.0.2",
}
services := []*mesh_proto.Dataplane_Networking_Ingress_AvailableService{
{
Expand Down Expand Up @@ -171,7 +171,7 @@ var _ = Describe("VIPOutbounds", func() {
for i := 1; i <= 5; i++ {
service := "service-" + strconv.Itoa(i)
vip := fmt.Sprintf("240.0.0.%d", i)
vipList[service] = vip
vipList[vips.NewServiceEntry(service)] = vip

otherDataplanes = append(otherDataplanes, &core_mesh.DataplaneResource{
Meta: &test_model.ResourceMeta{
Expand Down Expand Up @@ -229,9 +229,9 @@ var _ = Describe("VIPOutbounds", func() {
},
},
}
vipList["first-external-service"] = "240.0.0.6"
vipList["second-external-service"] = "240.0.0.7"
vipList["third-external-service"] = "240.0.0.8"
vipList[vips.NewServiceEntry("first-external-service")] = "240.0.0.6"
vipList[vips.NewServiceEntry("second-external-service")] = "240.0.0.7"
vipList[vips.NewServiceEntry("third-external-service")] = "240.0.0.8"

actual := &mesh_proto.Dataplane_Networking{}
actual.Outbound = dns.VIPOutbounds(model.MetaToResourceKey(dataplane.Meta), otherDataplanes, nil, vipList, externalServices)
Expand Down Expand Up @@ -299,10 +299,10 @@ var _ = Describe("VIPOutbounds", func() {
}

vipList := vips.List{
"old-ingress-svc-1": "240.0.0.0",
"old-ingress-svc-2": "240.0.0.1",
"new-ingress-svc-1": "240.0.0.2",
"new-ingress-svc-2": "240.0.0.3",
vips.NewServiceEntry("old-ingress-svc-1"): "240.0.0.0",
vips.NewServiceEntry("old-ingress-svc-2"): "240.0.0.1",
vips.NewServiceEntry("new-ingress-svc-1"): "240.0.0.2",
vips.NewServiceEntry("new-ingress-svc-2"): "240.0.0.3",
}

otherDataplanes := []*core_mesh.DataplaneResource{{
Expand Down
30 changes: 1 addition & 29 deletions pkg/dns/resolver/resolver.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,7 @@ type DNSResolver interface {
GetDomain() string
SetVIPs(list vips.List)
GetVIPs() vips.List

ForwardLookup(service string) (string, error)
ForwardLookupFQDN(name string) (string, error)
ReverseLookup(ip string) (string, error)
}

type dnsResolver struct {
Expand Down Expand Up @@ -50,18 +47,6 @@ func (s *dnsResolver) GetVIPs() vips.List {
return s.viplist
}

func (s *dnsResolver) ForwardLookup(service string) (string, error) {
s.RLock()
defer s.RUnlock()

ip, found := s.viplist[service]

if !found {
return "", errors.Errorf("service [%s] not found in domain [%s].", service, s.domain)
}
return ip, nil
}

func (s *dnsResolver) ForwardLookupFQDN(name string) (string, error) {
s.RLock()
defer s.RUnlock()
Expand All @@ -79,27 +64,14 @@ func (s *dnsResolver) ForwardLookupFQDN(name string) (string, error) {
return "", err
}

ip, found := s.viplist[service]
ip, found := s.viplist[vips.NewServiceEntry(service)]
if !found {
return "", errors.Errorf("service [%s] not found in domain [%s].", service, domain)
}

return ip, nil
}

func (s *dnsResolver) ReverseLookup(ip string) (string, error) {
s.RLock()
defer s.RUnlock()

for service, serviceIP := range s.viplist {
if serviceIP == ip {
return service + "." + s.domain, nil
}
}

return "", errors.Errorf("IP [%s] not found", ip)
}

func (s *dnsResolver) domainFromName(name string) (string, error) {
split := dns.SplitDomainName(name)
if len(split) < 1 {
Expand Down
22 changes: 11 additions & 11 deletions pkg/dns/server_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,8 @@ var _ = Describe("DNS server", func() {
It("should resolve", func() {
// given
var err error
dnsResolver.SetVIPs(map[string]string{
"service": "240.0.0.1",
dnsResolver.SetVIPs(vips.List{
vips.NewServiceEntry("service"): "240.0.0.1",
})
ip, err = dnsResolver.ForwardLookupFQDN("service.mesh")
Expect(err).ToNot(HaveOccurred())
Expand All @@ -86,8 +86,8 @@ var _ = Describe("DNS server", func() {

It("should resolve concurrent", func() {
// given
dnsResolver.SetVIPs(map[string]string{
"service": "240.0.0.1",
dnsResolver.SetVIPs(vips.List{
vips.NewServiceEntry("service"): "240.0.0.1",
})
ip, err := dnsResolver.ForwardLookupFQDN("service.mesh")
Expect(err).ToNot(HaveOccurred())
Expand Down Expand Up @@ -118,8 +118,8 @@ var _ = Describe("DNS server", func() {

It("should resolve IPv6 concurrent", func() {
// given
dnsResolver.SetVIPs(map[string]string{
"service": "fd00::1",
dnsResolver.SetVIPs(vips.List{
vips.NewServiceEntry("service"): "fd00::1",
})
ip, err := dnsResolver.ForwardLookupFQDN("service.mesh")
Expect(err).ToNot(HaveOccurred())
Expand Down Expand Up @@ -151,8 +151,8 @@ var _ = Describe("DNS server", func() {
It("should not resolve", func() {
// given
var err error
dnsResolver.SetVIPs(map[string]string{
"service": "240.0.0.1",
dnsResolver.SetVIPs(vips.List{
vips.NewServiceEntry("service"): "240.0.0.1",
})
ip, err = dnsResolver.ForwardLookupFQDN("service.mesh")
Expect(err).ToNot(HaveOccurred())
Expand All @@ -177,7 +177,7 @@ var _ = Describe("DNS server", func() {

It("should not resolve when no vips", func() {
// given
dnsResolver.SetVIPs(map[string]string{})
dnsResolver.SetVIPs(vips.List{})

// when
client := new(dns.Client)
Expand All @@ -202,7 +202,7 @@ var _ = Describe("DNS server", func() {
// given
var err error
dnsResolver.SetVIPs(vips.List{
"my.service": "240.0.0.1",
vips.NewServiceEntry("my.service"): "240.0.0.1",
})
ip, err = dnsResolver.ForwardLookupFQDN("my.service.mesh")
Expect(err).ToNot(HaveOccurred())
Expand All @@ -229,7 +229,7 @@ var _ = Describe("DNS server", func() {
// given
var err error
dnsResolver.SetVIPs(vips.List{
"my-service_test-namespace_svc_80": "240.0.0.1",
vips.NewServiceEntry("my-service_test-namespace_svc_80"): "240.0.0.1",
})
ip, err = dnsResolver.ForwardLookupFQDN("my-service_test-namespace_svc_80.mesh")
Expect(err).ToNot(HaveOccurred())
Expand Down
56 changes: 53 additions & 3 deletions pkg/dns/vips/interfaces.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,65 @@
package vips

type List map[string]string
import (
"fmt"
"sort"
)

type EntryType int

const (
Service EntryType = iota
Host
)

type Entry struct {
Type EntryType `json:"type"`
Name string `json:"name"`
}

func (e Entry) String() string {
return fmt.Sprintf("%v:%s", e.Type, e.Name)
}

func (e Entry) MarshalText() (text []byte, err error) {
return []byte(e.String()), nil
}

func (e *Entry) UnmarshalText(text []byte) error {
_, err := fmt.Sscanf(string(text), "%v:%s", &e.Type, &e.Name)
return err
}

func NewHostEntry(host string) Entry {
return Entry{Host, host}
}

func NewServiceEntry(name string) Entry {
return Entry{Service, name}
}

type EntrySet map[Entry]bool

func (s EntrySet) ToArray() (entries []Entry) {
for entry := range s {
entries = append(entries, entry)
}
sort.SliceStable(entries, func(i, j int) bool {
return entries[i].String() < entries[j].String()
})
return
}

type List map[Entry]string

func (vips List) Append(other List) {
for k, v := range other {
vips[k] = v
}
}

func (vips List) FQDNsByIPs() map[string]string {
ipToDomain := map[string]string{}
func (vips List) FQDNsByIPs() map[string]Entry {
ipToDomain := map[string]Entry{}
for domain, ip := range vips {
ipToDomain[ip] = domain
}
Expand Down
Loading