diff --git a/pkg/dns/outbound.go b/pkg/dns/outbound.go index 07ce23f75a6c..e7833b49bc29 100644 --- a/pkg/dns/outbound.go +++ b/pkg/dns/outbound.go @@ -20,14 +20,15 @@ func VIPOutbounds( resourceKey model.ResourceKey, dataplanes []*core_mesh.DataplaneResource, zoneIngresses []*core_mesh.ZoneIngressResource, - vips vips.List, + vipList vips.List, externalServices []*core_mesh.ExternalServiceResource, ) []*mesh_proto.Dataplane_Networking_Outbound { type vipEntry struct { - ip string - port uint32 + ip string + port uint32 + entryType vips.EntryType } - serviceVIPMap := map[string]vipEntry{} + serviceVIPMap := map[string][]vipEntry{} services := []string{} for _, dataplane := range dataplanes { // backwards compatibility @@ -37,9 +38,9 @@ func VIPOutbounds( // Only add outbounds for services in the same mesh inService := service.Tags[mesh_proto.ServiceTag] if _, found := serviceVIPMap[inService]; !found { - vip, err := ForwardLookup(vips, inService) + vip, err := ForwardLookup(vipList, vips.NewServiceEntry(inService)) if err == nil { - serviceVIPMap[inService] = vipEntry{vip, VIPListenPort} + serviceVIPMap[inService] = append(serviceVIPMap[inService], vipEntry{vip, VIPListenPort, vips.Service}) services = append(services, inService) } } @@ -49,9 +50,9 @@ func VIPOutbounds( for _, inbound := range dataplane.Spec.GetNetworking().GetInbound() { inService := inbound.GetTags()[mesh_proto.ServiceTag] if _, found := serviceVIPMap[inService]; !found { - vip, err := ForwardLookup(vips, inService) + vip, err := ForwardLookup(vipList, vips.NewServiceEntry(inService)) if err == nil { - serviceVIPMap[inService] = vipEntry{vip, VIPListenPort} + serviceVIPMap[inService] = append(serviceVIPMap[inService], vipEntry{vip, VIPListenPort, vips.Service}) services = append(services, inService) } } @@ -65,9 +66,9 @@ func VIPOutbounds( // Only add outbounds for services in the same mesh inService := service.Tags[mesh_proto.ServiceTag] if _, found := serviceVIPMap[inService]; !found { - vip, err := ForwardLookup(vips, inService) + vip, err := ForwardLookup(vipList, vips.NewServiceEntry(inService)) if err == nil { - serviceVIPMap[inService] = vipEntry{vip, VIPListenPort} + serviceVIPMap[inService] = append(serviceVIPMap[inService], vipEntry{vip, VIPListenPort, vips.Service}) services = append(services, inService) } } @@ -77,8 +78,9 @@ func VIPOutbounds( for _, externalService := range externalServices { inService := externalService.Spec.Tags[mesh_proto.ServiceTag] + host := externalService.Spec.GetHost() if _, found := serviceVIPMap[inService]; !found { - vip, err := ForwardLookup(vips, inService) + vip1, err := ForwardLookup(vipList, vips.NewHostEntry(host)) if err == nil { port := externalService.Spec.GetPort() var p32 uint32 @@ -87,7 +89,19 @@ func VIPOutbounds( } else { p32 = uint32(p64) } - serviceVIPMap[inService] = vipEntry{vip, p32} + serviceVIPMap[inService] = append(serviceVIPMap[inService], vipEntry{vip1, p32, vips.Host}) + services = append(services, inService) + } + vip2, err := ForwardLookup(vipList, vips.NewServiceEntry(inService)) + if err == nil { + port := externalService.Spec.GetPort() + var p32 uint32 + if p64, err := strconv.ParseUint(port, 10, 32); err != nil { + p32 = VIPListenPort + } else { + p32 = uint32(p64) + } + serviceVIPMap[inService] = append(serviceVIPMap[inService], vipEntry{vip2, p32, vips.Service}) services = append(services, inService) } } @@ -96,30 +110,34 @@ func VIPOutbounds( sort.Strings(services) outbounds := []*mesh_proto.Dataplane_Networking_Outbound{} for _, service := range services { - entry := serviceVIPMap[service] - outbounds = append(outbounds, &mesh_proto.Dataplane_Networking_Outbound{ - Address: entry.ip, - Port: entry.port, - Tags: map[string]string{mesh_proto.ServiceTag: service}, - }) - - // todo (lobkovilya): backwards compatibility, could be deleted in the next major release Kuma 1.2.x - if entry.port != VIPListenPort { + entries := serviceVIPMap[service] + for _, entry := range entries { outbounds = append(outbounds, &mesh_proto.Dataplane_Networking_Outbound{ Address: entry.ip, - Port: VIPListenPort, + Port: entry.port, Tags: map[string]string{mesh_proto.ServiceTag: service}, }) + + if entry.entryType != vips.Host { + // todo (lobkovilya): backwards compatibility, could be deleted in the next major release Kuma 1.2.x + if entry.port != VIPListenPort { + outbounds = append(outbounds, &mesh_proto.Dataplane_Networking_Outbound{ + Address: entry.ip, + Port: VIPListenPort, + Tags: map[string]string{mesh_proto.ServiceTag: service}, + }) + } + } } } return outbounds } -func ForwardLookup(vips vips.List, service string) (string, error) { - ip, found := vips[service] +func ForwardLookup(vips vips.List, entry vips.Entry) (string, error) { + ip, found := vips[entry] if !found { - return "", errors.Errorf("service [%s] not found", service) + return "", errors.Errorf("entry name [%s] not found", entry.Name) } return ip, nil } diff --git a/pkg/dns/outbound_test.go b/pkg/dns/outbound_test.go index 0b2a4253d83e..4866e3f70a94 100644 --- a/pkg/dns/outbound_test.go +++ b/pkg/dns/outbound_test.go @@ -48,7 +48,7 @@ var _ = Describe("VIPOutbounds", func() { for i := 1; i <= 5; i++ { service := "service-" + strconv.Itoa(i) vip := fmt.Sprintf("240.0.0.%d", i) - vipList[service] = vip + vipList[vips.NewServiceEntry(service)] = vip dataplanes.Items = append(dataplanes.Items, &core_mesh.DataplaneResource{ Meta: &test_model.ResourceMeta{ @@ -104,8 +104,8 @@ var _ = Describe("VIPOutbounds", func() { // given vipList := vips.List{ - "service-a": "240.0.0.1", - "service-b": "240.0.0.2", + vips.NewServiceEntry("service-a"): "240.0.0.1", + vips.NewServiceEntry("service-b"): "240.0.0.2", } services := []*mesh_proto.Dataplane_Networking_Ingress_AvailableService{ { @@ -171,7 +171,7 @@ var _ = Describe("VIPOutbounds", func() { for i := 1; i <= 5; i++ { service := "service-" + strconv.Itoa(i) vip := fmt.Sprintf("240.0.0.%d", i) - vipList[service] = vip + vipList[vips.NewServiceEntry(service)] = vip otherDataplanes = append(otherDataplanes, &core_mesh.DataplaneResource{ Meta: &test_model.ResourceMeta{ @@ -229,9 +229,9 @@ var _ = Describe("VIPOutbounds", func() { }, }, } - vipList["first-external-service"] = "240.0.0.6" - vipList["second-external-service"] = "240.0.0.7" - vipList["third-external-service"] = "240.0.0.8" + vipList[vips.NewServiceEntry("first-external-service")] = "240.0.0.6" + vipList[vips.NewServiceEntry("second-external-service")] = "240.0.0.7" + vipList[vips.NewServiceEntry("third-external-service")] = "240.0.0.8" actual := &mesh_proto.Dataplane_Networking{} actual.Outbound = dns.VIPOutbounds(model.MetaToResourceKey(dataplane.Meta), otherDataplanes, nil, vipList, externalServices) @@ -299,10 +299,10 @@ var _ = Describe("VIPOutbounds", func() { } vipList := vips.List{ - "old-ingress-svc-1": "240.0.0.0", - "old-ingress-svc-2": "240.0.0.1", - "new-ingress-svc-1": "240.0.0.2", - "new-ingress-svc-2": "240.0.0.3", + vips.NewServiceEntry("old-ingress-svc-1"): "240.0.0.0", + vips.NewServiceEntry("old-ingress-svc-2"): "240.0.0.1", + vips.NewServiceEntry("new-ingress-svc-1"): "240.0.0.2", + vips.NewServiceEntry("new-ingress-svc-2"): "240.0.0.3", } otherDataplanes := []*core_mesh.DataplaneResource{{ diff --git a/pkg/dns/resolver/resolver.go b/pkg/dns/resolver/resolver.go index fe7690ab5c8a..610ab7651bd7 100644 --- a/pkg/dns/resolver/resolver.go +++ b/pkg/dns/resolver/resolver.go @@ -14,10 +14,7 @@ type DNSResolver interface { GetDomain() string SetVIPs(list vips.List) GetVIPs() vips.List - - ForwardLookup(service string) (string, error) ForwardLookupFQDN(name string) (string, error) - ReverseLookup(ip string) (string, error) } type dnsResolver struct { @@ -50,18 +47,6 @@ func (s *dnsResolver) GetVIPs() vips.List { return s.viplist } -func (s *dnsResolver) ForwardLookup(service string) (string, error) { - s.RLock() - defer s.RUnlock() - - ip, found := s.viplist[service] - - if !found { - return "", errors.Errorf("service [%s] not found in domain [%s].", service, s.domain) - } - return ip, nil -} - func (s *dnsResolver) ForwardLookupFQDN(name string) (string, error) { s.RLock() defer s.RUnlock() @@ -79,7 +64,7 @@ func (s *dnsResolver) ForwardLookupFQDN(name string) (string, error) { return "", err } - ip, found := s.viplist[service] + ip, found := s.viplist[vips.NewServiceEntry(service)] if !found { return "", errors.Errorf("service [%s] not found in domain [%s].", service, domain) } @@ -87,19 +72,6 @@ func (s *dnsResolver) ForwardLookupFQDN(name string) (string, error) { return ip, nil } -func (s *dnsResolver) ReverseLookup(ip string) (string, error) { - s.RLock() - defer s.RUnlock() - - for service, serviceIP := range s.viplist { - if serviceIP == ip { - return service + "." + s.domain, nil - } - } - - return "", errors.Errorf("IP [%s] not found", ip) -} - func (s *dnsResolver) domainFromName(name string) (string, error) { split := dns.SplitDomainName(name) if len(split) < 1 { diff --git a/pkg/dns/server_test.go b/pkg/dns/server_test.go index 11dd9c1bc59b..ba15742412cf 100644 --- a/pkg/dns/server_test.go +++ b/pkg/dns/server_test.go @@ -60,8 +60,8 @@ var _ = Describe("DNS server", func() { It("should resolve", func() { // given var err error - dnsResolver.SetVIPs(map[string]string{ - "service": "240.0.0.1", + dnsResolver.SetVIPs(vips.List{ + vips.NewServiceEntry("service"): "240.0.0.1", }) ip, err = dnsResolver.ForwardLookupFQDN("service.mesh") Expect(err).ToNot(HaveOccurred()) @@ -86,8 +86,8 @@ var _ = Describe("DNS server", func() { It("should resolve concurrent", func() { // given - dnsResolver.SetVIPs(map[string]string{ - "service": "240.0.0.1", + dnsResolver.SetVIPs(vips.List{ + vips.NewServiceEntry("service"): "240.0.0.1", }) ip, err := dnsResolver.ForwardLookupFQDN("service.mesh") Expect(err).ToNot(HaveOccurred()) @@ -118,8 +118,8 @@ var _ = Describe("DNS server", func() { It("should resolve IPv6 concurrent", func() { // given - dnsResolver.SetVIPs(map[string]string{ - "service": "fd00::1", + dnsResolver.SetVIPs(vips.List{ + vips.NewServiceEntry("service"): "fd00::1", }) ip, err := dnsResolver.ForwardLookupFQDN("service.mesh") Expect(err).ToNot(HaveOccurred()) @@ -151,8 +151,8 @@ var _ = Describe("DNS server", func() { It("should not resolve", func() { // given var err error - dnsResolver.SetVIPs(map[string]string{ - "service": "240.0.0.1", + dnsResolver.SetVIPs(vips.List{ + vips.NewServiceEntry("service"): "240.0.0.1", }) ip, err = dnsResolver.ForwardLookupFQDN("service.mesh") Expect(err).ToNot(HaveOccurred()) @@ -177,7 +177,7 @@ var _ = Describe("DNS server", func() { It("should not resolve when no vips", func() { // given - dnsResolver.SetVIPs(map[string]string{}) + dnsResolver.SetVIPs(vips.List{}) // when client := new(dns.Client) @@ -202,7 +202,7 @@ var _ = Describe("DNS server", func() { // given var err error dnsResolver.SetVIPs(vips.List{ - "my.service": "240.0.0.1", + vips.NewServiceEntry("my.service"): "240.0.0.1", }) ip, err = dnsResolver.ForwardLookupFQDN("my.service.mesh") Expect(err).ToNot(HaveOccurred()) @@ -229,7 +229,7 @@ var _ = Describe("DNS server", func() { // given var err error dnsResolver.SetVIPs(vips.List{ - "my-service_test-namespace_svc_80": "240.0.0.1", + vips.NewServiceEntry("my-service_test-namespace_svc_80"): "240.0.0.1", }) ip, err = dnsResolver.ForwardLookupFQDN("my-service_test-namespace_svc_80.mesh") Expect(err).ToNot(HaveOccurred()) diff --git a/pkg/dns/vips/interfaces.go b/pkg/dns/vips/interfaces.go index 8a1f60e4c92a..61de0f6c0366 100644 --- a/pkg/dns/vips/interfaces.go +++ b/pkg/dns/vips/interfaces.go @@ -1,6 +1,56 @@ package vips -type List map[string]string +import ( + "fmt" + "sort" +) + +type EntryType int + +const ( + Service EntryType = iota + Host +) + +type Entry struct { + Type EntryType `json:"type"` + Name string `json:"name"` +} + +func (e Entry) String() string { + return fmt.Sprintf("%v:%s", e.Type, e.Name) +} + +func (e Entry) MarshalText() (text []byte, err error) { + return []byte(e.String()), nil +} + +func (e *Entry) UnmarshalText(text []byte) error { + _, err := fmt.Sscanf(string(text), "%v:%s", &e.Type, &e.Name) + return err +} + +func NewHostEntry(host string) Entry { + return Entry{Host, host} +} + +func NewServiceEntry(name string) Entry { + return Entry{Service, name} +} + +type EntrySet map[Entry]bool + +func (s EntrySet) ToArray() (entries []Entry) { + for entry := range s { + entries = append(entries, entry) + } + sort.SliceStable(entries, func(i, j int) bool { + return entries[i].String() < entries[j].String() + }) + return +} + +type List map[Entry]string func (vips List) Append(other List) { for k, v := range other { @@ -8,8 +58,8 @@ func (vips List) Append(other List) { } } -func (vips List) FQDNsByIPs() map[string]string { - ipToDomain := map[string]string{} +func (vips List) FQDNsByIPs() map[string]Entry { + ipToDomain := map[string]Entry{} for domain, ip := range vips { ipToDomain[ip] = domain } diff --git a/pkg/dns/vips/persistence.go b/pkg/dns/vips/persistence.go index d7e61b1c9fbc..67f605b458d6 100644 --- a/pkg/dns/vips/persistence.go +++ b/pkg/dns/vips/persistence.go @@ -64,8 +64,8 @@ func (m *Persistence) Get() (global List, meshed map[string]List, errs error) { if resource.Spec.Config == "" { continue } - v := List{} - if err := json.Unmarshal([]byte(resource.Spec.Config), &v); err != nil { + v, err := m.unmarshal(resource.Spec.GetConfig()) + if err != nil { errs = multierr.Append(errs, err) continue } @@ -75,23 +75,39 @@ func (m *Persistence) Get() (global List, meshed map[string]List, errs error) { return } +func (m *Persistence) unmarshal(config string) (List, error) { + v := List{} + if err := json.Unmarshal([]byte(config), &v); err == nil { + return v, nil + } + // backwards compatibility + backwardCompatible := map[string]string{} + if err := json.Unmarshal([]byte(config), &backwardCompatible); err != nil { + return nil, err + } + v = List{} + for service, vip := range backwardCompatible { + v[NewServiceEntry(service)] = vip + } + return v, nil +} + func (m *Persistence) GetByMesh(mesh string) (List, error) { name := fmt.Sprintf(template, mesh) - vips := List{} resource := config_model.NewConfigResource() err := m.configManager.Get(context.Background(), resource, store.GetByKey(name, "")) if err != nil { if store.IsResourceNotFound(err) { - return vips, nil + return List{}, nil } return nil, err } if resource.Spec.Config == "" { - return vips, nil + return List{}, nil } - err = json.Unmarshal([]byte(resource.Spec.Config), &vips) + vips, err := m.unmarshal(resource.Spec.GetConfig()) if err != nil { return nil, errors.Wrap(err, "could not unmarshal") } diff --git a/pkg/dns/vips/persistence_test.go b/pkg/dns/vips/persistence_test.go index 38aa1719de9f..4f31341f625d 100644 --- a/pkg/dns/vips/persistence_test.go +++ b/pkg/dns/vips/persistence_test.go @@ -21,6 +21,7 @@ import ( type countingConfigManager struct { create int update int + updates []*system.ConfigResource delete int deleteAll int get int @@ -40,6 +41,7 @@ func (t *countingConfigManager) Create(ctx context.Context, resource *system.Con func (t *countingConfigManager) Update(ctx context.Context, resource *system.ConfigResource, optionsFunc ...core_store.UpdateOptionsFunc) error { t.update++ + t.updates = append(t.updates, resource) return nil } @@ -113,18 +115,18 @@ var _ = Describe("Meshed Persistence", func() { Expect(err).ToNot(HaveOccurred()) expected := vips.List{ - "backend": "240.0.0.1", - "backend_2": "240.0.1.1", - "backend_3": "240.0.2.1", - "frontend": "240.0.0.3", - "frontend_2": "240.0.1.3", - "frontend_3": "240.0.2.3", - "postgres": "240.0.0.0", - "postgres_2": "240.0.1.0", - "postgres_3": "240.0.2.0", - "redis": "240.0.0.2", - "redis_2": "240.0.1.2", - "redis_3": "240.0.2.2", + vips.NewServiceEntry("backend"): "240.0.0.1", + vips.NewServiceEntry("backend_2"): "240.0.1.1", + vips.NewServiceEntry("backend_3"): "240.0.2.1", + vips.NewServiceEntry("frontend"): "240.0.0.3", + vips.NewServiceEntry("frontend_2"): "240.0.1.3", + vips.NewServiceEntry("frontend_3"): "240.0.2.3", + vips.NewServiceEntry("postgres"): "240.0.0.0", + vips.NewServiceEntry("postgres_2"): "240.0.1.0", + vips.NewServiceEntry("postgres_3"): "240.0.2.0", + vips.NewServiceEntry("redis"): "240.0.0.2", + vips.NewServiceEntry("redis_2"): "240.0.1.2", + vips.NewServiceEntry("redis_3"): "240.0.2.2", } Expect(actual).To(Equal(expected)) }) @@ -133,10 +135,10 @@ var _ = Describe("Meshed Persistence", func() { actual, err := meshedPersistence.GetByMesh("mesh-2") Expect(err).ToNot(HaveOccurred()) expected := vips.List{ - "backend_2": "240.0.1.1", - "frontend_2": "240.0.1.3", - "postgres_2": "240.0.1.0", - "redis_2": "240.0.1.2", + vips.NewServiceEntry("backend_2"): "240.0.1.1", + vips.NewServiceEntry("frontend_2"): "240.0.1.3", + vips.NewServiceEntry("postgres_2"): "240.0.1.0", + vips.NewServiceEntry("redis_2"): "240.0.1.2", } Expect(actual).To(Equal(expected)) }) @@ -154,7 +156,7 @@ var _ = Describe("Meshed Persistence", func() { It("should create a new config", func() { vipsMesh1 := vips.List{ - "backend": "240.0.0.1", + vips.NewServiceEntry("backend"): "240.0.0.1", } err := meshedPersistence.Set("mesh-1", vipsMesh1) Expect(err).ToNot(HaveOccurred()) @@ -163,7 +165,7 @@ var _ = Describe("Meshed Persistence", func() { Expect(countingCm.update).To(Equal(0)) vipsMesh2 := vips.List{ - "frontend": "240.0.0.2", + vips.NewServiceEntry("frontend"): "240.0.0.2", } err = meshedPersistence.Set("mesh-2", vipsMesh2) Expect(err).ToNot(HaveOccurred()) @@ -174,7 +176,7 @@ var _ = Describe("Meshed Persistence", func() { It("should update existing config", func() { vipsMesh1 := vips.List{ - "backend": "240.0.0.1", + vips.NewServiceEntry("backend"): "240.0.0.1", } err := meshedPersistence.Set("mesh-1", vipsMesh1) Expect(err).ToNot(HaveOccurred()) @@ -182,7 +184,7 @@ var _ = Describe("Meshed Persistence", func() { Expect(countingCm.create).To(Equal(1)) Expect(countingCm.update).To(Equal(0)) - vipsMesh1["frontend"] = "240.0.0.2" + vipsMesh1[vips.NewServiceEntry("frontend")] = "240.0.0.2" err = meshedPersistence.Set("mesh-1", vipsMesh1) Expect(err).ToNot(HaveOccurred()) Expect(countingCm.get).To(Equal(2)) @@ -190,4 +192,72 @@ var _ = Describe("Meshed Persistence", func() { Expect(countingCm.update).To(Equal(1)) }) }) + + Context("Old and new configs at the same time", func() { + var countingCm *countingConfigManager + + BeforeEach(func() { + countingCm = &countingConfigManager{ + configs: map[string]*system.ConfigResource{ + "kuma-mesh-1-dns-vips": { + Meta: &model.ResourceMeta{Name: "kuma-mesh-1-dns-vips"}, + Spec: &config_proto.Config{Config: `{"backend":"240.0.0.1","frontend":"240.0.0.3","postgres":"240.0.0.0","redis":"240.0.0.2"}`}, + }, + "kuma-mesh-2-dns-vips": { + Meta: &model.ResourceMeta{Name: "kuma-mesh-2-dns-vips"}, + Spec: &config_proto.Config{Config: `{"0:backend_2":"240.0.1.1","0:frontend_2":"240.0.1.3","1:host.com":"240.0.1.4"}`}, + }, + }, + } + meshedPersistence = vips.NewPersistence(rm, countingCm) + }) + + It("should return global and meshed vips", func() { + // when + global, meshed, err := meshedPersistence.Get() + Expect(err).ToNot(HaveOccurred()) + + // then + expectedGlobal := vips.List{ + vips.NewServiceEntry("backend"): "240.0.0.1", + vips.NewServiceEntry("frontend"): "240.0.0.3", + vips.NewServiceEntry("postgres"): "240.0.0.0", + vips.NewServiceEntry("redis"): "240.0.0.2", + vips.NewServiceEntry("backend_2"): "240.0.1.1", + vips.NewServiceEntry("frontend_2"): "240.0.1.3", + vips.NewHostEntry("host.com"): "240.0.1.4", + } + Expect(global).To(Equal(expectedGlobal)) + // and then + expectedMesh1 := vips.List{ + vips.NewServiceEntry("backend"): "240.0.0.1", + vips.NewServiceEntry("frontend"): "240.0.0.3", + vips.NewServiceEntry("postgres"): "240.0.0.0", + vips.NewServiceEntry("redis"): "240.0.0.2", + } + Expect(meshed["mesh-1"]).To(Equal(expectedMesh1)) + // and then + expectedMesh2 := vips.List{ + vips.NewServiceEntry("backend_2"): "240.0.1.1", + vips.NewServiceEntry("frontend_2"): "240.0.1.3", + vips.NewHostEntry("host.com"): "240.0.1.4", + } + Expect(meshed["mesh-2"]).To(Equal(expectedMesh2)) + }) + + It("should update old version with new one", func() { + newVIPs := vips.List{ + vips.NewServiceEntry("backend"): "240.0.0.1", + vips.NewServiceEntry("frontend"): "240.0.0.3", + vips.NewServiceEntry("postgres"): "240.0.0.0", + vips.NewServiceEntry("redis"): "240.0.0.2", + vips.NewHostEntry("kuma.io"): "240.0.1.4", + } + err := meshedPersistence.Set("mesh-1", newVIPs) + Expect(err).ToNot(HaveOccurred()) + + Expect(countingCm.updates).To(HaveLen(1)) + Expect(countingCm.updates[0].Spec.Config).To(Equal(`{"0:backend":"240.0.0.1","0:frontend":"240.0.0.3","0:postgres":"240.0.0.0","0:redis":"240.0.0.2","1:kuma.io":"240.0.1.4"}`)) + }) + }) }) diff --git a/pkg/dns/vips/vips_suite_test.go b/pkg/dns/vips/vips_suite_test.go new file mode 100644 index 000000000000..5eed07a16dbf --- /dev/null +++ b/pkg/dns/vips/vips_suite_test.go @@ -0,0 +1,13 @@ +package vips_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestVIPs(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "VIPs Suite") +} diff --git a/pkg/dns/vips_allocator.go b/pkg/dns/vips_allocator.go index 5965270972d1..fcd120d8a38b 100644 --- a/pkg/dns/vips_allocator.go +++ b/pkg/dns/vips_allocator.go @@ -2,12 +2,13 @@ package dns import ( "context" - "sort" "time" "github.com/pkg/errors" "go.uber.org/multierr" + "github.com/kumahq/kuma/pkg/core/resources/store" + config_manager "github.com/kumahq/kuma/pkg/core/config/manager" "github.com/kumahq/kuma/pkg/dns/resolver" "github.com/kumahq/kuma/pkg/dns/vips" @@ -16,7 +17,6 @@ import ( "github.com/kumahq/kuma/pkg/core" core_mesh "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" "github.com/kumahq/kuma/pkg/core/resources/manager" - "github.com/kumahq/kuma/pkg/core/resources/store" ) var vipsAllocatorLog = core.Log.WithName("dns-vips-allocator") @@ -146,18 +146,8 @@ func (d *VIPsAllocator) newIPAM(initialVIPs vips.List) (IPAM, error) { return ipam, nil } -type ServiceSet map[string]bool - -func (s ServiceSet) ToArray() (services []string) { - for service := range s { - services = append(services, service) - } - sort.Strings(services) - return -} - -func BuildServiceSet(rm manager.ReadOnlyResourceManager, mesh string) (ServiceSet, error) { - serviceSet := make(map[string]bool) +func BuildServiceSet(rm manager.ReadOnlyResourceManager, mesh string) (vips.EntrySet, error) { + serviceSet := make(vips.EntrySet) dataplanes := core_mesh.DataplaneResourceList{} if err := rm.List(context.Background(), &dataplanes); err != nil { @@ -178,11 +168,11 @@ func BuildServiceSet(rm manager.ReadOnlyResourceManager, mesh string) (ServiceSe if service.Mesh != mesh { continue } - serviceSet[service.Tags[mesh_proto.ServiceTag]] = true + serviceSet[vips.NewServiceEntry(service.Tags[mesh_proto.ServiceTag])] = true } } else { for _, inbound := range dp.Spec.GetNetworking().GetInbound() { - serviceSet[inbound.GetService()] = true + serviceSet[vips.NewServiceEntry(inbound.GetService())] = true } } } @@ -197,7 +187,7 @@ func BuildServiceSet(rm manager.ReadOnlyResourceManager, mesh string) (ServiceSe if service.Mesh != mesh { continue } - serviceSet[service.Tags[mesh_proto.ServiceTag]] = true + serviceSet[vips.NewServiceEntry(service.Tags[mesh_proto.ServiceTag])] = true } } @@ -206,14 +196,15 @@ func BuildServiceSet(rm manager.ReadOnlyResourceManager, mesh string) (ServiceSe return nil, err } for _, es := range externalServices.Items { - serviceSet[es.Spec.GetService()] = true + serviceSet[vips.NewServiceEntry(es.Spec.GetService())] = true + serviceSet[vips.NewHostEntry(es.Spec.GetHost())] = true } return serviceSet, nil } -func UpdateMeshedVIPs(global, meshed vips.List, ipam IPAM, serviceSet ServiceSet) (updated bool, errs error) { - for _, service := range serviceSet.ToArray() { +func UpdateMeshedVIPs(global, meshed vips.List, ipam IPAM, entrySet vips.EntrySet) (updated bool, errs error) { + for _, service := range entrySet.ToArray() { _, found := meshed[service] if found { continue @@ -234,7 +225,7 @@ func UpdateMeshedVIPs(global, meshed vips.List, ipam IPAM, serviceSet ServiceSet vipsAllocatorLog.Info("adding", "service", service, "ip", ip) } for service, ip := range meshed { - if _, found := serviceSet[service]; !found { + if _, found := entrySet[service]; !found { updated = true _ = ipam.FreeIP(ip) delete(meshed, service) diff --git a/pkg/dns/vips_allocator_test.go b/pkg/dns/vips_allocator_test.go index 781eedbe71c1..2f46707ec7b6 100644 --- a/pkg/dns/vips_allocator_test.go +++ b/pkg/dns/vips_allocator_test.go @@ -96,8 +96,8 @@ var _ = Describe("VIP Allocator", func() { vipList, err = persistence.GetByMesh("mesh-2") Expect(err).ToNot(HaveOccurred()) - for _, service := range []string{"backend", "frontend", "web"} { - ip, err := r.ForwardLookup(service) + for _, service := range []string{"backend.mesh", "frontend.mesh", "web.mesh"} { + ip, err := r.ForwardLookupFQDN(service) Expect(err).ToNot(HaveOccurred()) Expect(ip).To(HavePrefix("240.0.0")) } @@ -111,8 +111,8 @@ var _ = Describe("VIP Allocator", func() { // we add VIPs directly to the 'persistence' object // that emulates situation when IPAM is fresh and doesn't aware of allocated VIPs err := persistence.Set("mesh-1", vips.List{ - "frontend": "240.0.0.0", - "backend": "240.0.0.1", + vips.NewServiceEntry("frontend"): "240.0.0.0", + vips.NewServiceEntry("backend"): "240.0.0.1", }) Expect(err).ToNot(HaveOccurred()) @@ -127,9 +127,9 @@ var _ = Describe("VIP Allocator", func() { Expect(err).ToNot(HaveOccurred()) // then Expect(vipList).To(Equal(vips.List{ - "frontend": "240.0.0.0", - "backend": "240.0.0.1", - "database": "240.0.0.2", + vips.NewServiceEntry("frontend"): "240.0.0.0", + vips.NewServiceEntry("backend"): "240.0.0.1", + vips.NewServiceEntry("database"): "240.0.0.2", })) }) @@ -259,13 +259,14 @@ var _ = Describe("BuildServiceSet", func() { Expect(err).ToNot(HaveOccurred()) // then - Expect(serviceSet).To(Equal(dns.ServiceSet{ - "backend": true, - "frontend": true, - "database": true, - "metrics": true, - "ingress-svc": true, - "es-backend": true, + Expect(serviceSet).To(Equal(vips.EntrySet{ + vips.NewServiceEntry("backend"): true, + vips.NewServiceEntry("frontend"): true, + vips.NewServiceEntry("database"): true, + vips.NewServiceEntry("metrics"): true, + vips.NewServiceEntry("ingress-svc"): true, + vips.NewServiceEntry("es-backend"): true, + vips.NewHostEntry("external.service.com"): true, })) }) }) @@ -276,9 +277,9 @@ var _ = Describe("UpdateMeshedVIPs", func() { vipsList := vips.List{} ipam, err := dns.NewSimpleIPAM("240.0.0.0/4") Expect(err).ToNot(HaveOccurred()) - serviceSet := dns.ServiceSet{ - "backend": true, - "frontend": true, + serviceSet := vips.EntrySet{ + vips.NewServiceEntry("backend"): true, + vips.NewServiceEntry("frontend"): true, } // when updated, err := dns.UpdateMeshedVIPs(vipsList, vipsList, ipam, serviceSet) @@ -287,21 +288,21 @@ var _ = Describe("UpdateMeshedVIPs", func() { Expect(err).ToNot(HaveOccurred()) Expect(updated).To(BeTrue()) Expect(vipsList).To(Equal(vips.List{ - "backend": "240.0.0.0", - "frontend": "240.0.0.1", + vips.NewServiceEntry("backend"): "240.0.0.0", + vips.NewServiceEntry("frontend"): "240.0.0.1", })) }) It("should free IP for deleted service", func() { // setup vipsList := vips.List{ - "backend": "240.0.0.0", - "frontend": "240.0.0.1", + vips.NewServiceEntry("backend"): "240.0.0.0", + vips.NewServiceEntry("frontend"): "240.0.0.1", } ipam, err := dns.NewSimpleIPAM("240.0.0.0/4") Expect(err).ToNot(HaveOccurred()) - serviceSet := dns.ServiceSet{ - "backend": true, + serviceSet := vips.EntrySet{ + vips.NewServiceEntry("backend"): true, } // when updated, err := dns.UpdateMeshedVIPs(vipsList, vipsList, ipam, serviceSet) @@ -309,21 +310,21 @@ var _ = Describe("UpdateMeshedVIPs", func() { // then Expect(updated).To(BeTrue()) Expect(vipsList).To(Equal(vips.List{ - "backend": "240.0.0.0", + vips.NewServiceEntry("backend"): "240.0.0.0", })) }) It("should return updated=false if nothing changed", func() { // setup vipsList := vips.List{ - "backend": "240.0.0.0", - "frontend": "240.0.0.1", + vips.NewServiceEntry("backend"): "240.0.0.0", + vips.NewServiceEntry("frontend"): "240.0.0.1", } ipam, err := dns.NewSimpleIPAM("240.0.0.0/4") Expect(err).ToNot(HaveOccurred()) - serviceSet := dns.ServiceSet{ - "backend": true, - "frontend": true, + serviceSet := vips.EntrySet{ + vips.NewServiceEntry("backend"): true, + vips.NewServiceEntry("frontend"): true, } // when updated, err := dns.UpdateMeshedVIPs(vipsList, vipsList, ipam, serviceSet) @@ -331,28 +332,28 @@ var _ = Describe("UpdateMeshedVIPs", func() { // then Expect(updated).To(BeFalse()) Expect(vipsList).To(Equal(vips.List{ - "backend": "240.0.0.0", - "frontend": "240.0.0.1", + vips.NewServiceEntry("backend"): "240.0.0.0", + vips.NewServiceEntry("frontend"): "240.0.0.1", })) }) It("should generate the same VIP for services across meshes", func() { // setup global := vips.List{ - "backend": "240.0.0.0", - "frontend": "240.0.0.1", - "database": "240.0.0.10", + vips.NewServiceEntry("backend"): "240.0.0.0", + vips.NewServiceEntry("frontend"): "240.0.0.1", + vips.NewServiceEntry("database"): "240.0.0.10", } meshed := vips.List{ - "backend": "240.0.0.0", - "frontend": "240.0.0.1", + vips.NewServiceEntry("backend"): "240.0.0.0", + vips.NewServiceEntry("frontend"): "240.0.0.1", } ipam, err := dns.NewSimpleIPAM("240.0.0.0/4") Expect(err).ToNot(HaveOccurred()) - serviceSet := dns.ServiceSet{ - "backend": true, - "frontend": true, - "database": true, + serviceSet := vips.EntrySet{ + vips.NewServiceEntry("backend"): true, + vips.NewServiceEntry("frontend"): true, + vips.NewServiceEntry("database"): true, } // when updated, err := dns.UpdateMeshedVIPs(global, meshed, ipam, serviceSet) @@ -360,9 +361,9 @@ var _ = Describe("UpdateMeshedVIPs", func() { // then Expect(updated).To(BeTrue()) Expect(meshed).To(Equal(vips.List{ - "backend": "240.0.0.0", - "frontend": "240.0.0.1", - "database": "240.0.0.10", + vips.NewServiceEntry("backend"): "240.0.0.0", + vips.NewServiceEntry("frontend"): "240.0.0.1", + vips.NewServiceEntry("database"): "240.0.0.10", })) }) }) diff --git a/pkg/dns/vips_synchronizer_test.go b/pkg/dns/vips_synchronizer_test.go index ecf8cfeb5e50..dd2bc4867d94 100644 --- a/pkg/dns/vips_synchronizer_test.go +++ b/pkg/dns/vips_synchronizer_test.go @@ -78,18 +78,18 @@ var _ = Describe("DNS sync", func() { It("should sync web to DNS resolver and to the follower", func() { // then service "web" is synchronized to DNS Resolver Eventually(func() error { - _, err := dnsResolver.ForwardLookup("web") + _, err := dnsResolver.ForwardLookupFQDN("web.mesh") return err }, "5s").ShouldNot(HaveOccurred()) - ip, _ := dnsResolver.ForwardLookup("web") + ip, _ := dnsResolver.ForwardLookupFQDN("web.mesh") Expect(ip).Should(HavePrefix("240.0.0")) // and replicated to a follower Eventually(func() error { - _, err := dnsResolverFollower.ForwardLookup("web") + _, err := dnsResolverFollower.ForwardLookupFQDN("web.mesh") return err }, "5s").ShouldNot(HaveOccurred()) - ip2, _ := dnsResolverFollower.ForwardLookup("web") + ip2, _ := dnsResolverFollower.ForwardLookupFQDN("web.mesh") Expect(ip).To(Equal(ip2)) }) @@ -125,18 +125,18 @@ var _ = Describe("DNS sync", func() { // then service "backend" is synchronized to DNS Resolver Eventually(func() error { - _, err := dnsResolver.ForwardLookup("backend") + _, err := dnsResolver.ForwardLookupFQDN("backend.mesh") return err }, "5s").ShouldNot(HaveOccurred()) - ip, _ := dnsResolver.ForwardLookup("backend") + ip, _ := dnsResolver.ForwardLookupFQDN("backend.mesh") Expect(ip).Should(HavePrefix("240.0.0")) // and replicated to a follower Eventually(func() error { - _, err := dnsResolverFollower.ForwardLookup("backend") + _, err := dnsResolverFollower.ForwardLookupFQDN("backend.mesh") return err }, "5s").ShouldNot(HaveOccurred()) - ip2, _ := dnsResolverFollower.ForwardLookup("backend") + ip2, _ := dnsResolverFollower.ForwardLookupFQDN("backend.mesh") Expect(ip).To(Equal(ip2)) }) @@ -147,13 +147,13 @@ var _ = Describe("DNS sync", func() { // then service "web" is removed from DNS Resolver Eventually(func() error { - _, err := dnsResolver.ForwardLookup("web") + _, err := dnsResolver.ForwardLookupFQDN("web.mesh") return err }, "5s").Should(MatchError("service [web] not found in domain [mesh].")) // and replicated to a follower Eventually(func() error { - _, err := dnsResolverFollower.ForwardLookup("web") + _, err := dnsResolverFollower.ForwardLookupFQDN("web.mesh") return err }, "5s").Should(MatchError("service [web] not found in domain [mesh].")) }) diff --git a/pkg/plugins/runtime/universal/outbound/outbound_test.go b/pkg/plugins/runtime/universal/outbound/outbound_test.go index ff45011fe9e1..ca3cd4ee3f1b 100644 --- a/pkg/plugins/runtime/universal/outbound/outbound_test.go +++ b/pkg/plugins/runtime/universal/outbound/outbound_test.go @@ -5,6 +5,7 @@ import ( "time" "github.com/kumahq/kuma/pkg/dns/resolver" + "github.com/kumahq/kuma/pkg/dns/vips" "github.com/kumahq/kuma/pkg/plugins/runtime/universal/outbound" . "github.com/onsi/ginkgo" @@ -40,9 +41,9 @@ var _ = Describe("UpdateOutbound", func() { Expect(err).ToNot(HaveOccurred()) // and r := resolver.NewDNSResolver("mesh") - r.SetVIPs(map[string]string{ - "service-1": "240.0.0.1", - "service-2": "240.0.0.2", + r.SetVIPs(vips.List{ + vips.NewServiceEntry("service-1"): "240.0.0.1", + vips.NewServiceEntry("service-2"): "240.0.0.2", }) // and vipOutboundsReconciler, err = outbound.NewVIPOutboundsReconciler(rm, rm, r, time.Second) diff --git a/pkg/xds/generator/dns_generator.go b/pkg/xds/generator/dns_generator.go index 9f23bdd17388..bb68ca768e67 100644 --- a/pkg/xds/generator/dns_generator.go +++ b/pkg/xds/generator/dns_generator.go @@ -5,6 +5,8 @@ import ( "github.com/asaskevich/govalidator" + "github.com/kumahq/kuma/pkg/dns/vips" + mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" core_xds "github.com/kumahq/kuma/pkg/core/xds" xds_context "github.com/kumahq/kuma/pkg/xds/context" @@ -51,16 +53,21 @@ func (g DNSGenerator) computeVIPs(ctx xds_context.Context, proxy *core_xds.Proxy domainsByIPs := ctx.ControlPlane.DNSResolver.GetVIPs().FQDNsByIPs() meshedVips := map[string]string{} for _, outbound := range proxy.Dataplane.Spec.GetNetworking().GetOutbound() { - if domain, ok := domainsByIPs[outbound.Address]; ok { - // add regular .mesh domain - meshedVips[domain+"."+ctx.ControlPlane.DNSResolver.GetDomain()] = outbound.Address - meshedVips[strings.ReplaceAll(domain, "_", ".")+"."+ctx.ControlPlane.DNSResolver.GetDomain()] = outbound.Address - // add hostname from address in external service - endpoints := proxy.Routing.OutboundTargets[outbound.Tags[mesh_proto.ServiceTag]] - for _, endpoint := range endpoints { - if govalidator.IsDNSName(endpoint.Target) { - if endpoint.ExternalService != nil && endpoint.Target != "" { - meshedVips[endpoint.Target] = outbound.Address + if entry, ok := domainsByIPs[outbound.Address]; ok { + switch entry.Type { + case vips.Service: + domain := outbound.Tags[mesh_proto.ServiceTag] + // add regular .mesh domain + meshedVips[domain+"."+ctx.ControlPlane.DNSResolver.GetDomain()] = outbound.Address + meshedVips[strings.ReplaceAll(domain, "_", ".")+"."+ctx.ControlPlane.DNSResolver.GetDomain()] = outbound.Address + case vips.Host: + // add hostname from address in external service + endpoints := proxy.Routing.OutboundTargets[outbound.Tags[mesh_proto.ServiceTag]] + for _, endpoint := range endpoints { + if govalidator.IsDNSName(endpoint.Target) { + if endpoint.ExternalService != nil && endpoint.Target != "" { + meshedVips[endpoint.Target] = outbound.Address + } } } } diff --git a/pkg/xds/generator/dns_generator_test.go b/pkg/xds/generator/dns_generator_test.go index d339beffb0dd..026186f21dec 100644 --- a/pkg/xds/generator/dns_generator_test.go +++ b/pkg/xds/generator/dns_generator_test.go @@ -8,6 +8,8 @@ import ( . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" + "github.com/kumahq/kuma/pkg/dns/vips" + mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" mesh_core "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" model "github.com/kumahq/kuma/pkg/core/xds" @@ -33,9 +35,9 @@ var _ = Describe("DNSGenerator", func() { gen := &generator.DNSGenerator{} dnsResolver := resolver.NewDNSResolver("mesh") - dnsResolver.SetVIPs(map[string]string{ - "backend_test-ns_svc_8080": "240.0.0.0", - "httpbin": "240.0.0.1", + dnsResolver.SetVIPs(vips.List{ + vips.NewServiceEntry("backend_test-ns_svc_8080"): "240.0.0.0", + vips.NewServiceEntry("httpbin"): "240.0.0.1", }) ctx := xds_context.Context{ ConnectionInfo: xds_context.ConnectionInfo{ diff --git a/test/e2e/externalservices/externalservices_universal.go b/test/e2e/externalservices/externalservices_universal.go index 4a57f1f70577..b82a195bad28 100644 --- a/test/e2e/externalservices/externalservices_universal.go +++ b/test/e2e/externalservices/externalservices_universal.go @@ -70,6 +70,7 @@ networking: err = NewClusterSetup(). Install(externalservice.Install(externalservice.HttpServer, externalservice.UniversalAppEchoServer)). Install(externalservice.Install(externalservice.HttpsServer, externalservice.UniversalAppHttpsEchoServer)). + Install(externalservice.Install("http-server-80-81", externalservice.UniversalAppEchoServer, externalservice.UniversalAppEchoServer81)). Install(DemoClientUniversal(AppModeDemoClient, "default", demoClientToken, WithTransparentProxy(true))). Setup(cluster) Expect(err).ToNot(HaveOccurred()) @@ -109,6 +110,56 @@ networking: Expect(stdout).ToNot(ContainSubstring("HTTPS")) }) + It("should route to external-service", func() { + err := YamlUniversal(fmt.Sprintf(externalService, + es1, es1, + "kuma-3_externalservice-http-server-80-81:80", + "false", ""))(cluster) + Expect(err).ToNot(HaveOccurred()) + + err = YamlUniversal(fmt.Sprintf(externalService, + es2, es2, + "kuma-3_externalservice-http-server-80-81:81", + "false", ""))(cluster) + Expect(err).ToNot(HaveOccurred()) + + // when access the first external service with .mesh + stdout, _, err := cluster.ExecWithRetries("", "", "demo-client", + "curl", "-v", "-m", "3", "--fail", "external-service-1.mesh") + // then + Expect(err).ToNot(HaveOccurred()) + Expect(stdout).To(ContainSubstring("HTTP/1.1 200 OK")) + Expect(stdout).To(ContainSubstring("Echo 80")) + Expect(stdout).ToNot(ContainSubstring("HTTPS")) + + // when access the first external service using hostname + stdout, _, err = cluster.ExecWithRetries("", "", "demo-client", + "curl", "-v", "-m", "3", "--fail", "kuma-3_externalservice-http-server-80-81:80") + // then + Expect(err).ToNot(HaveOccurred()) + Expect(stdout).To(ContainSubstring("HTTP/1.1 200 OK")) + Expect(stdout).To(ContainSubstring("Echo 80")) + Expect(stdout).ToNot(ContainSubstring("HTTPS")) + + // when access the second external service name using .mesh + stdout, _, err = cluster.ExecWithRetries("", "", "demo-client", + "curl", "-v", "-m", "3", "--fail", "external-service-2.mesh") + // then + Expect(err).ToNot(HaveOccurred()) + Expect(stdout).To(ContainSubstring("HTTP/1.1 200 OK")) + Expect(stdout).To(ContainSubstring("Echo 81")) + Expect(stdout).ToNot(ContainSubstring("HTTPS")) + + // when access the second external service using the same hostname as first but with different port + stdout, _, err = cluster.ExecWithRetries("", "", "demo-client", + "curl", "-v", "-m", "3", "--fail", "kuma-3_externalservice-http-server-80-81:81") + // then + Expect(err).ToNot(HaveOccurred()) + Expect(stdout).To(ContainSubstring("HTTP/1.1 200 OK")) + Expect(stdout).To(ContainSubstring("Echo 81")) + Expect(stdout).ToNot(ContainSubstring("HTTPS")) + }) + It("should route to external-service over tls", func() { // when set invalid certificate otherCert := "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURMRENDQWhTZ0F3SUJBZ0lRSGRQaHhPZlhnV3VOeG9GbFYvRXdxVEFOQmdrcWhraUc5dzBCQVFzRkFEQVAKTVEwd0N3WURWUVFERXdScmRXMWhNQjRYRFRJd01Ea3hOakV5TWpnME5Gb1hEVE13TURreE5ERXlNamcwTkZvdwpEekVOTUFzR0ExVUVBeE1FYTNWdFlUQ0NBU0l3RFFZSktvWklodmNOQVFFQkJRQURnZ0VQQURDQ0FRb0NnZ0VCCkFPWkdiV2hTbFFTUnhGTnQ1cC8yV0NLRnlIWjNDdXdOZ3lMRVA3blM0Wlh5a3hzRmJZU3VWM2JJZ0Y3YlQvdXEKYTVRaXJlK0M2MGd1aEZicExjUGgyWjZVZmdJZDY5R2xRekhNVlljbUxHalZRdXlBdDRGTU1rVGZWRWw1STRPYQorMml0M0J2aWhWa0toVXo4eTVSUjVLYnFKZkdwNFoyMEZoNmZ0dG9DRmJlT0RtdkJzWUpGbVVRUytpZm95TVkvClAzUjAzU3U3ZzVpSXZuejd0bWt5ZG9OQzhuR1JEemRENUM4Zkp2clZJMVVYNkpSR3lMS3Q0NW9RWHQxbXhLMTAKNUthTjJ6TlYyV3RIc2FKcDlid3JQSCtKaVpHZVp5dnVoNVV3ckxkSENtcUs3c205VG9kR3p0VVpZMFZ6QWM0cQprWVZpWFk4Z1VqZk5tK2NRclBPMWtOOENBd0VBQWFPQmd6Q0JnREFPQmdOVkhROEJBZjhFQkFNQ0FxUXdIUVlEClZSMGxCQll3RkFZSUt3WUJCUVVIQXdFR0NDc0dBUVVGQndNQk1BOEdBMVVkRXdFQi93UUZNQU1CQWY4d0hRWUQKVlIwT0JCWUVGR01EQlBQaUJGSjNtdjJvQTlDVHFqZW1GVFYyTUI4R0ExVWRFUVFZTUJhQ0NXeHZZMkZzYUc5egpkSUlKYkc5allXeG9iM04wTUEwR0NTcUdTSWIzRFFFQkN3VUFBNElCQVFDLzE3UXdlT3BHZGIxTUVCSjhYUEc3CjNzSy91dG9XTFgxdGpmOFN1MURnYTZDRFQvZVRXSFpyV1JmODFLT1ZZMDdkbGU1U1JJREsxUWhmYkdHdEZQK1QKdlprcm9vdXNJOVVTMmFDV2xrZUNaV0dUbnF2TG1Eb091anFhZ0RvS1JSdWs0bVFkdE5Ob254aUwvd1p0VEZLaQorMWlOalVWYkxXaURYZEJMeG9SSVZkTE96cWIvTU54d0VsVXlhVERBa29wUXlPV2FURGtZUHJHbWFXamNzZlBHCmFPS293MHplK3pIVkZxVEhiam5DcUVWM2huc1V5UlV3c0JsbjkrakRKWGd3Wk0vdE1sVkpyWkNoMFNsZTlZNVoKTU9CMGZDZjZzVE1OUlRHZzVMcGw2dUlZTS81SU5wbUhWTW8zbjdNQlNucEVEQVVTMmJmL3VvNWdJaXE2WENkcAotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==" diff --git a/test/framework/deployments/externalservice/deployment.go b/test/framework/deployments/externalservice/deployment.go index ff28d8f90369..4db617aa79a6 100644 --- a/test/framework/deployments/externalservice/deployment.go +++ b/test/framework/deployments/externalservice/deployment.go @@ -16,6 +16,8 @@ type Deployment interface { ExternalService } +type Command []string + const ( DeploymentName = "externalservice-" HttpServer = "http-server" @@ -26,20 +28,24 @@ func From(cluster framework.Cluster, name string) ExternalService { return cluster.Deployment(DeploymentName + name).(ExternalService) } -func Install(name string, args []string) framework.InstallFunc { +func Install(name string, commands ...Command) framework.InstallFunc { return func(cluster framework.Cluster) error { var deployment Deployment + if len(commands) < 1 { + return errors.New("command list can't be empty") + } switch cluster.(type) { case *framework.K8sCluster: deployment = &k8SDeployment{ name: name, - args: args, + cmd: commands[0], } case *framework.UniversalCluster: deployment = &universalDeployment{ - name: name, - args: args, - ports: map[string]string{}, + name: name, + commands: commands, + ports: map[string]string{}, + verbose: cluster.Verbose(), } default: return errors.New("invalid cluster") diff --git a/test/framework/deployments/externalservice/kubernetes.go b/test/framework/deployments/externalservice/kubernetes.go index 0f4f9137b4d3..cee70e34c258 100644 --- a/test/framework/deployments/externalservice/kubernetes.go +++ b/test/framework/deployments/externalservice/kubernetes.go @@ -13,7 +13,7 @@ import ( type k8SDeployment struct { ip string name string - args []string + cmd Command } var _ Deployment = &k8SDeployment{} @@ -78,7 +78,7 @@ func (k *k8SDeployment) GetExternalAppAddress() string { } func (k *k8SDeployment) GetCert() string { - // We do not implement Runtime Ceritficate injection on K8s + // We do not implement Runtime Certificate injection on K8s // The functionality is test on Universal which is good for now panic("not implemented") } diff --git a/test/framework/deployments/externalservice/universal.go b/test/framework/deployments/externalservice/universal.go index b755bdaf65b7..d7fa6e84723b 100644 --- a/test/framework/deployments/externalservice/universal.go +++ b/test/framework/deployments/externalservice/universal.go @@ -20,17 +20,19 @@ type universalDeployment struct { ports map[string]string name string cert string - args []string + commands []Command app *framework.SshApp + verbose bool } var _ Deployment = &universalDeployment{} -var UniversalAppEchoServer = []string{"ncat", "-lk", "-p", "80", "--sh-exec", "'echo \"HTTP/1.1 200 OK\n\n Echo\n\"'"} -var UniversalAppHttpsEchoServer = []string{"ncat", +var UniversalAppEchoServer = Command([]string{"ncat", "-lk", "-p", "80", "--sh-exec", "'echo \"HTTP/1.1 200 OK\n\n Echo 80\n\"'"}) +var UniversalAppEchoServer81 = Command([]string{"ncat", "-lk", "-p", "81", "--sh-exec", "'echo \"HTTP/1.1 200 OK\n\n Echo 81\n\"'"}) +var UniversalAppHttpsEchoServer = Command([]string{"ncat", "-lk", "-p", "443", "--ssl", "--ssl-cert", "/server-cert.pem", "--ssl-key", "/server-key.pem", - "--sh-exec", "'echo \"HTTP/1.1 200 OK\n\n HTTPS Echo\n\"'"} + "--sh-exec", "'echo \"HTTP/1.1 200 OK\n\n HTTPS Echo\n\"'"}) func (u *universalDeployment) Name() string { return DeploymentName + u.name @@ -65,7 +67,6 @@ func (u *universalDeployment) Deploy(cluster framework.Cluster) error { u.ip = ip u.container = container - verbose := false port := u.ports["22"] env := []string{} @@ -75,22 +76,23 @@ func (u *universalDeployment) Deploy(cluster framework.Cluster) error { return err } - err = framework.NewSshApp(verbose, port, env, []string{"printf ", "--", "\"" + cert + "\"", ">", "/server-cert.pem"}).Run() + err = framework.NewSshApp(u.verbose, port, env, []string{"printf ", "--", "\"" + cert + "\"", ">", "/server-cert.pem"}).Run() if err != nil { panic(err) } - err = framework.NewSshApp(verbose, port, env, []string{"printf ", "--", "\"" + key + "\"", ">", "/server-key.pem"}).Run() + err = framework.NewSshApp(u.verbose, port, env, []string{"printf ", "--", "\"" + key + "\"", ">", "/server-key.pem"}).Run() if err != nil { panic(err) } u.cert = cert - u.app = framework.NewSshApp(verbose, port, env, u.args) - - err = u.app.Start() - if err != nil { - return err + for _, arg := range u.commands { + u.app = framework.NewSshApp(u.verbose, port, env, arg) + err = u.app.Start() + if err != nil { + return err + } } return nil diff --git a/test/framework/interface.go b/test/framework/interface.go index 5809b75a8dd2..72334af4a859 100644 --- a/test/framework/interface.go +++ b/test/framework/interface.go @@ -296,6 +296,7 @@ type Cluster interface { DeleteDeployment(name string) error WithTimeout(timeout time.Duration) Cluster WithRetries(retries int) Cluster + Verbose() bool // K8s GetKubectlOptions(namespace ...string) *k8s.KubectlOptions diff --git a/test/framework/k8s_cluster.go b/test/framework/k8s_cluster.go index 25b3047638d0..19e12a7fd4ab 100644 --- a/test/framework/k8s_cluster.go +++ b/test/framework/k8s_cluster.go @@ -88,6 +88,10 @@ func (c *K8sCluster) WithTimeout(timeout time.Duration) Cluster { return c } +func (c *K8sCluster) Verbose() bool { + return c.verbose +} + func (c *K8sCluster) WithRetries(retries int) Cluster { c.defaultRetries = retries diff --git a/test/framework/k8s_clusters.go b/test/framework/k8s_clusters.go index ca674683f9cd..78844b961980 100644 --- a/test/framework/k8s_clusters.go +++ b/test/framework/k8s_clusters.go @@ -62,6 +62,10 @@ func (cs *K8sClusters) WithTimeout(timeout time.Duration) Cluster { return cs } +func (c *K8sClusters) Verbose() bool { + return c.verbose +} + func (cs *K8sClusters) WithRetries(retries int) Cluster { for _, c := range cs.clusters { c.WithRetries(retries) diff --git a/test/framework/universal_cluster.go b/test/framework/universal_cluster.go index a956b91eb11b..e39fd8893099 100644 --- a/test/framework/universal_cluster.go +++ b/test/framework/universal_cluster.go @@ -70,6 +70,10 @@ func (c *UniversalCluster) DismissCluster() (errs error) { return } +func (c *UniversalCluster) Verbose() bool { + return c.verbose +} + func (c *UniversalCluster) DeployKuma(mode string, fs ...DeployOptionsFunc) error { c.controlplane = NewUniversalControlPlane(c.t, mode, c.name, c, c.verbose) opts := newDeployOpt(fs...) diff --git a/test/framework/universal_clusters.go b/test/framework/universal_clusters.go index c3b558a88eb8..242db2273a5a 100644 --- a/test/framework/universal_clusters.go +++ b/test/framework/universal_clusters.go @@ -47,6 +47,10 @@ func (cs *UniversalClusters) WithTimeout(timeout time.Duration) Cluster { return cs } +func (cs *UniversalClusters) Verbose() bool { + return cs.verbose +} + func (cs *UniversalClusters) WithRetries(retries int) Cluster { for _, c := range cs.clusters { c.WithRetries(retries)