From 3dc180d3dc21c7db64b50fc337215645c5fc1fc8 Mon Sep 17 00:00:00 2001 From: Andre Baptista Aguas Date: Sun, 25 Aug 2024 21:09:31 +0200 Subject: [PATCH] Add reverse proxy support Problem K8GB reads IP addresses from `Ingress.Status.LoadBalancer.Ingress` or from `Service.Status.LoadBalancer.Ingress` for ingress configured with Kubernetes Ingress and Istio Virtual Service, respectively. The IP addresses exposed by these resources are the IP addresses exposed by the Kubernetes Cluster. However, in some setups the clients do not route their traffic to these IP addresses because the cluster is behind a reverse proxy. Solution To support this setup, K8GB should expose DNS records with the IP address of the reverse proxy. Since the address is unknown to the cluster the K8GB administrator must provide it via configuration. This PR adds to K8GB the capability to read IP address from an annotation `k8gb.io/external-ips` on Ingress and Service resources. Examples ``` apiVersion: networking.k8s.io/v1 kind: Ingress metadata: labels: app: ingress annotations: k8gb.io/external-ips: "185.199.110.153" ``` ``` apiVersion: v1 kind: Service metadata: name: istio-ingressgateway namespace: istio-ingress labels: app: istio-ingressgateway annotations: k8gb.io/external-ips: "185.199.110.153,185.199.109.153" ``` Fixes #1275 Signed-off-by: Andre Baptista Aguas --- controllers/refresolver/ingress/ingress.go | 13 ++++++++- .../refresolver/ingress/ingress_test.go | 15 +++++++++++ .../ingress_annotation_multiple_ips.yaml | 27 +++++++++++++++++++ .../testdata/ingress_annotation_no_ips.yaml | 27 +++++++++++++++++++ .../ingress_annotation_single_ip.yaml | 27 +++++++++++++++++++ .../istiovirtualservice.go | 12 ++++++++- .../istiovirtualservice_test.go | 15 +++++++++++ ...istio_service_annotation_multiple_ips.yaml | 22 +++++++++++++++ .../istio_service_annotation_no_ips.yaml | 22 +++++++++++++++ .../istio_service_annotation_single_ip.yaml | 22 +++++++++++++++ 10 files changed, 200 insertions(+), 2 deletions(-) create mode 100644 controllers/refresolver/ingress/testdata/ingress_annotation_multiple_ips.yaml create mode 100644 controllers/refresolver/ingress/testdata/ingress_annotation_no_ips.yaml create mode 100644 controllers/refresolver/ingress/testdata/ingress_annotation_single_ip.yaml create mode 100644 controllers/refresolver/istiovirtualservice/testdata/istio_service_annotation_multiple_ips.yaml create mode 100644 controllers/refresolver/istiovirtualservice/testdata/istio_service_annotation_no_ips.yaml create mode 100644 controllers/refresolver/istiovirtualservice/testdata/istio_service_annotation_single_ip.yaml diff --git a/controllers/refresolver/ingress/ingress.go b/controllers/refresolver/ingress/ingress.go index 3eb59119f4..2892878e40 100644 --- a/controllers/refresolver/ingress/ingress.go +++ b/controllers/refresolver/ingress/ingress.go @@ -22,6 +22,7 @@ import ( "context" "fmt" "reflect" + "strings" k8gbv1beta1 "github.com/k8gb-io/k8gb/api/v1beta1" "github.com/k8gb-io/k8gb/controllers/logging" @@ -35,6 +36,11 @@ import ( var log = logging.Logger() +const ( + // comma separated list of external IP addresses + externalIPsAnnotation = "k8gb.io/external-ips" +) + type ReferenceResolver struct { ingress *netv1.Ingress } @@ -158,8 +164,13 @@ func (rr *ReferenceResolver) GetServers() ([]*k8gbv1beta1.Server, error) { // GetGslbExposedIPs retrieves the load balancer IP address of the GSLB func (rr *ReferenceResolver) GetGslbExposedIPs(edgeDNSServers utils.DNSList) ([]string, error) { - gslbIngressIPs := []string{} + // fetch the IP addresses of the reverse proxy from an annotation if it exists + if ingressIPsFromAnnotation, ok := rr.ingress.Annotations[externalIPsAnnotation]; ok { + return strings.Split(ingressIPsFromAnnotation, ","), nil + } + // if there is no annotation -> fetch the IP addresses from the Status of the Ingress resource + gslbIngressIPs := []string{} for _, ip := range rr.ingress.Status.LoadBalancer.Ingress { if len(ip.IP) > 0 { gslbIngressIPs = append(gslbIngressIPs, ip.IP) diff --git a/controllers/refresolver/ingress/ingress_test.go b/controllers/refresolver/ingress/ingress_test.go index e288101e81..3f4051ffbc 100644 --- a/controllers/refresolver/ingress/ingress_test.go +++ b/controllers/refresolver/ingress/ingress_test.go @@ -115,6 +115,21 @@ func TestGetGslbExposedIPs(t *testing.T) { ingressYaml: "./testdata/ingress_multiple_ips.yaml", expectedIPs: []string{"10.0.0.1", "10.0.0.2"}, }, + { + name: "annotation with no exposed IPs", + ingressYaml: "./testdata/ingress_annotation_no_ips.yaml", + expectedIPs: []string{""}, + }, + { + name: "annotation with single exposed IP", + ingressYaml: "./testdata/ingress_annotation_single_ip.yaml", + expectedIPs: []string{"185.199.110.153"}, + }, + { + name: "annotation with multiple exposed IPs", + ingressYaml: "./testdata/ingress_annotation_multiple_ips.yaml", + expectedIPs: []string{"185.199.110.153", "185.199.109.153"}, + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { diff --git a/controllers/refresolver/ingress/testdata/ingress_annotation_multiple_ips.yaml b/controllers/refresolver/ingress/testdata/ingress_annotation_multiple_ips.yaml new file mode 100644 index 0000000000..a18b4c77e4 --- /dev/null +++ b/controllers/refresolver/ingress/testdata/ingress_annotation_multiple_ips.yaml @@ -0,0 +1,27 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + labels: + app: ingress-referenced + annotations: + k8gb.io/external-ips: "185.199.110.153,185.199.109.153" + name: ingress-referenced + namespace: test-gslb + resourceVersion: "999" +spec: + ingressClassName: nginx + rules: + - host: ingress-referenced.cloud.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: referenced + port: + name: http +status: + loadBalancer: + ingress: + - ip: 10.0.0.1 diff --git a/controllers/refresolver/ingress/testdata/ingress_annotation_no_ips.yaml b/controllers/refresolver/ingress/testdata/ingress_annotation_no_ips.yaml new file mode 100644 index 0000000000..f2aeb1d883 --- /dev/null +++ b/controllers/refresolver/ingress/testdata/ingress_annotation_no_ips.yaml @@ -0,0 +1,27 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + labels: + app: ingress-referenced + annotations: + k8gb.io/external-ips: "" + name: ingress-referenced + namespace: test-gslb + resourceVersion: "999" +spec: + ingressClassName: nginx + rules: + - host: ingress-referenced.cloud.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: referenced + port: + name: http +status: + loadBalancer: + ingress: + - ip: 10.0.0.1 diff --git a/controllers/refresolver/ingress/testdata/ingress_annotation_single_ip.yaml b/controllers/refresolver/ingress/testdata/ingress_annotation_single_ip.yaml new file mode 100644 index 0000000000..a5a6408f81 --- /dev/null +++ b/controllers/refresolver/ingress/testdata/ingress_annotation_single_ip.yaml @@ -0,0 +1,27 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + labels: + app: ingress-referenced + annotations: + k8gb.io/external-ips: "185.199.110.153" + name: ingress-referenced + namespace: test-gslb + resourceVersion: "999" +spec: + ingressClassName: nginx + rules: + - host: ingress-referenced.cloud.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: referenced + port: + name: http +status: + loadBalancer: + ingress: + - ip: 10.0.0.1 diff --git a/controllers/refresolver/istiovirtualservice/istiovirtualservice.go b/controllers/refresolver/istiovirtualservice/istiovirtualservice.go index fdbde86899..1fbdf59041 100644 --- a/controllers/refresolver/istiovirtualservice/istiovirtualservice.go +++ b/controllers/refresolver/istiovirtualservice/istiovirtualservice.go @@ -38,6 +38,11 @@ import ( var log = logging.Logger() +const ( + // comma separated list of external IP addresses + externalIPsAnnotation = "k8gb.io/external-ips" +) + type ReferenceResolver struct { virtualService *istio.VirtualService lbService *corev1.Service @@ -199,8 +204,13 @@ func (rr *ReferenceResolver) GetServers() ([]*k8gbv1beta1.Server, error) { // GetGslbExposedIPs retrieves the load balancer IP address of the GSLB func (rr *ReferenceResolver) GetGslbExposedIPs(edgeDNSServers utils.DNSList) ([]string, error) { - gslbIngressIPs := []string{} + // fetch the IP addresses of the reverse proxy from an annotation if it exists + if ingressIPsFromAnnotation, ok := rr.lbService.Annotations[externalIPsAnnotation]; ok { + return strings.Split(ingressIPsFromAnnotation, ","), nil + } + // if there is no annotation -> fetch the IP addresses from the Status of the Ingress resource + gslbIngressIPs := []string{} for _, ip := range rr.lbService.Status.LoadBalancer.Ingress { if len(ip.IP) > 0 { gslbIngressIPs = append(gslbIngressIPs, ip.IP) diff --git a/controllers/refresolver/istiovirtualservice/istiovirtualservice_test.go b/controllers/refresolver/istiovirtualservice/istiovirtualservice_test.go index 8938e6dd10..2027c780e3 100644 --- a/controllers/refresolver/istiovirtualservice/istiovirtualservice_test.go +++ b/controllers/refresolver/istiovirtualservice/istiovirtualservice_test.go @@ -130,6 +130,21 @@ func TestGetGslbExposedIPs(t *testing.T) { serviceYaml: "./testdata/istio_service_multiple_ips.yaml", expectedIPs: []string{"10.0.0.1", "10.0.0.2"}, }, + { + name: "annotation with no exposed IPs", + serviceYaml: "./testdata/istio_service_annotation_no_ips.yaml", + expectedIPs: []string{""}, + }, + { + name: "annotation with single exposed IP", + serviceYaml: "./testdata/istio_service_annotation_single_ip.yaml", + expectedIPs: []string{"185.199.110.153"}, + }, + { + name: "annotation with multiple exposed IPs", + serviceYaml: "./testdata/istio_service_annotation_multiple_ips.yaml", + expectedIPs: []string{"185.199.110.153", "185.199.109.153"}, + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { diff --git a/controllers/refresolver/istiovirtualservice/testdata/istio_service_annotation_multiple_ips.yaml b/controllers/refresolver/istiovirtualservice/testdata/istio_service_annotation_multiple_ips.yaml new file mode 100644 index 0000000000..2eb2c750ee --- /dev/null +++ b/controllers/refresolver/istiovirtualservice/testdata/istio_service_annotation_multiple_ips.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Service +metadata: + name: istio-ingressgateway + namespace: istio-ingress + labels: + app: istio-ingressgateway + annotations: + k8gb.io/external-ips: "185.199.110.153,185.199.109.153" +spec: + ports: + - name: http + port: 80 + protocol: TCP + targetPort: 8080 + selector: + app: istio-ingressgateway + type: LoadBalancer +status: + loadBalancer: + ingress: + - ip: 10.0.0.1 diff --git a/controllers/refresolver/istiovirtualservice/testdata/istio_service_annotation_no_ips.yaml b/controllers/refresolver/istiovirtualservice/testdata/istio_service_annotation_no_ips.yaml new file mode 100644 index 0000000000..d011d2390a --- /dev/null +++ b/controllers/refresolver/istiovirtualservice/testdata/istio_service_annotation_no_ips.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Service +metadata: + name: istio-ingressgateway + namespace: istio-ingress + labels: + app: istio-ingressgateway + annotations: + k8gb.io/external-ips: "" +spec: + ports: + - name: http + port: 80 + protocol: TCP + targetPort: 8080 + selector: + app: istio-ingressgateway + type: LoadBalancer +status: + loadBalancer: + ingress: + - ip: 10.0.0.1 diff --git a/controllers/refresolver/istiovirtualservice/testdata/istio_service_annotation_single_ip.yaml b/controllers/refresolver/istiovirtualservice/testdata/istio_service_annotation_single_ip.yaml new file mode 100644 index 0000000000..fcd3393ccf --- /dev/null +++ b/controllers/refresolver/istiovirtualservice/testdata/istio_service_annotation_single_ip.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Service +metadata: + name: istio-ingressgateway + namespace: istio-ingress + labels: + app: istio-ingressgateway + annotations: + k8gb.io/external-ips: "185.199.110.153" +spec: + ports: + - name: http + port: 80 + protocol: TCP + targetPort: 8080 + selector: + app: istio-ingressgateway + type: LoadBalancer +status: + loadBalancer: + ingress: + - ip: 10.0.0.1