move prom service to new service

Azure · Oct 24, 2024 · b9a6cb0 · b9a6cb0
1 parent 40d1bce
commit b9a6cb0
Show file tree

Hide file tree

Showing 7 changed files with 189 additions and 31 deletions.
diff --git a/pkg/manifests/fixtures/nginx/full.json b/pkg/manifests/fixtures/nginx/full.json
@@ -390,8 +390,6 @@
           "app.kubernetes.io/name": "nginx"
         },
         "annotations": {
-          "prometheus.io/port": "10254",
-          "prometheus.io/scrape": "true",
           "service.beta.kubernetes.io/azure-load-balancer-internal": "true"
         }
       },
@@ -406,7 +404,37 @@
             "name": "https",
             "port": 443,
             "targetPort": "https"
-          },
+          }
+        ],
+        "selector": {
+          "app": "nginx"
+        },
+        "type": "LoadBalancer",
+        "externalTrafficPolicy": "Local"
+      },
+      "status": {
+        "loadBalancer": {}
+      }
+    },
+    {
+      "kind": "Service",
+      "apiVersion": "v1",
+      "metadata": {
+        "name": "nginx-metrics",
+        "namespace": "test-namespace",
+        "creationTimestamp": null,
+        "labels": {
+          "app.kubernetes.io/component": "ingress-controller",
+          "app.kubernetes.io/managed-by": "aks-app-routing-operator",
+          "app.kubernetes.io/name": "nginx"
+        },
+        "annotations": {
+          "prometheus.io/port": "10254",
+          "prometheus.io/scrape": "true"
+        }
+      },
+      "spec": {
+        "ports": [
           {
             "name": "prometheus",
             "port": 10254,
@@ -416,8 +444,7 @@
         "selector": {
           "app": "nginx"
         },
-        "type": "LoadBalancer",
-        "externalTrafficPolicy": "Local"
+        "type": "ClusterIP"
       },
       "status": {
         "loadBalancer": {}

diff --git a/pkg/manifests/fixtures/nginx/internal.json b/pkg/manifests/fixtures/nginx/internal.json
@@ -388,10 +388,6 @@
           "app.kubernetes.io/component": "ingress-controller",
           "app.kubernetes.io/managed-by": "aks-app-routing-operator",
           "app.kubernetes.io/name": "nginx"
-        },
-        "annotations": {
-          "prometheus.io/port": "10254",
-          "prometheus.io/scrape": "true"
         }
       },
       "spec": {
@@ -405,7 +401,37 @@
             "name": "https",
             "port": 443,
             "targetPort": "https"
-          },
+          }
+        ],
+        "selector": {
+          "app": "nginx"
+        },
+        "type": "LoadBalancer",
+        "externalTrafficPolicy": "Local"
+      },
+      "status": {
+        "loadBalancer": {}
+      }
+    },
+    {
+      "kind": "Service",
+      "apiVersion": "v1",
+      "metadata": {
+        "name": "nginx-metrics",
+        "namespace": "test-namespace",
+        "creationTimestamp": null,
+        "labels": {
+          "app.kubernetes.io/component": "ingress-controller",
+          "app.kubernetes.io/managed-by": "aks-app-routing-operator",
+          "app.kubernetes.io/name": "nginx"
+        },
+        "annotations": {
+          "prometheus.io/port": "10254",
+          "prometheus.io/scrape": "true"
+        }
+      },
+      "spec": {
+        "ports": [
           {
             "name": "prometheus",
             "port": 10254,
@@ -415,8 +441,7 @@
         "selector": {
           "app": "nginx"
         },
-        "type": "LoadBalancer",
-        "externalTrafficPolicy": "Local"
+        "type": "ClusterIP"
       },
       "status": {
         "loadBalancer": {}

diff --git a/pkg/manifests/fixtures/nginx/kube-system.json b/pkg/manifests/fixtures/nginx/kube-system.json
@@ -377,8 +377,6 @@
           "app.kubernetes.io/name": "nginx"
         },
         "annotations": {
-          "prometheus.io/port": "10254",
-          "prometheus.io/scrape": "true",
           "service.beta.kubernetes.io/azure-load-balancer-internal": "true"
         }
       },
@@ -393,7 +391,37 @@
             "name": "https",
             "port": 443,
             "targetPort": "https"
-          },
+          }
+        ],
+        "selector": {
+          "app": "nginx"
+        },
+        "type": "LoadBalancer",
+        "externalTrafficPolicy": "Local"
+      },
+      "status": {
+        "loadBalancer": {}
+      }
+    },
+    {
+      "kind": "Service",
+      "apiVersion": "v1",
+      "metadata": {
+        "name": "nginx-metrics",
+        "namespace": "kube-system",
+        "creationTimestamp": null,
+        "labels": {
+          "app.kubernetes.io/component": "ingress-controller",
+          "app.kubernetes.io/managed-by": "aks-app-routing-operator",
+          "app.kubernetes.io/name": "nginx"
+        },
+        "annotations": {
+          "prometheus.io/port": "10254",
+          "prometheus.io/scrape": "true"
+        }
+      },
+      "spec": {
+        "ports": [
           {
             "name": "prometheus",
             "port": 10254,
@@ -403,8 +431,7 @@
         "selector": {
           "app": "nginx"
         },
-        "type": "LoadBalancer",
-        "externalTrafficPolicy": "Local"
+        "type": "ClusterIP"
       },
       "status": {
         "loadBalancer": {}

diff --git a/pkg/manifests/fixtures/nginx/no-ownership.json b/pkg/manifests/fixtures/nginx/no-ownership.json
@@ -390,8 +390,6 @@
           "app.kubernetes.io/name": "nginx"
         },
         "annotations": {
-          "prometheus.io/port": "10254",
-          "prometheus.io/scrape": "true",
           "service.beta.kubernetes.io/azure-load-balancer-internal": "true"
         }
       },
@@ -406,7 +404,37 @@
             "name": "https",
             "port": 443,
             "targetPort": "https"
-          },
+          }
+        ],
+        "selector": {
+          "app": "nginx"
+        },
+        "type": "LoadBalancer",
+        "externalTrafficPolicy": "Local"
+      },
+      "status": {
+        "loadBalancer": {}
+      }
+    },
+    {
+      "kind": "Service",
+      "apiVersion": "v1",
+      "metadata": {
+        "name": "nginx-metrics",
+        "namespace": "test-namespace",
+        "creationTimestamp": null,
+        "labels": {
+          "app.kubernetes.io/component": "ingress-controller",
+          "app.kubernetes.io/managed-by": "aks-app-routing-operator",
+          "app.kubernetes.io/name": "nginx"
+        },
+        "annotations": {
+          "prometheus.io/port": "10254",
+          "prometheus.io/scrape": "true"
+        }
+      },
+      "spec": {
+        "ports": [
           {
             "name": "prometheus",
             "port": 10254,
@@ -416,8 +444,7 @@
         "selector": {
           "app": "nginx"
         },
-        "type": "LoadBalancer",
-        "externalTrafficPolicy": "Local"
+        "type": "ClusterIP"
       },
       "status": {
         "loadBalancer": {}

diff --git a/pkg/manifests/fixtures/nginx/optional-features-disabled.json b/pkg/manifests/fixtures/nginx/optional-features-disabled.json
@@ -387,8 +387,6 @@
           "app.kubernetes.io/name": "nginx"
         },
         "annotations": {
-          "prometheus.io/port": "10254",
-          "prometheus.io/scrape": "true",
           "service.beta.kubernetes.io/azure-load-balancer-internal": "true"
         }
       },
@@ -403,7 +401,37 @@
             "name": "https",
             "port": 443,
             "targetPort": "https"
-          },
+          }
+        ],
+        "selector": {
+          "app": "nginx"
+        },
+        "type": "LoadBalancer",
+        "externalTrafficPolicy": "Local"
+      },
+      "status": {
+        "loadBalancer": {}
+      }
+    },
+    {
+      "kind": "Service",
+      "apiVersion": "v1",
+      "metadata": {
+        "name": "nginx-metrics",
+        "namespace": "test-namespace",
+        "creationTimestamp": null,
+        "labels": {
+          "app.kubernetes.io/component": "ingress-controller",
+          "app.kubernetes.io/managed-by": "aks-app-routing-operator",
+          "app.kubernetes.io/name": "nginx"
+        },
+        "annotations": {
+          "prometheus.io/port": "10254",
+          "prometheus.io/scrape": "true"
+        }
+      },
+      "spec": {
+        "ports": [
           {
             "name": "prometheus",
             "port": 10254,
@@ -413,8 +441,7 @@
         "selector": {
           "app": "nginx"
         },
-        "type": "LoadBalancer",
-        "externalTrafficPolicy": "Local"
+        "type": "ClusterIP"
       },
       "status": {
         "loadBalancer": {}

diff --git a/pkg/manifests/nginx.go b/pkg/manifests/nginx.go
@@ -133,6 +133,7 @@ func GetNginxResources(conf *config.Config, ingressConfig *NginxIngressConfig) *
 		ClusterRoleBinding:      newNginxIngressControllerClusterRoleBinding(conf, ingressConfig),
 		RoleBinding:             newNginxIngressControllerRoleBinding(conf, ingressConfig),
 		Service:                 newNginxIngressControllerService(conf, ingressConfig),
+		PromService:             newNginxIngressControllerPromService(conf, ingressConfig),
 		Deployment:              newNginxIngressControllerDeployment(conf, ingressConfig),
 		ConfigMap:               newNginxIngressControllerConfigmap(conf, ingressConfig),
 		HorizontalPodAutoscaler: newNginxIngressControllerHPA(conf, ingressConfig),
@@ -362,10 +363,6 @@ func newNginxIngressControllerRoleBinding(conf *config.Config, ingressConfig *Ng
 
 func newNginxIngressControllerService(conf *config.Config, ingressConfig *NginxIngressConfig) *corev1.Service {
 	annotations := make(map[string]string)
-	for k, v := range promAnnotations {
-		annotations[k] = v
-	}
-
 	if ingressConfig != nil && ingressConfig.ServiceConfig != nil {
 		for k, v := range ingressConfig.ServiceConfig.Annotations {
 			annotations[k] = v
@@ -398,6 +395,32 @@ func newNginxIngressControllerService(conf *config.Config, ingressConfig *NginxI
 					Port:       443,
 					TargetPort: intstr.FromString("https"),
 				},
+			},
+		},
+	}
+}
+
+func newNginxIngressControllerPromService(conf *config.Config, ingressConfig *NginxIngressConfig) *corev1.Service {
+	annotations := make(map[string]string)
+	for k, v := range promAnnotations {
+		annotations[k] = v
+	}
+
+	return &corev1.Service{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Service",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        ingressConfig.ResourceName + "-metrics",
+			Namespace:   conf.NS,
+			Labels:      addComponentLabel(GetTopLevelLabels(), "ingress-controller"),
+			Annotations: annotations,
+		},
+		Spec: corev1.ServiceSpec{
+			Type:     corev1.ServiceTypeClusterIP,
+			Selector: ingressConfig.PodLabels(),
+			Ports: []corev1.ServicePort{
 				promServicePort,
 			},
 		},

diff --git a/pkg/manifests/types.go b/pkg/manifests/types.go
@@ -29,6 +29,7 @@ type NginxResources struct {
 	ClusterRoleBinding      *rbacv1.ClusterRoleBinding
 	RoleBinding             *rbacv1.RoleBinding
 	Service                 *corev1.Service
+	PromService             *corev1.Service
 	Deployment              *appsv1.Deployment
 	ConfigMap               *corev1.ConfigMap
 	HorizontalPodAutoscaler *autov1.HorizontalPodAutoscaler
@@ -44,6 +45,7 @@ func (n *NginxResources) Objects() []client.Object {
 		n.ClusterRoleBinding,
 		n.RoleBinding,
 		n.Service,
+		n.PromService,
 		n.Deployment,
 		n.ConfigMap,
 		n.HorizontalPodAutoscaler,