Anti Affinity rules added to deployment spec

.changelog/202.txt

@@ -0,0 +1,3 @@
+```release-note:feature
+Define anti-affinity rules so that the scheduler will attempt to evenly spread gateway pods across all available nodes
+```

internal/k8s/builder/builder.go

@@ -35,6 +35,8 @@ const (
 
 	consulCALocalPath = "/consul/tls"
 	consulCALocalFile = consulCALocalPath + "/ca.pem"
+
+	k8sHostnameTopologyKey = "kubernetes.io/hostname"
 )
 
 type Builder interface {

internal/k8s/builder/gateway.go

@@ -190,7 +190,24 @@ func (b *GatewayDeploymentBuilder) podSpec() corev1.PodSpec {
 		defaultServiceAccount = b.gateway.Name
 	}
 
+	labels := utils.LabelsForGateway(b.gateway)
+
 	return corev1.PodSpec{
+		Affinity: &corev1.Affinity{
+			PodAntiAffinity: &corev1.PodAntiAffinity{
+				PreferredDuringSchedulingIgnoredDuringExecution: []corev1.WeightedPodAffinityTerm{
+					{
+						Weight: 1,
+						PodAffinityTerm: corev1.PodAffinityTerm{
+							LabelSelector: &metav1.LabelSelector{
+								MatchLabels: labels,
+							},
+							TopologyKey: k8sHostnameTopologyKey,
+						},
+					},
+				},
+			},
+		},
 		NodeSelector:       b.gwConfig.Spec.NodeSelector,
 		ServiceAccountName: orDefault(b.gwConfig.Spec.ConsulSpec.AuthSpec.Account, defaultServiceAccount),
 		// the init container copies the binary into the

internal/k8s/builder/testdata/clusterip.deployment.golden.yaml

@@ -26,6 +26,18 @@ spec:
         api-gateway.consul.hashicorp.com/name: test-clusterip
         api-gateway.consul.hashicorp.com/namespace: ""
     spec:
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - podAffinityTerm:
+              labelSelector:
+                matchLabels:
+                  api-gateway.consul.hashicorp.com/created: "-62135596800"
+                  api-gateway.consul.hashicorp.com/managed: "true"
+                  api-gateway.consul.hashicorp.com/name: test-clusterip
+                  api-gateway.consul.hashicorp.com/namespace: ""
+              topologyKey: kubernetes.io/hostname
+            weight: 1
       containers:
       - command:
         - /bin/sh

internal/k8s/builder/testdata/loadbalancer.deployment.golden.yaml

@@ -26,6 +26,18 @@ spec:
         api-gateway.consul.hashicorp.com/name: test-loadbalancer
         api-gateway.consul.hashicorp.com/namespace: ""
     spec:
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - podAffinityTerm:
+              labelSelector:
+                matchLabels:
+                  api-gateway.consul.hashicorp.com/created: "-62135596800"
+                  api-gateway.consul.hashicorp.com/managed: "true"
+                  api-gateway.consul.hashicorp.com/name: test-loadbalancer
+                  api-gateway.consul.hashicorp.com/namespace: ""
+              topologyKey: kubernetes.io/hostname
+            weight: 1
       containers:
       - command:
         - /bin/sh

internal/k8s/builder/testdata/max-instances.deployment.golden.yaml

@@ -3,16 +3,16 @@ metadata:
   labels:
     api-gateway.consul.hashicorp.com/created: "-62135596800"
     api-gateway.consul.hashicorp.com/managed: "true"
-    api-gateway.consul.hashicorp.com/name: test-multiple-instances
+    api-gateway.consul.hashicorp.com/name: test-max-instances
     api-gateway.consul.hashicorp.com/namespace: ""
-  name: test-multiple-instances
+  name: test-max-instances
 spec:
   replicas: 5
   selector:
     matchLabels:
       api-gateway.consul.hashicorp.com/created: "-62135596800"
       api-gateway.consul.hashicorp.com/managed: "true"
-      api-gateway.consul.hashicorp.com/name: test-multiple-instances
+      api-gateway.consul.hashicorp.com/name: test-max-instances
       api-gateway.consul.hashicorp.com/namespace: ""
   strategy: {}
   template:
@@ -23,9 +23,21 @@ spec:
       labels:
         api-gateway.consul.hashicorp.com/created: "-62135596800"
         api-gateway.consul.hashicorp.com/managed: "true"
-        api-gateway.consul.hashicorp.com/name: test-multiple-instances
+        api-gateway.consul.hashicorp.com/name: test-max-instances
         api-gateway.consul.hashicorp.com/namespace: ""
     spec:
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - podAffinityTerm:
+              labelSelector:
+                matchLabels:
+                  api-gateway.consul.hashicorp.com/created: "-62135596800"
+                  api-gateway.consul.hashicorp.com/managed: "true"
+                  api-gateway.consul.hashicorp.com/name: test-max-instances
+                  api-gateway.consul.hashicorp.com/namespace: ""
+              topologyKey: kubernetes.io/hostname
+            weight: 1
       containers:
       - command:
         - /bin/sh
@@ -36,7 +48,7 @@ spec:
           exec /bootstrap/consul-api-gateway exec -log-json \
             -log-level info \
             -gateway-host "$(IP)" \
-            -gateway-name test-multiple-instances \
+            -gateway-name test-max-instances \
             -gateway-namespace test \
             -consul-http-address $(HOST_IP) \
             -consul-http-port 8500 \

internal/k8s/builder/testdata/max-instances.service.golden.yaml

@@ -3,9 +3,9 @@ metadata:
   labels:
     api-gateway.consul.hashicorp.com/created: "-62135596800"
     api-gateway.consul.hashicorp.com/managed: "true"
-    api-gateway.consul.hashicorp.com/name: test-multiple-instances
+    api-gateway.consul.hashicorp.com/name: test-max-instances
     api-gateway.consul.hashicorp.com/namespace: ""
-  name: test-multiple-instances
+  name: test-max-instances
 spec:
   ports:
   - name: http
@@ -19,7 +19,7 @@ spec:
   selector:
     api-gateway.consul.hashicorp.com/created: "-62135596800"
     api-gateway.consul.hashicorp.com/managed: "true"
-    api-gateway.consul.hashicorp.com/name: test-multiple-instances
+    api-gateway.consul.hashicorp.com/name: test-max-instances
     api-gateway.consul.hashicorp.com/namespace: ""
   type: ClusterIP
 status:

internal/k8s/builder/testdata/max-instances.yaml

@@ -25,7 +25,7 @@ spec:
 apiVersion: gateway.networking.k8s.io/v1alpha2
 kind: Gateway
 metadata:
-  name: test-multiple-instances
+  name: test-max-instances
 spec:
   gatewayClassName: test-gateway-class
   listeners:

internal/k8s/builder/testdata/min-instances.deployment.golden.yaml

@@ -3,16 +3,16 @@ metadata:
   labels:
     api-gateway.consul.hashicorp.com/created: "-62135596800"
     api-gateway.consul.hashicorp.com/managed: "true"
-    api-gateway.consul.hashicorp.com/name: test-multiple-instances
+    api-gateway.consul.hashicorp.com/name: test-min-instances
     api-gateway.consul.hashicorp.com/namespace: ""
-  name: test-multiple-instances
+  name: test-min-instances
 spec:
   replicas: 5
   selector:
     matchLabels:
       api-gateway.consul.hashicorp.com/created: "-62135596800"
       api-gateway.consul.hashicorp.com/managed: "true"
-      api-gateway.consul.hashicorp.com/name: test-multiple-instances
+      api-gateway.consul.hashicorp.com/name: test-min-instances
       api-gateway.consul.hashicorp.com/namespace: ""
   strategy: {}
   template:
@@ -23,9 +23,21 @@ spec:
       labels:
         api-gateway.consul.hashicorp.com/created: "-62135596800"
         api-gateway.consul.hashicorp.com/managed: "true"
-        api-gateway.consul.hashicorp.com/name: test-multiple-instances
+        api-gateway.consul.hashicorp.com/name: test-min-instances
         api-gateway.consul.hashicorp.com/namespace: ""
     spec:
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - podAffinityTerm:
+              labelSelector:
+                matchLabels:
+                  api-gateway.consul.hashicorp.com/created: "-62135596800"
+                  api-gateway.consul.hashicorp.com/managed: "true"
+                  api-gateway.consul.hashicorp.com/name: test-min-instances
+                  api-gateway.consul.hashicorp.com/namespace: ""
+              topologyKey: kubernetes.io/hostname
+            weight: 1
       containers:
       - command:
         - /bin/sh
@@ -36,7 +48,7 @@ spec:
           exec /bootstrap/consul-api-gateway exec -log-json \
             -log-level info \
             -gateway-host "$(IP)" \
-            -gateway-name test-multiple-instances \
+            -gateway-name test-min-instances \
             -gateway-namespace test \
             -consul-http-address $(HOST_IP) \
             -consul-http-port 8500 \

internal/k8s/builder/testdata/min-instances.service.golden.yaml

@@ -3,9 +3,9 @@ metadata:
   labels:
     api-gateway.consul.hashicorp.com/created: "-62135596800"
     api-gateway.consul.hashicorp.com/managed: "true"
-    api-gateway.consul.hashicorp.com/name: test-multiple-instances
+    api-gateway.consul.hashicorp.com/name: test-min-instances
     api-gateway.consul.hashicorp.com/namespace: ""
-  name: test-multiple-instances
+  name: test-min-instances
 spec:
   ports:
   - name: http
@@ -19,7 +19,7 @@ spec:
   selector:
     api-gateway.consul.hashicorp.com/created: "-62135596800"
     api-gateway.consul.hashicorp.com/managed: "true"
-    api-gateway.consul.hashicorp.com/name: test-multiple-instances
+    api-gateway.consul.hashicorp.com/name: test-min-instances
     api-gateway.consul.hashicorp.com/namespace: ""
   type: ClusterIP
 status:

internal/k8s/builder/testdata/min-instances.yaml

@@ -25,7 +25,7 @@ spec:
 apiVersion: gateway.networking.k8s.io/v1alpha2
 kind: Gateway
 metadata:
-  name: test-multiple-instances
+  name: test-min-instances
 spec:
   gatewayClassName: test-gateway-class
   listeners:

internal/k8s/builder/testdata/multiple-instances.deployment.golden.yaml

@@ -26,6 +26,18 @@ spec:
         api-gateway.consul.hashicorp.com/name: test-multiple-instances
         api-gateway.consul.hashicorp.com/namespace: ""
     spec:
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - podAffinityTerm:
+              labelSelector:
+                matchLabels:
+                  api-gateway.consul.hashicorp.com/created: "-62135596800"
+                  api-gateway.consul.hashicorp.com/managed: "true"
+                  api-gateway.consul.hashicorp.com/name: test-multiple-instances
+                  api-gateway.consul.hashicorp.com/namespace: ""
+              topologyKey: kubernetes.io/hostname
+            weight: 1
       containers:
       - command:
         - /bin/sh

internal/k8s/builder/testdata/static-mapping.deployment.golden.yaml

@@ -26,6 +26,18 @@ spec:
         api-gateway.consul.hashicorp.com/name: test-static-mapping
         api-gateway.consul.hashicorp.com/namespace: ""
     spec:
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - podAffinityTerm:
+              labelSelector:
+                matchLabels:
+                  api-gateway.consul.hashicorp.com/created: "-62135596800"
+                  api-gateway.consul.hashicorp.com/managed: "true"
+                  api-gateway.consul.hashicorp.com/name: test-static-mapping
+                  api-gateway.consul.hashicorp.com/namespace: ""
+              topologyKey: kubernetes.io/hostname
+            weight: 1
       containers:
       - command:
         - /bin/sh

internal/k8s/builder/testdata/tls-cert.deployment.golden.yaml

@@ -26,6 +26,18 @@ spec:
         api-gateway.consul.hashicorp.com/name: tls-cert-test
         api-gateway.consul.hashicorp.com/namespace: ""
     spec:
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - podAffinityTerm:
+              labelSelector:
+                matchLabels:
+                  api-gateway.consul.hashicorp.com/created: "-62135596800"
+                  api-gateway.consul.hashicorp.com/managed: "true"
+                  api-gateway.consul.hashicorp.com/name: tls-cert-test
+                  api-gateway.consul.hashicorp.com/namespace: ""
+              topologyKey: kubernetes.io/hostname
+            weight: 1
       containers:
       - command:
         - /bin/sh