add adntiaffinity rules for host

.changelog/154.txt

@@ -0,0 +1,3 @@
+```release-note:breaking-change
+Gateway listener `certificateRefs` to secrets in a different namespace now require a [ReferencePolicy](https://gateway-api.sigs.k8s.io/v1alpha2/references/spec/#gateway.networking.k8s.io%2fv1alpha2.ReferencePolicy)
+```

.changelog/202.txt

@@ -0,0 +1,3 @@
+```release-note:feature
+Define anti-affinity rules so that the scheduler will attempt to evenly spread gateway pods across all available nodes
+```

config/crd/bases/api-gateway.consul.hashicorp.com_gatewayclassconfigs.yaml

@@ -102,6 +102,20 @@ spec:
                     maximum: 8
                     minimum: 1
                     type: integer
+                  maxInstances:
+                    default: 8
+                    description: Max allowed number of gateway instances
+                    format: int32
+                    maximum: 8
+                    minimum: 1
+                    type: integer
+                  minInstances:
+                    default: 1
+                    description: Minimum allowed number of gateway instances
+                    format: int32
+                    maximum: 8
+                    minimum: 1
+                    type: integer
                 type: object
               image:
                 description: Configuration information about the images to use

internal/commands/server/k8s_e2e_test.go

@@ -113,7 +113,6 @@ func TestGatewayWithClassConfigChange(t *testing.T) {
 }
 
 func TestGatewayWithReplicas(t *testing.T) {
-	t.Parallel()
 	feature := features.New("gateway class config configure instances").
 		Assess("gateway is created with appropriate number of replicas set", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
 			namespace := e2e.Namespace(ctx)
@@ -149,7 +148,6 @@ func TestGatewayWithReplicas(t *testing.T) {
 }
 
 func TestGatewayWithReplicasCanScale(t *testing.T) {
-	t.Parallel()
 	feature := features.New("gateway class config doesn't override manual scaling").
 		Assess("gateway deployment doesn't get overriden with kubectl scale operation", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
 			namespace := e2e.Namespace(ctx)
@@ -192,12 +190,77 @@ func TestGatewayWithReplicasCanScale(t *testing.T) {
 	testenv.Test(t, feature.Feature())
 }
 
+func TestGatewayWithReplicasRespectMinMax(t *testing.T) {
+	t.Parallel()
+	feature := features.New("gateway class config min max fields are respected").
+		Assess("gateway deployment min maxes appropriately", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+			namespace := e2e.Namespace(ctx)
+			resources := cfg.Client().Resources(namespace)
+
+			var initialReplicas int32 = 3
+			var minReplicas int32 = 2
+			var maxReplicas int32 = 8
+			var exceedsMin int32 = minReplicas - 1
+			var exceedsMax int32 = maxReplicas + 1
+			useHostPorts := false
+
+			// Create a GatewayClassConfig
+			gatewayClassConfig, gatewayClass := createGatewayClassWithParams(ctx, t, resources, GatewayClassConfigParams{
+				UseHostPorts:     &useHostPorts,
+				DefaultInstances: &initialReplicas,
+				MaxInstances:     &maxReplicas,
+				MinInstances:     &minReplicas,
+			})
+
+			require.Eventually(t, gatewayClassStatusCheck(ctx, resources, gatewayClass.Name, namespace, conditionAccepted), 30*time.Second, checkInterval, "gatewayclass not accepted in the allotted time")
+
+			// Create an HTTPS Gateway Listener to pass when creating gateways
+			httpsListener := createHTTPSListener(ctx, t, 443)
+
+			// Create a Gateway and wait for it to be ready
+			gatewayName := envconf.RandomName("gw", 16)
+			gateway := createGateway(ctx, t, resources, gatewayName, gatewayClass, []gateway.Listener{httpsListener})
+
+			require.Eventually(t, gatewayStatusCheck(ctx, resources, gatewayName, namespace, conditionReady), checkTimeout, checkInterval, "no gateway found in the allotted time")
+
+			require.Eventually(t, gatewayStatusCheck(ctx, resources, gatewayName, namespace, conditionReady), 30*time.Second, checkInterval, "no gateway found in the allotted time")
+			checkGatewayConfigAnnotation(ctx, t, resources, gatewayName, namespace, gatewayClassConfig)
+
+			// Fetch the deployment created by the gateway and check the number of replicas
+			deployment := &appsv1.Deployment{}
+			require.NoError(t, resources.Get(ctx, gatewayName, namespace, deployment))
+			assert.Equal(t, initialReplicas, *deployment.Spec.Replicas)
+
+			// Scale the deployment up
+			deployment.Spec.Replicas = &exceedsMax
+			assert.NoError(t, resources.Update(ctx, deployment))
+
+			// Double check that replicas was set appropriately
+			assert.NoError(t, resources.Get(ctx, gatewayName, namespace, deployment))
+			assert.Eventually(t, deploymentReplicasSetAsExpected(ctx, resources, gatewayName, namespace, maxReplicas), 30*time.Second, checkInterval, "replicas not scaled down to max in the alloted time")
+
+			// Scale the deployment down
+			assert.NoError(t, resources.Get(ctx, gatewayName, namespace, deployment))
+			deployment.Spec.Replicas = &exceedsMin
+			assert.NoError(t, resources.Update(ctx, deployment))
+
+			// Double check that replicas was set appropriately
+			assert.NoError(t, resources.Get(ctx, gatewayName, namespace, deployment))
+			assert.Eventually(t, deploymentReplicasSetAsExpected(ctx, resources, gatewayName, namespace, minReplicas), 30*time.Second, checkInterval, "replicas not scaled up to min in the alloted time")
+
+			assert.NoError(t, resources.Delete(ctx, gateway))
+
+			return ctx
+		})
+
+	testenv.Test(t, feature.Feature())
+}
+
 func TestGatewayBasic(t *testing.T) {
 	feature := features.New("gateway admission").
 		Assess("basic admission and status updates", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
 			namespace := e2e.Namespace(ctx)
 			resources := cfg.Client().Resources(namespace)
-
 			gatewayName := envconf.RandomName("gw", 16)
 
 			useHostPorts := false
@@ -1166,6 +1229,21 @@ func gatewayStatusCheck(ctx context.Context, resources *resources.Resources, gat
 	}
 }
 
+func deploymentReplicasSetAsExpected(ctx context.Context, resources *resources.Resources, gatewayName, namespace string, expectedReplicas int32) func() bool {
+	return func() bool {
+		deployment := &appsv1.Deployment{}
+		if err := resources.Get(ctx, gatewayName, namespace, deployment); err != nil {
+			return false
+		}
+
+		if deployment.Spec.Replicas == nil {
+			return false
+		}
+
+		return *deployment.Spec.Replicas == expectedReplicas
+	}
+}
+
 func gatewayClassStatusCheck(ctx context.Context, resources *resources.Resources, gatewayClassName, namespace string, checkFn func([]meta.Condition) bool) func() bool {
 	return func() bool {
 		updated := &gateway.GatewayClass{}
@@ -1308,6 +1386,8 @@ func createGateway(ctx context.Context, t *testing.T, resources *resources.Resou
 type GatewayClassConfigParams struct {
 	UseHostPorts     *bool
 	DefaultInstances *int32
+	MinInstances     *int32
+	MaxInstances     *int32
 }
 
 func createGatewayClass(ctx context.Context, t *testing.T, resources *resources.Resources) (*apigwv1alpha1.GatewayClassConfig, *gateway.GatewayClass) {
@@ -1349,6 +1429,8 @@ func createGatewayClassWithParams(ctx context.Context, t *testing.T, resources *
 			LogLevel:     "trace",
 			DeploymentSpec: apigwv1alpha1.DeploymentSpec{
 				DefaultInstances: &defaultInstances,
+				MaxInstances:     params.MaxInstances,
+				MinInstances:     params.MinInstances,
 			},
 			ConsulSpec: apigwv1alpha1.ConsulSpec{
 				Address: hostRoute,

internal/k8s/builder/builder.go

@@ -35,6 +35,8 @@ const (
 
 	consulCALocalPath = "/consul/tls"
 	consulCALocalFile = consulCALocalPath + "/ca.pem"
+
+	k8sHostnameTopologyKey = "kubernetes.io/hostname"
 )
 
 type Builder interface {
@@ -43,7 +45,7 @@ type Builder interface {
 
 type DeploymentBuilder interface {
 	Builder
-	Build() *v1.Deployment
+	Build(*int32) *v1.Deployment
 }
 
 type ServiceBuilder interface {

internal/k8s/builder/gateway.go

@@ -124,7 +124,7 @@ func (b *GatewayDeploymentBuilder) Validate() error {
 	return nil
 }
 
-func (b *GatewayDeploymentBuilder) Build() *v1.Deployment {
+func (b *GatewayDeploymentBuilder) Build(currentReplicas *int32) *v1.Deployment {
 	labels := utils.LabelsForGateway(b.gateway)
 
 	return &v1.Deployment{
@@ -134,7 +134,7 @@ func (b *GatewayDeploymentBuilder) Build() *v1.Deployment {
 			Labels:    labels,
 		},
 		Spec: v1.DeploymentSpec{
-			Replicas: b.instances(),
+			Replicas: b.instances(currentReplicas),
 			Selector: &metav1.LabelSelector{
 				MatchLabels: labels,
 			},
@@ -151,13 +151,36 @@ func (b *GatewayDeploymentBuilder) Build() *v1.Deployment {
 	}
 }
 
-func (b *GatewayDeploymentBuilder) instances() *int32 {
-	if b.gwConfig.Spec.DeploymentSpec.DefaultInstances == nil {
-		instances := defaultInstances
-		return &instances
+func (b *GatewayDeploymentBuilder) instances(currentReplicas *int32) *int32 {
+
+	instanceValue := defaultInstances
+
+	//if currentReplicas is not nil use current value when building deployment
+	if currentReplicas != nil {
+		instanceValue = *currentReplicas
+	} else if b.gwConfig.Spec.DeploymentSpec.DefaultInstances != nil {
+		// otherwise use the default value on the GatewayClassConfig if set
+		instanceValue = *b.gwConfig.Spec.DeploymentSpec.DefaultInstances
+	}
+
+	if b.gwConfig.Spec.DeploymentSpec.MaxInstances != nil {
+
+		//check if over maximum and lower to maximum
+		maxValue := *b.gwConfig.Spec.DeploymentSpec.MaxInstances
+		if instanceValue > maxValue {
+			instanceValue = maxValue
+		}
 	}
 
-	return b.gwConfig.Spec.DeploymentSpec.DefaultInstances
+	if b.gwConfig.Spec.DeploymentSpec.MinInstances != nil {
+		//check if less than minimum and raise to minimum
+		minValue := *b.gwConfig.Spec.DeploymentSpec.MinInstances
+		if instanceValue < minValue {
+			instanceValue = minValue
+		}
+
+	}
+	return &instanceValue
 }
 
 func (b *GatewayDeploymentBuilder) podSpec() corev1.PodSpec {
@@ -167,7 +190,24 @@ func (b *GatewayDeploymentBuilder) podSpec() corev1.PodSpec {
 		defaultServiceAccount = b.gateway.Name
 	}
 
+	labels := utils.LabelsForGateway(b.gateway)
+
 	return corev1.PodSpec{
+		Affinity: &corev1.Affinity{
+			PodAntiAffinity: &corev1.PodAntiAffinity{
+				PreferredDuringSchedulingIgnoredDuringExecution: []corev1.WeightedPodAffinityTerm{
+					{
+						Weight: 1,
+						PodAffinityTerm: corev1.PodAffinityTerm{
+							LabelSelector: &metav1.LabelSelector{
+								MatchLabels: labels,
+							},
+							TopologyKey: k8sHostnameTopologyKey,
+						},
+					},
+				},
+			},
+		},
 		NodeSelector:       b.gwConfig.Spec.NodeSelector,
 		ServiceAccountName: orDefault(b.gwConfig.Spec.ConsulSpec.AuthSpec.Account, defaultServiceAccount),
 		// the init container copies the binary into the

internal/k8s/builder/gateway_test.go

@@ -24,6 +24,8 @@ var (
 		"clusterip",
 		"loadbalancer",
 		"multiple-instances",
+		"max-instances",
+		"min-instances",
 	}
 )
 
@@ -53,7 +55,7 @@ func (g *gatewayTestConfig) EncodeDeployment() runtime.Object {
 	b.WithClassConfig(*g.gatewayClassConfig)
 	b.WithConsulCA("CONSUL_CA_MOCKED")
 	b.WithConsulGatewayNamespace("test")
-	return b.Build()
+	return b.Build(nil)
 }
 
 func (g *gatewayTestConfig) EncodeService() runtime.Object {

internal/k8s/builder/testdata/clusterip.deployment.golden.yaml

@@ -26,6 +26,18 @@ spec:
         api-gateway.consul.hashicorp.com/name: test-clusterip
         api-gateway.consul.hashicorp.com/namespace: ""
     spec:
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - podAffinityTerm:
+              labelSelector:
+                matchLabels:
+                  api-gateway.consul.hashicorp.com/created: "-62135596800"
+                  api-gateway.consul.hashicorp.com/managed: "true"
+                  api-gateway.consul.hashicorp.com/name: test-clusterip
+                  api-gateway.consul.hashicorp.com/namespace: ""
+              topologyKey: kubernetes.io/hostname
+            weight: 1
       containers:
       - command:
         - /bin/sh

internal/k8s/builder/testdata/loadbalancer.deployment.golden.yaml

@@ -26,6 +26,18 @@ spec:
         api-gateway.consul.hashicorp.com/name: test-loadbalancer
         api-gateway.consul.hashicorp.com/namespace: ""
     spec:
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - podAffinityTerm:
+              labelSelector:
+                matchLabels:
+                  api-gateway.consul.hashicorp.com/created: "-62135596800"
+                  api-gateway.consul.hashicorp.com/managed: "true"
+                  api-gateway.consul.hashicorp.com/name: test-loadbalancer
+                  api-gateway.consul.hashicorp.com/namespace: ""
+              topologyKey: kubernetes.io/hostname
+            weight: 1
       containers:
       - command:
         - /bin/sh