diff --git a/Gopkg.lock b/Gopkg.lock index f66699c0fa..0a9fb9b8cd 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -801,6 +801,7 @@ "k8s.io/apimachinery/pkg/util/clock", "k8s.io/apimachinery/pkg/util/intstr", "k8s.io/apimachinery/pkg/util/runtime", + "k8s.io/apimachinery/pkg/util/uuid", "k8s.io/apimachinery/pkg/util/validation/field", "k8s.io/apimachinery/pkg/util/wait", "k8s.io/apimachinery/pkg/util/yaml", diff --git a/README.md b/README.md index b48269981f..a650b370bc 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,7 @@ Documentation and usage guides on how to develop and host dedicated game servers - [Create a Game Server](./docs/create_gameserver.md) - [Create a Game Server Fleet](./docs/create_fleet.md) - [Create a Fleet Autoscaler](./docs/create_fleetautoscaler.md) + - [Create a Webhook for FleetAutoscaler](./docs/create_webhook_fleetautoscaler.md) - [Edit Your First Game Server (Go)](./docs/edit_first_game_server.md) ### Guides diff --git a/docs/create_webhook_fleetautoscaler.md b/docs/create_webhook_fleetautoscaler.md new file mode 100644 index 0000000000..380a62c761 --- /dev/null +++ b/docs/create_webhook_fleetautoscaler.md @@ -0,0 +1,238 @@ +# Quickstart Create a Fleet Autoscaler with Webhook Policy + +This guide covers how you can create webhook fleet autoscaler policy. +The main difference from the Buffer policy is that the logic on how many target replicas you need is delegated to a separate pod. +This type of Autoscaler would send an HTTP request to the webhook endpoint every sync period (which is currently 30s) with a JSON body, and scale the target fleet based on the data that is returned. + +## Prerequisites + +It is assumed that you have read the instructions to [Create a Game Server Fleet](./create_fleet.md) +and you have a running fleet of game servers or you could run command from Step #1. + +## Objectives + +- Deploy the Webhook Pod and service for autoscaling +- Create a Fleet Autoscaler with Webhook policy type in Kubernetes using Agones custom resource +- Watch the Fleet scale up when allocating GameServers +- Watch the Fleet scale down after GameServer shutdown + +### 1. Deploy the fleet + +Run a fleet in a cluster: +``` +kubectl apply -f https://raw.githubusercontent.com/GoogleCloudPlatform/agones/master/examples/simple-udp/fleet.yaml +``` + +### 2. Deploy a Webhook service for autoscaling + +We need to create a pod which will handle HTTP requests with json payload [`FleetAutoscaleReview`](./fleetautoscaler_spec.md#webhook-endpoint-specification) and return back it with [`FleetAutoscaleResponse`](./fleetautoscaler_spec.md#webhook-endpoint-specification) populated. +The `Scale` flag and `Replicas` values returned in the `FleetAutoscaleResponse` and `Replicas` value tells the FleetAutoscaler what target size the backing Fleet should be scaled up or down to. If `Scale` is false - no scalling occurs. + +You can see the source code for this webhook [here](../examples/autoscaler-webhook). + +Run next command to create a service and a Webhook pod in a cluster: +``` +kubectl apply -f https://raw.githubusercontent.com/GoogleCloudPlatform/agones/master/examples/autoscaler-webhook/autoscaler-service.yaml +``` + +To check that it is running and liveness probe is fine: +``` +kubectl describe pod autoscaler-webhook +``` + +``` +Name: autoscaler-webhook-86944884c4-sdtqh +Namespace: default +Node: gke-test-cluster-default-1c5dec79-h0tq/10.138.0.2 +... +Status: Running +``` + +### 3. Create a Fleet Autoscaler + +Let's create a Fleet Autoscaler using the following command: + +``` +kubectl apply -f https://raw.githubusercontent.com/GoogleCloudPlatform/agones/master/examples/webhookfleetautoscaler.yaml +``` + +You should see a successful ouput similar to this: + +``` +fleetautoscaler.stable.agones.sev "webhook-fleet-autoscaler" created +``` + +This has created a FleetAutoscaler record inside Kubernetes. +It has the link to Webhook service we deployed above. + +### 4. See the fleet and autoscaler status. + +In order to track the list of gameservers which run in your fleet you can run this command in a separate terminal tab: + +``` + watch "kubectl get gs -n default" +``` + +In order to get autoscaler status use the following command: + +``` +kubectl describe fleetautoscaler webhook-fleet-autoscaler +``` + +It should look something like this: + +``` +Name: webhook-fleet-autoscaler +Namespace: default +Labels: +Annotations: kubectl.kubernetes.io/last-applied-configuration={"apiVersion":"stable.agones.dev/v1alpha1","kind":"FleetAutoscaler","metadata":{"annotations":{},"name":"webhook-fleet-autoscaler","namespace":"default... +API Version: stable.agones.dev/v1alpha1 +Kind: FleetAutoscaler +etadata: + Cluster Name: + Creation Timestamp: 2018-12-22T12:52:23Z + Generation: 1 + Resource Version: 2274579 + Self Link: /apis/stable.agones.dev/v1alpha1/namespaces/default/fleetautoscalers/webhook-fleet-autoscaler + UID: 6d03eae4-05e8-11e9-84c2-42010a8a01c9 +Spec: + Fleet Name: simple-udp + Policy: + Type: Webhook + Webhook: + Service: + Name: autoscaler-webhook-service + Namespace: default + Path: scale + URL: +Status: + Able To Scale: true + Current Replicas: 2 + Desired Replicas: 2 + Last Scale Time: + Scaling Limited: false +Events: +``` + +You can see the status (able to scale, not limited), the last time the fleet was scaled (nil for never), current and desired fleet size. + +The autoscaler make a query to a webhoook service deployed on step 1 and on response changing the target Replica size, and the fleet creates/deletes game server instances +to achieve that number. The convergence is achieved in time, which is usually measured in seconds. + +### 5. Allocate Game Servers from the Fleet to trigger scale up + +If you're interested in more details for game server allocation, you should consult the [Create a Game Server Fleet](./create_fleet.md) page. +Here we only interested in triggering allocations to see the autoscaler in action. + +``` +kubectl create -f https://raw.githubusercontent.com/GoogleCloudPlatform/agones/master/examples/simple-udp/fleetallocation.yaml -o yaml +``` + +You should get in return the allocated game server details, which should end with something like: +``` + status: + address: 35.247.13.175 + nodeName: gke-test-cluster-default-1c5dec79-qrqv + ports: + - name: default + port: 7047 + state: Allocated +``` + +Note the address and port, you might need them later to connect to the server. + +Run the above kubectl command one more time so that we have both servers allocated. + +### 6. Check new Autoscaler and Fleet status + +Now let's wait a few seconds to allow the autoscaler to detect the change in the fleet and check again its status + +``` +kubectl describe fleetautoscaler webhook-fleet-autoscaler +``` + +The last part should look similar to this: + +``` +Spec: + Fleet Name: simple-udp + Policy: + Type: Webhook + Webhook: + Service: + Name: autoscaler-webhook-service + Namespace: default + Path: scale + URL: +Status: + Able To Scale: true + Current Replicas: 4 + Desired Replicas: 4 + Last Scale Time: 2018-12-22T12:53:47Z + Scaling Limited: false +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal AutoScalingFleet 35s fleetautoscaler-controller Scaling fleet simple-udp from 2 to 4 +``` + +You can see that the fleet size has increased in particular case doubled to 4 gameservers (based on our custom logic in our webhook), the autoscaler having compensated for the two allocated instances. +Last Scale Time has been updated and a scaling event has been logged. + +Double-check the actual number of game server instances and status by running: + +``` + kubectl get gs -n default +``` + +This will get you a list of all the current `GameSevers` and their `Status > State`. + +``` +NAME STATUS IP PORT +simple-udp-dmkp4-8pkk2 Ready 35.247.13.175 [map[name:default port:7386]] +simple-udp-dmkp4-b7x87 Allocated 35.247.13.175 [map[name:default port:7219]] +simple-udp-dmkp4-r4qtt Allocated 35.247.13.175 [map[name:default port:7220]] +simple-udp-dmkp4-rsr6n Ready 35.247.13.175 [map[name:default port:7297]] +``` + +### 7. Check down scaling using Webhook Autoscaler policy + +Based on our custom webhook deployed earlier, if the fraction of allocated replicas in whole Replicas count would be less that threshold (0.3) than fleet would scale down by scaleFactor, in our example by 2. + +Note that example webhook server have a limitation that it would not decrease fleet replica count under `minReplicasCount`, which is equal to 2. + +We need to run EXIT command on one gameserver (Use IP address and port of the allocated gameserver from the previous step) in order to decrease the number of allocated gameservers in a fleet (<0.3). +``` +nc -u 35.247.13.175 7220 +EXIT +``` + +Server would be in shutdown state. +Wait about 30 seconds. +Then you should see scaling down event in the output of next command: +``` +kubectl describe fleetautoscaler webhook-fleet-autoscaler +``` + +You should see these lines in events: +``` + Normal AutoScalingFleet 11m fleetautoscaler-controller Scaling fleet simple-udp from 2 to 4 + Normal AutoScalingFleet 1m fleetautoscaler-controller Scaling fleet simple-udp from 4 to 2 +``` + +And get gameservers command output: +``` +kubectl get gs -n default +``` + +``` +NAME STATUS IP PORT +simple-udp-884fg-6q5sk Ready 35.247.117.202 7373 +simple-udp-884fg-b7l58 Allocated 35.247.117.202 7766 +``` + +## Next Steps + +Read the advanced [Scheduling and Autoscaling](scheduling_autoscaling.md) guide, for more details on autoscaling. + +If you want to use your own GameServer container make sure you have properly integrated the [Agones SDK](../sdks/). diff --git a/docs/fleetautoscaler_spec.md b/docs/fleetautoscaler_spec.md index f74073ea51..46392d4502 100644 --- a/docs/fleetautoscaler_spec.md +++ b/docs/fleetautoscaler_spec.md @@ -21,6 +21,24 @@ spec: maxReplicas: 20 ``` +Or for Webhook FleetAutoscaler below and in [example folder](../examples/webhookfleetautoscaler.yaml): + +```yaml +apiVersion: "stable.agones.dev/v1alpha1" +kind: FleetAutoscaler +metadata: + name: fleet-autoscaler-example +spec: + + fleetName: fleet-example + policy: + type: Webhook + webhook: + name: "fleet-autoscaler-webhook" + namespace: "default" + path: "/scale" +``` + Since Agones defines a new [Custom Resources Definition (CRD)](https://kubernetes.io/docs/concepts/api-extension/custom-resources/) we can define a new resource using the kind `FleetAutoscaler` with the custom group `stable.agones.dev` and API @@ -31,12 +49,79 @@ The `spec` field is the actual `FleetAutoscaler` specification and it is compose - `fleetName` is name of the fleet to attach to and control. Must be an existing `Fleet` in the same namespace as this `FleetAutoscaler`. - `policy` is the autoscaling policy - - `type` is type of the policy. For now, only "Buffer" is available - - `buffer` parameters of the buffer policy + - `type` is type of the policy. "Buffer" and "Webhook" are available + - `buffer` parameters of the buffer policy type - `bufferSize` is the size of a buffer of "ready" game server instances The FleetAutoscaler will scale the fleet up and down trying to maintain this buffer, as instances are being allocated or terminated it can be specified either in absolute (i.e. 5) or percentage format (i.e. 5%) - `minReplicas` is the minimum fleet size to be set by this FleetAutoscaler. if not specified, the minimum fleet size will be bufferSize - - `maxReplicas` is the maximum fleet size that can be set by this FleetAutoscaler. Required. \ No newline at end of file + - `maxReplicas` is the maximum fleet size that can be set by this FleetAutoscaler. Required. + - `webhook` parameters of the webhook policy type + - `service` is a reference to the service for this webhook. Either `service` or `url` must be specified. If the webhook is running within the cluster, then you should use `service`. Port 8000 will be used if it is open, otherwise it is an error. + - `name` is the service name bound to Deployment of autoscaler webhook. Required [(see example)](../examples/autoscaler-webhook/autoscaler-service.yaml) + The FleetAutoscaler will scale the fleet up and down based on the response from this webhook server + - `namespace` is the kubernetes namespace where webhook is deployed. Optional + If not specified, the "default" would be used + - `path` is an optional URL path which will be sent in any request to this service. (i. e. /scale) + - `url` gives the location of the webhook, in standard URL form (`[scheme://]host:port/path`). Exactly one of `url` or `service` must be specified. The `host` should not refer to a service running in the cluster; use the `service` field instead. (optional, instead of service) + +Note: only one `buffer` or `webhook` could be defined for FleetAutoscaler which is based on the `type` field. + + +# Webhook Endpoint Specification + +Webhook endpoint is used to delegate the scaling logic to a separate pod or server. + +FleetAutoscaler would send a request to the webhook endpoint every sync period (which is currently 30s) with a JSON body, and scale the target fleet based on the data that is returned. +JSON payload with a FleetAutoscaleReview data structure would be sent to webhook endpoint and received from it with FleetAutoscaleResponse field populated. FleetAutoscaleResponse contains target Replica count which would trigger scaling of the fleet according to it. + +The connection to this webhook endpoint should be defined in `FleetAutoscaler` using Webhook policy type. + +```go +type FleetAutoscaleRequest struct { + // UID is an identifier for the individual request/response. It allows us to distinguish instances of requests which are + // otherwise identical (parallel requests, requests when earlier requests did not modify etc) + // The UID is meant to track the round trip (request/response) between the Autoscaler and the WebHook, not the user request. + // It is suitable for correlating log entries between the webhook and apiserver, for either auditing or debugging. + UID types.UID `json:"uid""` + // Name is the name of the Fleet being scaled + Name string `json:"name"` + // Namespace is the namespace associated with the request (if any). + Namespace string `json:"namespace"` + // The Fleet's status values + Status v1alpha1.FleetStatus `json:"status"` +} + +type FleetAutoscaleResponse struct { + // UID is an identifier for the individual request/response. + // This should be copied over from the corresponding FleetAutoscaleRequest. + UID types.UID `json:"uid"` + // Set to false if no scaling should occur to the Fleet + Scale bool `json:"scale"` + // The targeted replica count + Replicas int32 `json:"replicas"` +} + +// FleetAutoscaleReview is passed to the webhook with a populated Request value, +// and then returned with a populated Response. +type FleetAutoscaleReview struct { + Request *FleetAutoscaleRequest `json:"request"` + Response *FleetAutoscaleResponse `json:"response"` +} + +// FleetStatus is the status of a Fleet +type FleetStatus struct { + // Replicas the total number of current GameServer replicas + Replicas int32 `json:"replicas"` + // ReadyReplicas are the number of Ready GameServer replicas + ReadyReplicas int32 `json:"readyReplicas"` + // AllocatedReplicas are the number of Allocated GameServer replicas + AllocatedReplicas int32 `json:"allocatedReplicas"` +} +``` + +The example of the webhook written in Go could be found [here](../examples/autoscaler-webhook/main.go). + +It implements the [scaling logic](../examples/autoscaler-webhook/README.md) based on the percentage of allocated gameservers in a fleet. diff --git a/examples/autoscaler-webhook/Dockerfile b/examples/autoscaler-webhook/Dockerfile new file mode 100644 index 0000000000..4f3961dc3e --- /dev/null +++ b/examples/autoscaler-webhook/Dockerfile @@ -0,0 +1,34 @@ +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Gather dependencies and build the executable +FROM golang:1.11.4 as builder +WORKDIR /go/src/autoscaler-webhook + +COPY examples/autoscaler-webhook/main.go . +COPY . /go/src/agones.dev/agones +RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o server . + +# Create the final image that will run the webhook server for FleetAutoscaler webhook policy +FROM alpine:3.8 +RUN adduser -D server + +COPY --from=builder /go/src/autoscaler-webhook \ + /home/server + +RUN chown -R server /home/server && \ + chmod o+x /home/server/server + +USER server +ENTRYPOINT /home/server/server diff --git a/examples/autoscaler-webhook/Makefile b/examples/autoscaler-webhook/Makefile new file mode 100644 index 0000000000..af14332cd8 --- /dev/null +++ b/examples/autoscaler-webhook/Makefile @@ -0,0 +1,42 @@ +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Makefile for building a simple udp client and server +# + +# __ __ _ _ _ +# \ \ / /_ _ _ __(_) __ _| |__ | | ___ ___ +# \ \ / / _` | '__| |/ _` | '_ \| |/ _ \ __| +# \ V / (_| | | | | (_| | |_) | | __\__ \ +# \_/ \__,_|_| |_|\__,_|_.__/|_|\___|___/ +# + +REPOSITORY = gcr.io/agones-images + +mkfile_path := $(abspath $(lastword $(MAKEFILE_LIST))) +project_path := $(dir $(mkfile_path)) +autoscaler_webhook_tag = $(REPOSITORY)/autoscaler-webhook:0.1 +root_path = $(realpath $(project_path)/../..) + +# _____ _ +# |_ _|_ _ _ __ __ _ ___| |_ ___ +# | |/ _` | '__/ _` |/ _ \ __/ __| +# | | (_| | | | (_| | __/ |_\__ \ +# |_|\__,_|_| \__, |\___|\__|___/ +# |___/ + +# Build a docker image for the server, and tag it +build: + cd $(root_path) && docker build -f $(project_path)/Dockerfile --tag=$(autoscaler_webhook_tag) . diff --git a/examples/autoscaler-webhook/README.md b/examples/autoscaler-webhook/README.md new file mode 100644 index 0000000000..ce07f6440d --- /dev/null +++ b/examples/autoscaler-webhook/README.md @@ -0,0 +1,12 @@ +# Simple Webhook Autoscaler Service + +This service provides an example of the webhook fleetautoscaler service which is used to control the number of GameServers in a Fleet (`Replica` count). + +## Autoscaler Service +The service exposes an endpoint which allows client calls to custom scaling logic. + +When this endpoint is called, target Replica count gets calculated. If fleet don't need to scale we return nil as a `Replica` parameter. Endpoint receives and returns the JSON encoded [`FleetAutoscaleReview`](./fleetautoscaler_spec.md#webhook-endpoint-specification) . + +Note that scaling up logic is based on the percentage of allocated gameservers in a fleet. If this fraction is more than threshold (i. e. 0.7) than `Replica` value is returned increased by the `scaleFactor` (in this example twice) which results in creating more Ready GameServers. If the fraction below the threshold (i. e. 0.3) we decrease a count of gameservers. + +To learn how to deploy the fleet to GKE, please see the tutorial [Create a Fleet (Go)](../../docs/create_fleet.md). \ No newline at end of file diff --git a/examples/autoscaler-webhook/autoscaler-service.yaml b/examples/autoscaler-webhook/autoscaler-service.yaml new file mode 100644 index 0000000000..7778314f2c --- /dev/null +++ b/examples/autoscaler-webhook/autoscaler-service.yaml @@ -0,0 +1,64 @@ +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Define a Service for the autoscaler-webhook + +apiVersion: v1 +kind: Service +metadata: + name: autoscaler-webhook-service + labels: + app: autoscaler-webhook +spec: + selector: + app: autoscaler-webhook + ports: + - port: 8000 + protocol: TCP + name: https + targetPort: autoscaler +--- +# Deploy a pod to run the autoscaler-webhook code +apiVersion: apps/v1 +kind: Deployment +metadata: + name: autoscaler-webhook + namespace: default + labels: + app: autoscaler-webhook +spec: + replicas: 1 + selector: + matchLabels: + app: autoscaler-webhook + template: + metadata: + labels: + app: autoscaler-webhook + spec: + #serviceAccount: autoscaler-webhook + containers: + - name: autoscaler-webhook + image: gcr.io/agones-images/autoscaler-webhook:0.1 + imagePullPolicy: Always + ports: + - name: autoscaler + containerPort: 8000 + livenessProbe: + httpGet: + scheme: HTTP + path: /health + port: 8000 + initialDelaySeconds: 3 + periodSeconds: 5 diff --git a/examples/autoscaler-webhook/main.go b/examples/autoscaler-webhook/main.go new file mode 100644 index 0000000000..58800c01d3 --- /dev/null +++ b/examples/autoscaler-webhook/main.go @@ -0,0 +1,111 @@ +// Copyright 2018 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//Autoscaler webhook server which handles FleetAutoscaleReview json payload +package main + +import ( + "encoding/json" + "io" + "io/ioutil" + "net/http" + + v1alpha1 "agones.dev/agones/pkg/apis/stable/v1alpha1" + "agones.dev/agones/pkg/util/runtime" // for the logger +) + +// Constants which define thresholds to trigger scalling up and scale factor +const ( + replicaUpperThreshold = 0.7 + replicaLowerThreshold = 0.3 + scaleFactor = 2 + minReplicasCount = 2 +) + +// Variables for the logger +var ( + logger = runtime.NewLoggerWithSource("main") +) + +// Main will set up an http server and three endpoints +func main() { + // Serve 200 status on /health for k8s health checks + http.HandleFunc("/health", handleHealth) + + // Return the target replica count which is used by Webhook fleet autoscaling policy + http.HandleFunc("/scale", handleAutoscale) + + logger.Info("Starting HTTP server on port 8000") + if err := http.ListenAndServe(":8000", nil); err != nil { + logger.WithError(err).Fatal("HTTP server failed to run") + } +} + +// Let /health return Healthy and status code 200 +func handleHealth(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, err := io.WriteString(w, "Healthy") + if err != nil { + logger.WithError(err).Fatal("Error writing string Healthy from /health") + } +} + +// handleAutoscale is a handler function which return the replica count +// based on received status of the fleet +func handleAutoscale(w http.ResponseWriter, r *http.Request) { + if r == nil { + http.Error(w, "Empty request", http.StatusInternalServerError) + return + } + + var faReq v1alpha1.FleetAutoscaleReview + res, err := ioutil.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + err = json.Unmarshal(res, &faReq) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + faResp := v1alpha1.FleetAutoscaleResponse{ + Scale: false, + Replicas: faReq.Request.Status.Replicas, + UID: faReq.Request.UID, + } + if faReq.Request.Status.Replicas != 0 { + allocatedPercent := float32(faReq.Request.Status.AllocatedReplicas) / float32(faReq.Request.Status.Replicas) + if allocatedPercent > replicaUpperThreshold { + // After scaling we would have percentage of 0.7/2 = 0.35 > replicaLowerThreshold + // So we won't scale down immediately after scale up + faResp.Scale = true + faResp.Replicas = faReq.Request.Status.Replicas * scaleFactor + } else if allocatedPercent < replicaLowerThreshold && faReq.Request.Status.Replicas > minReplicasCount { + faResp.Scale = true + faResp.Replicas = faReq.Request.Status.Replicas / scaleFactor + } + } + w.Header().Set("Content-Type", "application/json") + review := &v1alpha1.FleetAutoscaleReview{ + Request: faReq.Request, + Response: &faResp, + } + logger.Infof("FleetAutoscaleReview Request: %+v ; Response: %+v", *review.Request, *review.Response) + result, _ := json.Marshal(&review) + + _, err = io.WriteString(w, string(result)) + if err != nil { + logger.WithError(err).Fatal("Error writing json from /scale") + } +} diff --git a/examples/fleet.yaml b/examples/fleet.yaml index 7ef1eec9c5..cddbd1dcde 100644 --- a/examples/fleet.yaml +++ b/examples/fleet.yaml @@ -66,5 +66,5 @@ spec: template: spec: containers: - - name: example-server - image: gcr.io/agones/test-server:0.1 \ No newline at end of file + - name: simple-udp + image: gcr.io/agones-images/udp-server:0.5 \ No newline at end of file diff --git a/examples/webhookfleetautoscaler.yaml b/examples/webhookfleetautoscaler.yaml new file mode 100644 index 0000000000..1e683ad8b4 --- /dev/null +++ b/examples/webhookfleetautoscaler.yaml @@ -0,0 +1,37 @@ +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Full example of a FleetAutoscaler - this is used to scale a Fleet +# automatically depending on load +# +apiVersion: "stable.agones.dev/v1alpha1" +kind: FleetAutoscaler +metadata: + name: webhook-fleet-autoscaler +spec: + fleetName: simple-udp + policy: + # type of the policy - this example is Webhook + type: Webhook + # parameters for the webhook policy - this is a WebhookClientConfig, as per other K8s webhooks + webhook: + # use a service, or URL + service: + name: autoscaler-webhook-service + namespace: default + path: scale + # optional for URL defined webhooks + # url: "" + # caBundle: TBD (optional, if you want to provide your own ca cert to test against) \ No newline at end of file diff --git a/install/helm/agones/templates/crds/fleetautoscaler.yaml b/install/helm/agones/templates/crds/fleetautoscaler.yaml index 4d02ff4fdb..145f644f00 100644 --- a/install/helm/agones/templates/crds/fleetautoscaler.yaml +++ b/install/helm/agones/templates/crds/fleetautoscaler.yaml @@ -55,6 +55,7 @@ spec: type: string enum: - Buffer + - Webhook buffer: required: - maxReplicas @@ -65,4 +66,16 @@ spec: maxReplicas: type: integer minimum: 1 + webhook: + properties: + service: + properties: + name: + type: string + namespace: + type: string + path: + type: string + url: + type: string {{- end }} \ No newline at end of file diff --git a/install/yaml/install.yaml b/install/yaml/install.yaml index 88e999f346..34ff55bc82 100644 --- a/install/yaml/install.yaml +++ b/install/yaml/install.yaml @@ -442,6 +442,7 @@ spec: type: string enum: - Buffer + - Webhook buffer: required: - maxReplicas @@ -452,6 +453,18 @@ spec: maxReplicas: type: integer minimum: 1 + webhook: + properties: + service: + properties: + name: + type: string + namespace: + type: string + path: + type: string + url: + type: string --- # Source: agones/templates/crds/gameserver.yaml # Copyright 2018 Google Inc. All Rights Reserved. diff --git a/pkg/apis/stable/v1alpha1/fleetautoscaler.go b/pkg/apis/stable/v1alpha1/fleetautoscaler.go index d4f6dfb180..bf0370ce7c 100644 --- a/pkg/apis/stable/v1alpha1/fleetautoscaler.go +++ b/pkg/apis/stable/v1alpha1/fleetautoscaler.go @@ -14,8 +14,12 @@ package v1alpha1 -import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -import "k8s.io/apimachinery/pkg/util/intstr" +import ( + admregv1b "k8s.io/api/admissionregistration/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" +) // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -55,6 +59,9 @@ type FleetAutoscalerPolicy struct { // Buffer policy config params. Present only if FleetAutoscalerPolicyType = Buffer. // +optional Buffer *BufferPolicy `json:"buffer,omitempty"` + // Webhook policy config params. Present only if FleetAutoscalerPolicyType = Webhook. + // +optional + Webhook *WebhookPolicy `json:"webhook,omitempty"` } // FleetAutoscalerPolicyType is the policy for autoscaling @@ -65,6 +72,9 @@ const ( // BufferPolicyType FleetAutoscalerPolicyType is a simple buffering strategy for Ready // GameServers BufferPolicyType FleetAutoscalerPolicyType = "Buffer" + // WebhookPolicyType is a simple webhook strategy used for horizontal fleet scaling + // GameServers + WebhookPolicyType FleetAutoscalerPolicyType = "Webhook" ) // BufferPolicy controls the desired behavior of the buffer policy. @@ -91,6 +101,11 @@ type BufferPolicy struct { BufferSize intstr.IntOrString `json:"bufferSize"` } +// WebhookPolicy controls the desired behavior of the webhook policy. +// It contains the description of the webhook autoscaler service +// used to form url which is accessible inside the cluster +type WebhookPolicy admregv1b.WebhookClientConfig + // FleetAutoscalerStatus defines the current status of a FleetAutoscaler type FleetAutoscalerStatus struct { // CurrentReplicas is the current number of gameserver replicas @@ -113,10 +128,73 @@ type FleetAutoscalerStatus struct { ScalingLimited bool `json:"scalingLimited"` } +// FleetAutoscaleRequest defines the request to webhook autoscaler endpoint +type FleetAutoscaleRequest struct { + // UID is an identifier for the individual request/response. It allows us to distinguish instances of requests which are + // otherwise identical (parallel requests, requests when earlier requests did not modify etc) + // The UID is meant to track the round trip (request/response) between the Autoscaler and the WebHook, not the user request. + // It is suitable for correlating log entries between the webhook and apiserver, for either auditing or debugging. + UID types.UID `json:"uid"` + // Name is the name of the Fleet being scaled + Name string `json:"name"` + // Namespace is the namespace associated with the request (if any). + Namespace string `json:"namespace"` + // The Fleet's status values + Status FleetStatus `json:"status"` +} + +// FleetAutoscaleResponse defines the response of webhook autoscaler endpoint +type FleetAutoscaleResponse struct { + // UID is an identifier for the individual request/response. + // This should be copied over from the corresponding FleetAutoscaleRequest. + UID types.UID `json:"uid"` + // Set to false if no scaling should occur to the Fleet + Scale bool `json:"scale"` + // The targeted replica count + Replicas int32 `json:"replicas"` +} + +// FleetAutoscaleReview is passed to the webhook with a populated Request value, +// and then returned with a populated Response. +type FleetAutoscaleReview struct { + Request *FleetAutoscaleRequest `json:"request"` + Response *FleetAutoscaleResponse `json:"response"` +} + // Validate validates the FleetAutoscaler scaling settings func (fas *FleetAutoscaler) Validate(causes []metav1.StatusCause) []metav1.StatusCause { - if fas.Spec.Policy.Type == BufferPolicyType { + switch fas.Spec.Policy.Type { + case BufferPolicyType: causes = fas.Spec.Policy.Buffer.ValidateBufferPolicy(causes) + + case WebhookPolicyType: + causes = fas.Spec.Policy.Webhook.ValidateWebhookPolicy(causes) + } + return causes +} + +// ValidateWebhookPolicy validates the FleetAutoscaler Webhook policy settings +func (w *WebhookPolicy) ValidateWebhookPolicy(causes []metav1.StatusCause) []metav1.StatusCause { + if w == nil { + return append(causes, metav1.StatusCause{ + Type: metav1.CauseTypeFieldValueInvalid, + Field: "webhook", + Message: "webhook policy config params are missing", + }) + } + if w.Service == nil && w.URL == nil { + causes = append(causes, metav1.StatusCause{ + Type: metav1.CauseTypeFieldValueNotFound, + Field: "url", + Message: "url should be provided", + }) + } + if w.Service != nil && w.URL != nil { + causes = append(causes, metav1.StatusCause{ + Type: metav1.CauseTypeFieldValueNotFound, + Field: "url", + Message: "service and url cannot be used simultaneously", + }) } return causes } diff --git a/pkg/apis/stable/v1alpha1/fleetautoscaler_test.go b/pkg/apis/stable/v1alpha1/fleetautoscaler_test.go index c6b4841bf1..67b164d646 100644 --- a/pkg/apis/stable/v1alpha1/fleetautoscaler_test.go +++ b/pkg/apis/stable/v1alpha1/fleetautoscaler_test.go @@ -18,6 +18,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + admregv1b "k8s.io/api/admissionregistration/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" ) @@ -99,9 +100,60 @@ func TestFleetAutoscalerValidateUpdate(t *testing.T) { assert.Equal(t, "bufferSize", causes[0].Field) }) } +func TestFleetAutoscalerWebhookValidateUpdate(t *testing.T) { + t.Parallel() + + t.Run("good service value", func(t *testing.T) { + fas := webhookFixture() + causes := fas.Validate(nil) + + assert.Len(t, causes, 0) + }) + + t.Run("good url value", func(t *testing.T) { + fas := webhookFixture() + causes := fas.Validate(nil) + url := "http://good.example.com" + fas.Spec.Policy.Webhook.URL = &url + fas.Spec.Policy.Webhook.Service = nil + + assert.Len(t, causes, 0) + }) + + t.Run("bad URL and service value", func(t *testing.T) { + fas := webhookFixture() + fas.Spec.Policy.Webhook.URL = nil + fas.Spec.Policy.Webhook.Service = nil + causes := fas.Validate(nil) + + assert.Len(t, causes, 1) + assert.Equal(t, "url", causes[0].Field) + }) + + t.Run("both URL and service value are used - fail", func(t *testing.T) { + + fas := webhookFixture() + url := "123" + fas.Spec.Policy.Webhook.URL = &url + + causes := fas.Validate(nil) + + assert.Len(t, causes, 1) + assert.Equal(t, "url", causes[0].Field) + }) + +} func defaultFixture() *FleetAutoscaler { - return &FleetAutoscaler{ + return customFixture(BufferPolicyType) +} + +func webhookFixture() *FleetAutoscaler { + return customFixture(WebhookPolicyType) +} + +func customFixture(t FleetAutoscalerPolicyType) *FleetAutoscaler { + res := &FleetAutoscaler{ ObjectMeta: metav1.ObjectMeta{Name: "test"}, Spec: FleetAutoscalerSpec{ FleetName: "testing", @@ -114,4 +166,19 @@ func defaultFixture() *FleetAutoscaler { }, }, } + switch t { + case BufferPolicyType: + case WebhookPolicyType: + res.Spec.Policy.Type = WebhookPolicyType + res.Spec.Policy.Buffer = nil + url := "/scale" + res.Spec.Policy.Webhook = &WebhookPolicy{ + Service: &admregv1b.ServiceReference{ + Name: "service1", + Namespace: "default", + Path: &url, + }, + } + } + return res } diff --git a/pkg/apis/stable/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/stable/v1alpha1/zz_generated.deepcopy.go index d85b58df42..a6645bfae1 100644 --- a/pkg/apis/stable/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/stable/v1alpha1/zz_generated.deepcopy.go @@ -21,6 +21,7 @@ package v1alpha1 import ( + v1beta1 "k8s.io/api/admissionregistration/v1beta1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -202,6 +203,73 @@ func (in *FleetAllocationStatus) DeepCopy() *FleetAllocationStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FleetAutoscaleRequest) DeepCopyInto(out *FleetAutoscaleRequest) { + *out = *in + out.Status = in.Status + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FleetAutoscaleRequest. +func (in *FleetAutoscaleRequest) DeepCopy() *FleetAutoscaleRequest { + if in == nil { + return nil + } + out := new(FleetAutoscaleRequest) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FleetAutoscaleResponse) DeepCopyInto(out *FleetAutoscaleResponse) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FleetAutoscaleResponse. +func (in *FleetAutoscaleResponse) DeepCopy() *FleetAutoscaleResponse { + if in == nil { + return nil + } + out := new(FleetAutoscaleResponse) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FleetAutoscaleReview) DeepCopyInto(out *FleetAutoscaleReview) { + *out = *in + if in.Request != nil { + in, out := &in.Request, &out.Request + if *in == nil { + *out = nil + } else { + *out = new(FleetAutoscaleRequest) + **out = **in + } + } + if in.Response != nil { + in, out := &in.Response, &out.Response + if *in == nil { + *out = nil + } else { + *out = new(FleetAutoscaleResponse) + **out = **in + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FleetAutoscaleReview. +func (in *FleetAutoscaleReview) DeepCopy() *FleetAutoscaleReview { + if in == nil { + return nil + } + out := new(FleetAutoscaleReview) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *FleetAutoscaler) DeepCopyInto(out *FleetAutoscaler) { *out = *in @@ -275,6 +343,15 @@ func (in *FleetAutoscalerPolicy) DeepCopyInto(out *FleetAutoscalerPolicy) { **out = **in } } + if in.Webhook != nil { + in, out := &in.Webhook, &out.Webhook + if *in == nil { + *out = nil + } else { + *out = new(WebhookPolicy) + (*in).DeepCopyInto(*out) + } + } return } @@ -660,3 +737,42 @@ func (in *Health) DeepCopy() *Health { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WebhookPolicy) DeepCopyInto(out *WebhookPolicy) { + *out = *in + if in.URL != nil { + in, out := &in.URL, &out.URL + if *in == nil { + *out = nil + } else { + *out = new(string) + **out = **in + } + } + if in.Service != nil { + in, out := &in.Service, &out.Service + if *in == nil { + *out = nil + } else { + *out = new(v1beta1.ServiceReference) + (*in).DeepCopyInto(*out) + } + } + if in.CABundle != nil { + in, out := &in.CABundle, &out.CABundle + *out = make([]byte, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WebhookPolicy. +func (in *WebhookPolicy) DeepCopy() *WebhookPolicy { + if in == nil { + return nil + } + out := new(WebhookPolicy) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/fleetautoscalers/controller_test.go b/pkg/fleetautoscalers/controller_test.go index 3f3e98a7eb..b49fd86d7c 100644 --- a/pkg/fleetautoscalers/controller_test.go +++ b/pkg/fleetautoscalers/controller_test.go @@ -25,6 +25,7 @@ import ( "github.com/heptiolabs/healthcheck" "github.com/stretchr/testify/assert" admv1beta1 "k8s.io/api/admission/v1beta1" + admregv1b "k8s.io/api/admissionregistration/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" @@ -73,6 +74,45 @@ func TestControllerCreationValidationHandler(t *testing.T) { }) } +func TestWebhookControllerCreationValidationHandler(t *testing.T) { + t.Parallel() + + t.Run("valid fleet autoscaler", func(t *testing.T) { + c, m := newFakeController() + fas, _ := defaultWebhookFixtures() + _, cancel := agtesting.StartInformers(m) + defer cancel() + + review, err := newAdmissionReview(*fas) + assert.Nil(t, err) + + result, err := c.validationHandler(review) + assert.Nil(t, err) + assert.True(t, result.Response.Allowed, fmt.Sprintf("%#v", result.Response)) + }) + + t.Run("invalid fleet autoscaler", func(t *testing.T) { + c, m := newFakeController() + fas, _ := defaultWebhookFixtures() + // this make it invalid + fas.Spec.Policy.Webhook = nil + + _, cancel := agtesting.StartInformers(m) + defer cancel() + + review, err := newAdmissionReview(*fas) + assert.Nil(t, err) + + result, err := c.validationHandler(review) + fmt.Printf("%+v", result) + assert.Nil(t, err) + assert.False(t, result.Response.Allowed, fmt.Sprintf("%#v", result.Response)) + assert.Equal(t, metav1.StatusFailure, result.Response.Result.Status) + assert.Equal(t, metav1.StatusReasonInvalid, result.Response.Result.Reason) + assert.NotEmpty(t, result.Response.Result.Details) + }) +} + // nolint:dupl func TestControllerSyncFleetAutoscaler(t *testing.T) { t.Parallel() @@ -445,6 +485,21 @@ func defaultFixtures() (*v1alpha1.FleetAutoscaler, *v1alpha1.Fleet) { return fas, f } +func defaultWebhookFixtures() (*v1alpha1.FleetAutoscaler, *v1alpha1.Fleet) { + fas, f := defaultFixtures() + fas.Spec.Policy.Type = v1alpha1.WebhookPolicyType + fas.Spec.Policy.Buffer = nil + url := "/autoscaler" + fas.Spec.Policy.Webhook = &v1alpha1.WebhookPolicy{ + Service: &admregv1b.ServiceReference{ + Name: "fleetautoscaler-service", + Path: &url, + }, + } + + return fas, f +} + // newFakeController returns a controller, backed by the fake Clientset func newFakeController() (*Controller, agtesting.Mocks) { m := agtesting.NewMocks() diff --git a/pkg/fleetautoscalers/fleetautoscalers.go b/pkg/fleetautoscalers/fleetautoscalers.go index 5c1f510775..777973a38c 100644 --- a/pkg/fleetautoscalers/fleetautoscalers.go +++ b/pkg/fleetautoscalers/fleetautoscalers.go @@ -17,20 +17,94 @@ package fleetautoscalers import ( + "encoding/json" + "fmt" + "io/ioutil" "math" + "net/http" + "strings" + "time" + "agones.dev/agones/pkg/apis/stable/v1alpha1" stablev1alpha1 "agones.dev/agones/pkg/apis/stable/v1alpha1" + "github.com/pkg/errors" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/uuid" ) +var client = http.Client{ + Timeout: 15 * time.Second, +} + // computeDesiredFleetSize computes the new desired size of the given fleet func computeDesiredFleetSize(fas *stablev1alpha1.FleetAutoscaler, f *stablev1alpha1.Fleet) (int32, bool, error) { switch fas.Spec.Policy.Type { case stablev1alpha1.BufferPolicyType: return applyBufferPolicy(fas.Spec.Policy.Buffer, f) + case stablev1alpha1.WebhookPolicyType: + return applyWebhookPolicy(fas.Spec.Policy.Webhook, f) } + return f.Status.Replicas, false, errors.New("wrong policy type") +} + +func applyWebhookPolicy(w *stablev1alpha1.WebhookPolicy, f *stablev1alpha1.Fleet) (int32, bool, error) { + faReq := v1alpha1.FleetAutoscaleReview{ + Request: &v1alpha1.FleetAutoscaleRequest{ + UID: uuid.NewUUID(), + Name: f.Name, + Namespace: f.Namespace, + Status: f.Status, + }, + Response: nil, + } + b, err := json.Marshal(faReq) + url := "" + if w.URL != nil { + url = *w.URL + } + var faResp v1alpha1.FleetAutoscaleReview + servicePath := "" + if w.Service != nil { + if w.Service.Path != nil { + servicePath = *w.Service.Path + } + if err != nil { + return f.Status.Replicas, false, err + } + + if w.Service.Namespace == "" { + w.Service.Namespace = "default" + } + url = fmt.Sprintf("http://%s.%s.svc:8000/%s", w.Service.Name, w.Service.Namespace, servicePath) + } + if url == "" { + return f.Status.Replicas, false, errors.New("URL was not provided") + } + res, err := client.Post( + url, + "application/json", + strings.NewReader(string(b)), + ) + if err != nil { + return f.Status.Replicas, false, err + } + defer res.Body.Close() // nolint: errcheck + if res.StatusCode != http.StatusOK { + return f.Status.Replicas, false, errors.New("bad status code from the server") + } + result, err := ioutil.ReadAll(res.Body) + if err != nil { + return f.Status.Replicas, false, err + } + err = json.Unmarshal(result, &faResp) + if err != nil { + return f.Status.Replicas, false, err + } + if faResp.Response.Scale { + return faResp.Response.Replicas, false, nil + } return f.Status.Replicas, false, nil } diff --git a/pkg/fleetautoscalers/fleetautoscalers_test.go b/pkg/fleetautoscalers/fleetautoscalers_test.go index c68093dc55..63569c0f73 100644 --- a/pkg/fleetautoscalers/fleetautoscalers_test.go +++ b/pkg/fleetautoscalers/fleetautoscalers_test.go @@ -17,12 +17,22 @@ package fleetautoscalers import ( + "encoding/json" + "io" + "io/ioutil" + "net/http" + "net/http/httptest" "testing" + "agones.dev/agones/pkg/apis/stable/v1alpha1" "github.com/stretchr/testify/assert" "k8s.io/apimachinery/pkg/util/intstr" ) +const ( + scaleFactor = 2 +) + func TestComputeDesiredFleetSize(t *testing.T) { t.Parallel() @@ -44,7 +54,7 @@ func TestComputeDesiredFleetSize(t *testing.T) { f.Status.Replicas = 61 fas.Spec.Policy.Type = "" replicas, limited, err = computeDesiredFleetSize(fas, f) - assert.Nil(t, err) + assert.NotNil(t, err) assert.Equal(t, replicas, int32(61)) assert.Equal(t, limited, false) } @@ -112,3 +122,85 @@ func TestApplyBufferPolicy(t *testing.T) { assert.Equal(t, replicas, int32(2)) assert.Equal(t, limited, false) } + +type testServer struct{} + +func (t testServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r == nil { + http.Error(w, "Empty request", http.StatusInternalServerError) + return + } + + var faRequest v1alpha1.FleetAutoscaleReview + + res, err := ioutil.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + err = json.Unmarshal(res, &faRequest) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + faReq := faRequest.Request + faResp := v1alpha1.FleetAutoscaleResponse{ + Scale: false, + Replicas: faReq.Status.Replicas, + UID: faReq.UID, + } + allocatedPercent := float32(faReq.Status.AllocatedReplicas) / float32(faReq.Status.Replicas) + if allocatedPercent > 0.7 { + faResp.Scale = true + faResp.Replicas = faReq.Status.Replicas * scaleFactor + } + w.Header().Set("Content-Type", "application/json") + review := &v1alpha1.FleetAutoscaleReview{ + Request: faReq, + Response: &faResp, + } + result, _ := json.Marshal(&review) + + _, err = io.WriteString(w, string(result)) + if err != nil { + http.Error(w, "Error writing json from /address", http.StatusInternalServerError) + } +} + +func TestApplyWebhookPolicy(t *testing.T) { + t.Parallel() + + fas, f := defaultWebhookFixtures() + w := fas.Spec.Policy.Webhook + w.Service = nil + + ts := testServer{} + server := httptest.NewServer(ts) + defer server.Close() + w.URL = &(server.URL) + f.Spec.Replicas = 50 + f.Status.Replicas = f.Spec.Replicas + f.Status.AllocatedReplicas = 10 + f.Status.ReadyReplicas = 40 + + replicas, limited, err := applyWebhookPolicy(w, f) + assert.Nil(t, err) + assert.Equal(t, f.Spec.Replicas, replicas) + assert.Equal(t, limited, false) + + f.Spec.Replicas = 50 + f.Status.Replicas = f.Spec.Replicas + f.Status.AllocatedReplicas = 40 + f.Status.ReadyReplicas = 10 + replicas, limited, err = applyWebhookPolicy(w, f) + assert.Nil(t, err) + assert.Equal(t, f.Status.Replicas*scaleFactor, replicas) + assert.Equal(t, limited, false) + + f.Spec.Replicas = 50 + f.Status.Replicas = f.Spec.Replicas + f.Status.AllocatedReplicas = 35 + f.Status.ReadyReplicas = 15 + replicas, limited, err = applyWebhookPolicy(w, f) + assert.Nil(t, err) + assert.Equal(t, replicas, f.Spec.Replicas) + assert.Equal(t, limited, false) +} diff --git a/test/e2e/fleetautoscaler_test.go b/test/e2e/fleetautoscaler_test.go index fe5dac958b..dc2aa3b72a 100644 --- a/test/e2e/fleetautoscaler_test.go +++ b/test/e2e/fleetautoscaler_test.go @@ -23,6 +23,8 @@ import ( e2e "agones.dev/agones/test/e2e/framework" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" + admregv1b "k8s.io/api/admissionregistration/v1beta1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" @@ -221,3 +223,108 @@ func getAllocation(f *v1alpha1.Fleet) *v1alpha1.FleetAllocation { }, } } + +//Test fleetautoscaler with webhook policy type +// scaling from Replicas equals to 1 to 2 +func TestAutoscalerWebhook(t *testing.T) { + t.Parallel() + pod, svc := defaultAutoscalerWebhook() + pod, err := framework.KubeClient.CoreV1().Pods(defaultNs).Create(pod) + if assert.Nil(t, err) { + defer framework.KubeClient.CoreV1().Pods(defaultNs).Delete(pod.ObjectMeta.Name, nil) // nolint:errcheck + } else { + // if we could not create the autoscaler, their is no point going further + logrus.Error("Failed creating autoscaler, aborting TestAutoscalerBasicFunctions") + return + } + svc, err = framework.KubeClient.CoreV1().Services(defaultNs).Create(svc) + if assert.Nil(t, err) { + defer framework.KubeClient.CoreV1().Services(defaultNs).Delete(svc.ObjectMeta.Name, nil) // nolint:errcheck + } else { + // if we could not create the autoscaler, their is no point going further + logrus.Error("Failed creating autoscaler, aborting TestAutoscalerBasicFunctions") + return + } + + alpha1 := framework.AgonesClient.StableV1alpha1() + fleets := alpha1.Fleets(defaultNs) + flt := defaultFleet() + initialReplicasCount := int32(1) + flt.Spec.Replicas = initialReplicasCount + flt, err = fleets.Create(flt) + if assert.Nil(t, err) { + defer fleets.Delete(flt.ObjectMeta.Name, nil) // nolint:errcheck + } + + err = framework.WaitForFleetCondition(flt, e2e.FleetReadyCount(flt.Spec.Replicas)) + assert.Nil(t, err, "fleet not ready") + + fleetautoscalers := alpha1.FleetAutoscalers(defaultNs) + fas := defaultFleetAutoscaler(flt) + fas.Spec.Policy.Type = v1alpha1.WebhookPolicyType + fas.Spec.Policy.Buffer = nil + path := "scale" + fas.Spec.Policy.Webhook = &v1alpha1.WebhookPolicy{ + Service: &admregv1b.ServiceReference{ + Name: svc.ObjectMeta.Name, + Namespace: defaultNs, + Path: &path, + }, + } + fas, err = fleetautoscalers.Create(fas) + if assert.Nil(t, err) { + defer fleetautoscalers.Delete(fas.ObjectMeta.Name, nil) // nolint:errcheck + } else { + // if we could not create the autoscaler, their is no point going further + logrus.Error("Failed creating autoscaler, aborting TestAutoscalerBasicFunctions") + return + } + fa := getAllocation(flt) + fa, err = alpha1.FleetAllocations(defaultNs).Create(fa) + assert.Nil(t, err) + assert.Equal(t, v1alpha1.Allocated, fa.Status.GameServer.Status.State) + err = framework.WaitForFleetCondition(flt, func(fleet *v1alpha1.Fleet) bool { + return fleet.Status.AllocatedReplicas == 1 + }) + assert.Nil(t, err) + + err = framework.WaitForFleetCondition(flt, func(fleet *v1alpha1.Fleet) bool { + return fleet.Status.Replicas == 2*initialReplicasCount + }) + assert.Nil(t, err) +} + +func defaultAutoscalerWebhook() (*corev1.Pod, *corev1.Service) { + l := make(map[string]string) + l["app"] = "autoscaler-webhook" + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + GenerateName: "auto-webhook", + Namespace: defaultNs, + Labels: l, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: "webhook", + Image: "gcr.io/agones-images/autoscaler-webhook:0.1", + ImagePullPolicy: corev1.PullIfNotPresent, + Ports: []corev1.ContainerPort{{ + ContainerPort: 8000, + Name: "autoscaler", + }}, + }}, + }, + } + m := make(map[string]string) + m["app"] = "autoscaler-webhook" + service := &corev1.Service{ObjectMeta: metav1.ObjectMeta{GenerateName: "auto-webhook", Namespace: defaultNs}, + Spec: corev1.ServiceSpec{ + Selector: m, + Ports: []corev1.ServicePort{{ + Name: "newport", + Port: 8000, + TargetPort: intstr.IntOrString{StrVal: "autoscaler"}, + }}, + }, + } + + return pod, service +}