add hpa for raw deployment (kubeflow#1830)

* add autoscaler hpa

* update validation

* add test code

* add behavior default value

* add validation for target utilization percentage

config/rbac/role.yaml

@@ -31,6 +31,18 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - autoscaling
+  resources:
+  - horizontalpodautoscalers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - ""
   resources:

pkg/apis/serving/v1beta1/inference_service_validation.go

@@ -18,9 +18,11 @@ package v1beta1
 import (
 	"fmt"
 	"reflect"
+	"strconv"
 
 	"regexp"
 
+	"github.com/kserve/kserve/pkg/constants"
 	"github.com/kserve/kserve/pkg/utils"
 	"k8s.io/apimachinery/pkg/runtime"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
@@ -50,6 +52,13 @@ func (isvc *InferenceService) ValidateCreate() error {
 		return err
 	}
 
+	if err := validateInferenceServiceAutoscaler(isvc); err != nil {
+		return err
+	}
+
+	if err := validateAutoscalerTargetUtilizationPercentage(isvc); err != nil {
+		return err
+	}
 	for _, component := range []Component{
 		&isvc.Spec.Predictor,
 		isvc.Spec.Transformer,
@@ -96,3 +105,56 @@ func validateInferenceServiceName(isvc *InferenceService) error {
 	}
 	return nil
 }
+
+//Validation of isvc autoscaler class
+func validateInferenceServiceAutoscaler(isvc *InferenceService) error {
+	annotations := isvc.ObjectMeta.Annotations
+	value, ok := annotations[constants.AutoscalerClass]
+	class := constants.AutoscalerClassType(value)
+	if ok {
+		for _, item := range constants.AutoscalerAllowedClassList {
+			if class == item {
+				switch class {
+				case constants.AutoscalerClassHPA:
+					if metric, ok := annotations[constants.AutoscalerMetrics]; ok {
+						return validateHPAMetrics(metric)
+					} else {
+						return nil
+					}
+				default:
+					return fmt.Errorf("unknown autoscaler class [%s]", class)
+				}
+			}
+		}
+		return fmt.Errorf("[%s] is not a supported autoscaler class type.\n", value)
+	}
+
+	return nil
+}
+
+//Validate of autoscaler HPA metrics
+func validateHPAMetrics(metric string) error {
+	for _, item := range constants.AutoscalerAllowedMetricsList {
+		if item == constants.AutoscalerMetricsType(metric) {
+			return nil
+		}
+	}
+	return fmt.Errorf("[%s] is not a supported metric.\n", metric)
+
+}
+
+//Validate of autoscaler targetUtilizationPercentage
+func validateAutoscalerTargetUtilizationPercentage(isvc *InferenceService) error {
+	annotations := isvc.ObjectMeta.Annotations
+	if value, ok := annotations[constants.TargetUtilizationPercentage]; ok {
+		t, err := strconv.Atoi(value)
+		if err != nil {
+			return fmt.Errorf("The target utilization percentage should be a [1-100] integer.")
+		} else {
+			if t < 1 || t > 100 {
+				return fmt.Errorf("The target utilization percentage should be a [1-100] integer.")
+			}
+		}
+	}
+	return nil
+}

pkg/apis/serving/v1beta1/inference_service_validation_test.go

@@ -26,6 +26,32 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
+func makeTestRawInferenceService() InferenceService {
+	inferenceservice := InferenceService{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "foo",
+			Namespace: "default",
+			Annotations: map[string]string{
+				"serving.kserve.io/deploymentMode":              "RawDeployment",
+				"serving.kserve.io/autoscalerClass":             "hpa",
+				"serving.kserve.io/metrics":                     "cpu",
+				"serving.kserve.io/targetUtilizationPercentage": "75",
+			},
+		},
+		Spec: InferenceServiceSpec{
+			Predictor: PredictorSpec{
+				Tensorflow: &TFServingSpec{
+					PredictorExtensionSpec: PredictorExtensionSpec{
+						StorageURI:     proto.String("gs://testbucket/testmodel"),
+						RuntimeVersion: proto.String("0.14.0"),
+					},
+				},
+			},
+		},
+	}
+	return inferenceservice
+}
+
 func makeTestInferenceService() InferenceService {
 	inferenceservice := InferenceService{
 		ObjectMeta: metav1.ObjectMeta{
@@ -46,6 +72,49 @@ func makeTestInferenceService() InferenceService {
 	return inferenceservice
 }
 
+func TestValidAutoscalerClassTypeAndHPAMetrics(t *testing.T) {
+	g := gomega.NewGomegaWithT(t)
+	isvc := makeTestRawInferenceService()
+	g.Expect(isvc.ValidateCreate()).Should(gomega.Succeed())
+}
+
+func TestInvalidAutoscalerClassType(t *testing.T) {
+	g := gomega.NewGomegaWithT(t)
+	isvc := makeTestRawInferenceService()
+	isvc.ObjectMeta.Annotations["serving.kserve.io/autoscalerClass"] = "test"
+	g.Expect(isvc.ValidateCreate()).ShouldNot(gomega.Succeed())
+}
+
+func TestValidTargetUtilizationPercentage(t *testing.T) {
+	g := gomega.NewGomegaWithT(t)
+	isvc := makeTestRawInferenceService()
+	isvc.ObjectMeta.Annotations["serving.kserve.io/targetUtilizationPercentage"] = "70"
+	g.Expect(isvc.ValidateCreate()).Should(gomega.Succeed())
+}
+
+func TestInvalidTargetUtilizationPercentage(t *testing.T) {
+	g := gomega.NewGomegaWithT(t)
+	isvc := makeTestRawInferenceService()
+	isvc.ObjectMeta.Annotations["serving.kserve.io/targetUtilizationPercentage"] = "101"
+	g.Expect(isvc.ValidateCreate()).ShouldNot(gomega.Succeed())
+
+	isvc.ObjectMeta.Annotations["serving.kserve.io/targetUtilizationPercentage"] = "abc"
+	g.Expect(isvc.ValidateCreate()).ShouldNot(gomega.Succeed())
+
+	isvc.ObjectMeta.Annotations["serving.kserve.io/targetUtilizationPercentage"] = "0"
+	g.Expect(isvc.ValidateCreate()).ShouldNot(gomega.Succeed())
+
+	isvc.ObjectMeta.Annotations["serving.kserve.io/targetUtilizationPercentage"] = "99.9"
+	g.Expect(isvc.ValidateCreate()).ShouldNot(gomega.Succeed())
+}
+
+func TestInvalidAutoscalerHPAMetrics(t *testing.T) {
+	g := gomega.NewGomegaWithT(t)
+	isvc := makeTestRawInferenceService()
+	isvc.ObjectMeta.Annotations["serving.kserve.io/metrics"] = "test"
+	g.Expect(isvc.ValidateCreate()).ShouldNot(gomega.Succeed())
+}
+
 func TestValidStorageURIPrefixOK(t *testing.T) {
 	g := gomega.NewGomegaWithT(t)
 	for _, prefix := range SupportedStorageURIPrefixList {

pkg/constants/constants.go

@@ -68,6 +68,9 @@ var (
 	InferenceServiceGKEAcceleratorAnnotationKey = KServeAPIGroupName + "/gke-accelerator"
 	DeploymentMode                              = KServeAPIGroupName + "/deploymentMode"
 	EnableRoutingTagAnnotationKey               = KServeAPIGroupName + "/enable-tag-routing"
+	AutoscalerClass                             = KServeAPIGroupName + "/autoscalerClass"
+	AutoscalerMetrics                           = KServeAPIGroupName + "/metrics"
+	TargetUtilizationPercentage                 = KServeAPIGroupName + "/targetUtilizationPercentage"
 )
 
 // InferenceService Internal Annotations
@@ -98,6 +101,39 @@ var (
 	DefaultMinReplicas        int   = 1
 )
 
+type AutoscalerClassType string
+type AutoscalerMetricsType string
+
+// Autoscaler Default Class
+var (
+	DefaultAutoscalerClass = AutoscalerClassHPA
+)
+
+// Autoscaler Class
+var (
+	AutoscalerClassHPA AutoscalerClassType = "hpa"
+)
+
+// Autoscaler Metrics
+var (
+	AutoScalerMetricsCPU AutoscalerMetricsType = "cpu"
+)
+
+// Autoscaler Class Allowed List
+var AutoscalerAllowedClassList = []AutoscalerClassType{
+	AutoscalerClassHPA,
+}
+
+// Autoscaler Metrics Allowed List
+var AutoscalerAllowedMetricsList = []AutoscalerMetricsType{
+	AutoScalerMetricsCPU,
+}
+
+// Autoscaler Default Metrics Value
+var (
+	DefaultCPUUtilization int32 = 80
+)
+
 // Webhook Constants
 var (
 	EnableKServeMutatingWebhook            = "enabled"

pkg/controller/v1beta1/inferenceservice/components/explainer.go

@@ -96,8 +96,11 @@ func (e *Explainer) Reconcile(isvc *v1beta1.InferenceService) error {
 
 	// Here we allow switch between knative and vanilla deployment
 	if isvcutils.GetDeploymentMode(annotations, deployConfig) == constants.RawDeployment {
-		r := raw.NewRawKubeReconciler(e.client, e.scheme, objectMeta, &isvc.Spec.Explainer.ComponentExtensionSpec,
+		r, err := raw.NewRawKubeReconciler(e.client, e.scheme, objectMeta, &isvc.Spec.Explainer.ComponentExtensionSpec,
 			&podSpec)
+		if err != nil {
+			return errors.Wrapf(err, "fails to create NewRawKubeReconciler for explainer")
+		}
 		//set Deployment Controller
 		if err := controllerutil.SetControllerReference(isvc, r.Deployment.Deployment, e.scheme); err != nil {
 			return errors.Wrapf(err, "fails to set deployment owner reference for explainer")
@@ -106,6 +109,12 @@ func (e *Explainer) Reconcile(isvc *v1beta1.InferenceService) error {
 		if err := controllerutil.SetControllerReference(isvc, r.Service.Service, e.scheme); err != nil {
 			return errors.Wrapf(err, "fails to set service owner reference for explainer")
 		}
+		//set autoscaler Controller
+		if r.Scaler.Autoscaler.AutoscalerClass == constants.AutoscalerClassHPA {
+			if err := controllerutil.SetControllerReference(isvc, r.Scaler.Autoscaler.HPA.HPA, e.scheme); err != nil {
+				return errors.Wrapf(err, "fails to set HPA owner reference for explainer")
+			}
+		}
 
 		deployment, err := r.Reconcile()
 		if err != nil {

pkg/controller/v1beta1/inferenceservice/components/predictor.go

@@ -106,8 +106,11 @@ func (p *Predictor) Reconcile(isvc *v1beta1.InferenceService) error {
 
 	// Here we allow switch between knative and vanilla deployment
 	if isvcutils.GetDeploymentMode(annotations, deployConfig) == constants.RawDeployment {
-		r := raw.NewRawKubeReconciler(p.client, p.scheme, objectMeta, &isvc.Spec.Predictor.ComponentExtensionSpec,
+		r, err := raw.NewRawKubeReconciler(p.client, p.scheme, objectMeta, &isvc.Spec.Predictor.ComponentExtensionSpec,
 			&podSpec)
+		if err != nil {
+			return errors.Wrapf(err, "fails to create NewRawKubeReconciler for predictor")
+		}
 		//set Deployment Controller
 		if err := controllerutil.SetControllerReference(isvc, r.Deployment.Deployment, p.scheme); err != nil {
 			return errors.Wrapf(err, "fails to set deployment owner reference for predictor")
@@ -116,6 +119,12 @@ func (p *Predictor) Reconcile(isvc *v1beta1.InferenceService) error {
 		if err := controllerutil.SetControllerReference(isvc, r.Service.Service, p.scheme); err != nil {
 			return errors.Wrapf(err, "fails to set service owner reference for predictor")
 		}
+		//set autoscaler Controller
+		if r.Scaler.Autoscaler.AutoscalerClass == constants.AutoscalerClassHPA {
+			if err := controllerutil.SetControllerReference(isvc, r.Scaler.Autoscaler.HPA.HPA, p.scheme); err != nil {
+				return errors.Wrapf(err, "fails to set HPA owner reference for predictor")
+			}
+		}
 
 		deployment, err := r.Reconcile()
 		if err != nil {

pkg/controller/v1beta1/inferenceservice/components/transformer.go

@@ -98,8 +98,11 @@ func (p *Transformer) Reconcile(isvc *v1beta1.InferenceService) error {
 	}
 	// Here we allow switch between knative and vanilla deployment
 	if isvcutils.GetDeploymentMode(annotations, deployConfig) == constants.RawDeployment {
-		r := raw.NewRawKubeReconciler(p.client, p.scheme, objectMeta, &isvc.Spec.Transformer.ComponentExtensionSpec,
+		r, err := raw.NewRawKubeReconciler(p.client, p.scheme, objectMeta, &isvc.Spec.Transformer.ComponentExtensionSpec,
 			&podSpec)
+		if err != nil {
+			return errors.Wrapf(err, "fails to create NewRawKubeReconciler for transformer")
+		}
 		//set Deployment Controller
 		if err := controllerutil.SetControllerReference(isvc, r.Deployment.Deployment, p.scheme); err != nil {
 			return errors.Wrapf(err, "fails to set deployment owner reference for transformer")
@@ -108,6 +111,12 @@ func (p *Transformer) Reconcile(isvc *v1beta1.InferenceService) error {
 		if err := controllerutil.SetControllerReference(isvc, r.Service.Service, p.scheme); err != nil {
 			return errors.Wrapf(err, "fails to set service owner reference for transformer")
 		}
+		//set autoscaler Controller
+		if r.Scaler.Autoscaler.AutoscalerClass == constants.AutoscalerClassHPA {
+			if err := controllerutil.SetControllerReference(isvc, r.Scaler.Autoscaler.HPA.HPA, p.scheme); err != nil {
+				return errors.Wrapf(err, "fails to set HPA owner reference for transformer")
+			}
+		}
 
 		deployment, err := r.Reconcile()
 		if err != nil {

pkg/controller/v1beta1/inferenceservice/controller.go

@@ -53,6 +53,7 @@ import (
 // +kubebuilder:rbac:groups=serving.knative.dev,resources=services/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=networking.istio.io,resources=virtualservices,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=networking.istio.io,resources=virtualservices/finalizers,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=autoscaling,resources=horizontalpodautoscalers,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=core,resources=serviceaccounts,verbs=get;list;watch
 // +kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch;create