Merge branch 'kserve/master' into 'odh/master'

.github/workflows/e2e-test.yml

@@ -788,11 +788,25 @@ jobs:
           kubectl get pods -n kserve
           kubectl describe pods -n kserve
 
+      - name: Log the config map
+        run: |
+          kubectl describe configmaps -n kserve inferenceservice-config
+
       - name: Run E2E tests
         timeout-minutes: 30
         run: |
           ./test/scripts/gh-actions/run-e2e-tests.sh "raw" "6"
 
+      - name: Patch inferenceservice config for cluster ip none
+        run: |
+          kubectl patch configmaps -n kserve inferenceservice-config --patch-file config/overlays/test/configmap/inferenceservice-enable-cluster-ip.yaml
+          kubectl describe configmaps -n kserve inferenceservice-config
+
+      - name: Run E2E tests - cluster ip none
+        timeout-minutes: 30
+        run: |
+          ./test/scripts/gh-actions/run-e2e-tests.sh "rawcipn" "1"
+
       - name: Check system status
         if: always()
         run: |

.github/workflows/verify-codegen.yml

@@ -58,6 +58,6 @@ jobs:
               for x in $(git diff-index --name-only HEAD -- ./pkg ./python ./charts); do
                   echo "::error file=$x::Please run make generate.%0A$(git diff $x | urlencode)"
               done
-              echo "${{ github.repository }} is out of date. Please run make generate"
+              echo "${{ github.repository }} is out of date. Please run make generate | manifest"
               exit 1
           fi

charts/kserve-resources/README.md

@@ -37,8 +37,10 @@ $ helm install kserve oci://ghcr.io/kserve/charts/kserve --version v0.14.0
 | kserve.controller.imagePullSecrets | list | `[]` | Reference to one or more secrets to be used when pulling images. For more information, see [Pull an Image from a Private Registry](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/).  For example:  imagePullSecrets:    - name: "image-pull-secret" |
 | kserve.controller.knativeAddressableResolver | object | `{"enabled":false}` | Indicates whether to create an addressable resolver ClusterRole for Knative Eventing. This ClusterRole grants the necessary permissions for the Knative's DomainMapping reconciler to resolve InferenceService addressables. |
 | kserve.controller.labels | object | `{}` | Optional additional labels to add to the controller deployment. |
+| kserve.controller.metricsBindAddress | string | `"127.0.0.1"` | Metrics bind address |
+| kserve.controller.metricsBindPort | string | `"8080"` | Metrics bind port |
 | kserve.controller.nodeSelector | object | `{}` | The nodeSelector on Pods tells Kubernetes to schedule Pods on the nodes with matching labels. For more information, see [Assigning Pods to Nodes](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/).  |
-| kserve.controller.podAnnotations | object | `{}` | Optional additional labels to add to the controller Pods. |
+| kserve.controller.podAnnotations | object | `{}` | Optional additional annotations to add to the controller Pods. |
 | kserve.controller.podLabels | object | `{}` | Optional additional labels to add to the controller Pods. |
 | kserve.controller.rbacProxy.resources.limits.cpu | string | `"100m"` |  |
 | kserve.controller.rbacProxy.resources.limits.memory | string | `"300Mi"` |  |
@@ -52,18 +54,22 @@ $ helm install kserve oci://ghcr.io/kserve/charts/kserve --version v0.14.0
 | kserve.controller.rbacProxyImage | string | `"quay.io/brancz/kube-rbac-proxy:v0.18.0"` | KServe controller manager rbac proxy contrainer image |
 | kserve.controller.resources | object | `{"limits":{"cpu":"100m","memory":"300Mi"},"requests":{"cpu":"100m","memory":"300Mi"}}` | Resources to provide to the kserve controller pod.  For example:  requests:    cpu: 10m    memory: 32Mi  For more information, see [Resource Management for Pods and Containers](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/). |
 | kserve.controller.securityContext | object | `{"runAsNonRoot":true}` | Pod Security Context. For more information, see [Configure a Security Context for a Pod or Container](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/). |
+| kserve.controller.serviceAnnotations | object | `{}` | Optional additional annotations to add to the controller service. |
 | kserve.controller.tag | string | `"v0.14.0"` | KServe controller contrainer image tag. |
 | kserve.controller.tolerations | list | `[]` | A list of Kubernetes Tolerations, if required. For more information, see [Toleration v1 core](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#toleration-v1-core).  For example:   tolerations:   - key: foo.bar.com/role     operator: Equal     value: master     effect: NoSchedule |
 | kserve.controller.topologySpreadConstraints | list | `[]` | A list of Kubernetes TopologySpreadConstraints, if required. For more information, see [Topology spread constraint v1 core](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#topologyspreadconstraint-v1-core  For example:   topologySpreadConstraints:   - maxSkew: 2     topologyKey: topology.kubernetes.io/zone     whenUnsatisfiable: ScheduleAnyway     labelSelector:       matchLabels:         app.kubernetes.io/instance: kserve-controller-manager         app.kubernetes.io/component: controller |
+| kserve.controller.webhookServiceAnnotations | object | `{}` | Optional additional annotations to add to the webhook service. |
 | kserve.localmodel.agent.hostPath | string | `"/mnt/models"` |  |
 | kserve.localmodel.agent.image | string | `"kserve/kserve-localmodelnode-agent"` |  |
 | kserve.localmodel.agent.nodeSelector | object | `{}` |  |
+| kserve.localmodel.agent.reconcilationFrequencyInSecs | int | `60` |  |
 | kserve.localmodel.agent.tag | string | `"v0.14.0"` |  |
 | kserve.localmodel.controller.image | string | `"kserve/kserve-localmodel-controller"` |  |
 | kserve.localmodel.controller.tag | string | `"v0.14.0"` |  |
 | kserve.localmodel.enabled | bool | `false` |  |
 | kserve.localmodel.jobNamespace | string | `"kserve-localmodel-jobs"` |  |
-| kserve.localmodel.securityContext.FSGroup | int | `1000` |  |
+| kserve.localmodel.jobTTLSecondsAfterFinished | int | `3600` |  |
+| kserve.localmodel.securityContext.fsGroup | int | `1000` |  |
 | kserve.metricsaggregator.enableMetricAggregation | string | `"false"` | configures metric aggregation annotation. This adds the annotation serving.kserve.io/enable-metric-aggregation to every service with the specified boolean value. If true enables metric aggregation in queue-proxy by setting env vars in the queue proxy container to configure scraping ports. |
 | kserve.metricsaggregator.enablePrometheusScraping | string | `"false"` | If true, prometheus annotations are added to the pod to scrape the metrics. If serving.kserve.io/enable-metric-aggregation is false, the prometheus port is set with the default prometheus scraping port 9090, otherwise the prometheus port annotation is set with the metric aggregation port. |
 | kserve.modelmesh.config.modelmeshImage | string | `"kserve/modelmesh"` |  |
@@ -88,6 +94,7 @@ $ helm install kserve oci://ghcr.io/kserve/charts/kserve --version v0.14.0
 | kserve.router.image | string | `"kserve/router"` |  |
 | kserve.router.tag | string | `"v0.14.0"` |  |
 | kserve.security.autoMountServiceAccountToken | bool | `true` |  |
+| kserve.service.serviceClusterIPNone | bool | `false` |  |
 | kserve.servingruntime.art.defaultVersion | string | `"v0.14.0"` |  |
 | kserve.servingruntime.art.image | string | `"kserve/art-explainer"` |  |
 | kserve.servingruntime.art.imagePullSecrets | list | `[]` |  |

charts/kserve-resources/templates/configmap.yaml

@@ -420,6 +420,19 @@ data:
          "defaultDeploymentMode": "Serverless"
        }
 
+     # ====================================== SERVICE CONFIGURATION ======================================
+     # Example
+     service: |-
+       {
+         "serviceClusterIPNone": "false"
+       }
+     service: |-
+       {
+          # ServiceClusterIPNone is a flag to indicate if the service should have a clusterIP set to None.
+          # If the DeploymentMode is Raw, the default value for ServiceClusterIPNone if not set is false
+          # "serviceClusterIPNone": "false"
+       }
+
      # ====================================== METRICS CONFIGURATION ======================================
      # Example
      metricsAggregator: |-
@@ -451,7 +464,12 @@ data:
          # defaultJobImage specifies the default image used for the download job.
          "defaultJobImage" : "kserve/storage-initializer:latest",
          # Kubernetes modifies the filesystem group ID on the attached volume.
-         "FSGroup": 1000
+         "fsGroup": 1000,
+         # TTL for the download job after it is finished.
+         "jobTTLSecondsAfterFinished": 3600,
+         # The frequency at which the local model agent reconciles the local models
+         # This is to detect if models are missing from local disk
+         "reconcilationFrequencyInSecs": {{ .Values.kserve.localmodel.agent.reconcilationFrequencyInSecs }}
        }
 
   agent: |-
@@ -503,6 +521,10 @@ data:
     {
       "defaultDeploymentMode": "{{ .Values.kserve.controller.deploymentMode }}"
     }
+  service: |-
+    {
+      "serviceClusterIPNone": "{{ .Values.kserve.service.serviceClusterIPNone }}"
+    }
   explainers: |-
     {
         "art": {
@@ -560,8 +582,10 @@ data:
     {
       "enabled": {{ .Values.kserve.localmodel.enabled }},
       "jobNamespace": "{{ .Values.kserve.localmodel.jobNamespace }}",
-      "defaultJobImage" : "kserve/storage-initializer:latest",
-      "FSGroup": {{ .Values.kserve.localmodel.securityContext.FSGroup }}
+      "jobTTLSecondsAfterFinished": {{ .Values.kserve.localmodel.jobTTLSecondsAfterFinished }},
+      "defaultJobImage": "kserve/storage-initializer:latest",
+      "fsGroup": {{ .Values.kserve.localmodel.securityContext.fsGroup }},
+      "reconcilationFrequencyInSecs": {{ .Values.kserve.localmodel.agent.reconcilationFrequencyInSecs }}
     }
   security: |-
     {

charts/kserve-resources/templates/deployment.yaml

@@ -86,7 +86,7 @@ spec:
           {{- toYaml . | nindent 10 }}
           {{- end }}
         args:
-        - "--metrics-addr=127.0.0.1:8080"
+        - "--metrics-addr={{ .Values.kserve.controller.metricsBindAddress }}:{{ .Values.kserve.controller.metricsBindPort }}"
         - "--leader-elect"
         env:
           - name: POD_NAMESPACE

charts/kserve-resources/templates/service.yaml

@@ -4,6 +4,9 @@ kind: Service
 metadata:
   name: kserve-webhook-server-service
   namespace: {{ .Release.Namespace }}
+  {{- with .Values.kserve.controller.webhookServiceAnnotations }}
+  annotations: {{ toYaml . | nindent 4 }}
+  {{- end }}
 spec:
   ports:
     - port: 443
@@ -20,6 +23,9 @@ metadata:
   labels:
     control-plane: kserve-controller-manager
     controller-tools.k8s.io: "1.0"
+  {{- with .Values.kserve.controller.serviceAnnotations }}
+  annotations: {{ toYaml . | nindent 4 }}
+  {{- end }}
 spec:
   selector:
     control-plane: kserve-controller-manager

charts/kserve-resources/values.yaml

@@ -7,6 +7,8 @@ kserve:
   router:
     image: kserve/router
     tag: *defaultVersion
+  service:
+    serviceClusterIPNone: false
   storage:
     image: kserve/storage-initializer
     tag: *defaultVersion
@@ -111,9 +113,15 @@ kserve:
     # -- Optional additional annotations to add to the controller deployment.
     annotations: {}
 
-    # -- Optional additional labels to add to the controller Pods.
+    # -- Optional additional annotations to add to the controller Pods.
     podAnnotations: {}
 
+    # -- Optional additional annotations to add to the controller service.
+    serviceAnnotations: {}
+
+    # -- Optional additional annotations to add to the webhook service.
+    webhookServiceAnnotations: {}
+
     # -- Pod Security Context.
     # For more information, see [Configure a Security Context for a Pod or Container](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/).
     securityContext:
@@ -130,6 +138,12 @@ kserve:
       readOnlyRootFilesystem: true
       runAsNonRoot: true
 
+    # -- Metrics bind address
+    metricsBindAddress: "127.0.0.1"
+
+    # -- Metrics bind port
+    metricsBindPort: "8080"
+
     gateway:
       # -- Ingress domain for RawDeployment mode, for Serverless it is configured in Knative.
       domain: example.com
@@ -401,12 +415,14 @@ kserve:
         image: kserve/kserve-localmodel-controller
         tag: *defaultVersion
       jobNamespace: kserve-localmodel-jobs
+      jobTTLSecondsAfterFinished: 3600
       securityContext:
-        FSGroup: 1000
+        fsGroup: 1000
       agent:
         nodeSelector: {}
         hostPath: /mnt/models
         image: kserve/kserve-localmodelnode-agent
         tag: *defaultVersion
+        reconcilationFrequencyInSecs: 60
   security:
     autoMountServiceAccountToken: true

config/configmap/inferenceservice.yaml

@@ -430,7 +430,20 @@ data:
          # ModelMesh https://kserve.github.io/website/master/admin/modelmesh/
          "defaultDeploymentMode": "Serverless"
        }
-     
+    
+     # ====================================== SERVICE CONFIGURATION ======================================
+     # Example
+     service: |-
+       {
+         "serviceClusterIPNone":  false
+       }
+     service: |-
+       {
+          # ServiceClusterIPNone is a boolean flag to indicate if the service should have a clusterIP set to None.
+          # If the DeploymentMode is Raw, the default value for ServiceClusterIPNone if not set is false
+          # "serviceClusterIPNone":  false
+       }
+
      # ====================================== METRICS CONFIGURATION ======================================
      # Example
      metricsAggregator: |-
@@ -462,7 +475,12 @@ data:
          # defaultJobImage specifies the default image used for the download job.
          "defaultJobImage" : "kserve/storage-initializer:latest",
          # Kubernetes modifies the filesystem group ID on the attached volume.
-         "FSGroup": 1000
+         "fsGroup": 1000,
+         # TTL for the download job after it is finished.
+         "jobTTLSecondsAfterFinished": 3600,
+         # The frequency at which the local model agent reconciles the local models
+         # This is to detect if models are missing from local disk
+         "reconcilationFrequencyInSecs": 60
        }
 
   explainers: |-
@@ -577,7 +595,7 @@ data:
       "enabled": false,
       "jobNamespace": "kserve-localmodel-jobs",
       "defaultJobImage" : "kserve/storage-initializer:latest",
-      "FSGroup": 1000
+      "fsGroup": 1000
     }
   
   security: |-

config/crd/full/serving.kserve.io_localmodelcaches.yaml

@@ -32,16 +32,20 @@ spec:
                 - type: string
                 pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
                 x-kubernetes-int-or-string: true
-              nodeGroup:
-                type: string
+              nodeGroups:
+                items:
+                  type: string
+                maxItems: 1
+                minItems: 1
+                type: array
               sourceModelUri:
                 type: string
                 x-kubernetes-validations:
                 - message: StorageUri is immutable
                   rule: self == oldSelf
             required:
             - modelSize
-            - nodeGroup
+            - nodeGroups
             - sourceModelUri
             type: object
           status:

config/overlays/test/configmap/inferenceservice-enable-cluster-ip.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: inferenceservice-config
+  namespace: kserve
+data:
+  service: |-
+    {
+        "serviceClusterIPNone": true
+    }

config/overlays/test/configmap/inferenceservice.yaml

@@ -74,4 +74,8 @@ data:
         "memoryLimit": "500Mi",
         "cpuRequest": "100m",
         "cpuLimit": "100m"
+    }
+  service: |-
+    {
+        "serviceClusterIPNone": false
     }

hack/update-codegen.sh

@@ -22,6 +22,9 @@ SCRIPT_DIR="$(dirname "${BASH_SOURCE[0]}")"
 SCRIPT_ROOT="${SCRIPT_DIR}/.."
 CODEGEN_VERSION=$(cd "${SCRIPT_ROOT}" && grep 'k8s.io/code-generator' go.mod | awk '{print $2}')
 
+# For debugging purposes
+echo "Codegen version ${CODEGEN_VERSION}"
+
 if [ -z "${GOPATH:-}" ]; then
     GOPATH=$(go env GOPATH)
     export GOPATH

hack/violation_exceptions.list

@@ -1,6 +1,7 @@
 API rule violation: list_type_missing,github.com/kserve/kserve/pkg/apis/serving/v1alpha1,BuiltInAdapter,Env
 API rule violation: list_type_missing,github.com/kserve/kserve/pkg/apis/serving/v1alpha1,InferenceGraphList,Items
 API rule violation: list_type_missing,github.com/kserve/kserve/pkg/apis/serving/v1alpha1,InferenceRouter,Steps
+API rule violation: list_type_missing,github.com/kserve/kserve/pkg/apis/serving/v1alpha1,LocalModelCacheSpec,NodeGroups
 API rule violation: list_type_missing,github.com/kserve/kserve/pkg/apis/serving/v1alpha1,LocalModelNodeSpec,LocalModels
 API rule violation: list_type_missing,github.com/kserve/kserve/pkg/apis/serving/v1alpha1,ServingRuntimePodSpec,Containers
 API rule violation: list_type_missing,github.com/kserve/kserve/pkg/apis/serving/v1alpha1,ServingRuntimePodSpec,ImagePullSecrets

pkg/agent/storage/https.go

@@ -191,7 +191,7 @@ func extractZipFiles(reader io.Reader, dest string) error {
 			return fmt.Errorf("unable to open file: %w", err)
 		}
 
-		_, err = io.CopyN(file, rc, DEFAULT_MAX_DECOMPRESSION_SIZE) // gosec G110
+		_, ioErr := io.CopyN(file, rc, DEFAULT_MAX_DECOMPRESSION_SIZE) // gosec G110
 		closeErr := file.Close()
 		if closeErr != nil {
 			return closeErr
@@ -200,7 +200,7 @@ func extractZipFiles(reader io.Reader, dest string) error {
 		if closeErr != nil {
 			return closeErr
 		}
-		if err != nil {
+		if ioErr != nil && !errors.Is(ioErr, io.EOF) {
 			return fmt.Errorf("unable to copy file content: %w", err)
 		}
 	}
@@ -246,7 +246,8 @@ func extractTarFiles(reader io.Reader, dest string) error {
 		}
 
 		// gosec G110
-		if _, err := io.CopyN(newFile, tr, DEFAULT_MAX_DECOMPRESSION_SIZE); err != nil {
+		_, ioErr := io.CopyN(newFile, tr, DEFAULT_MAX_DECOMPRESSION_SIZE)
+		if ioErr != nil && !errors.Is(ioErr, io.EOF) {
 			return fmt.Errorf("unable to copy contents to %s: %w", header.Name, err)
 		}
 	}

pkg/apis/serving/v1alpha1/doc.go

@@ -15,7 +15,7 @@ limitations under the License.
 */
 
 // +groupName=serving.kserve.io
-// +k8s:deepcopy-gen=package
+// +kubebuilder:object:generate=true
 
 // Package v1alpha1 contains API Schema definitions for the serving v1alpha1 API group
 package v1alpha1

pkg/apis/serving/v1alpha1/inference_graph.go

@@ -25,7 +25,6 @@ import (
 
 // InferenceGraph is the Schema for the InferenceGraph API for multiple models
 // +k8s:openapi-gen=true
-// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 // +genclient
 // +kubebuilder:object:root=true
 // +kubebuilder:subresource:status
@@ -310,7 +309,6 @@ type InferenceGraphStatus struct {
 // InferenceGraphList contains a list of InferenceGraph
 // +k8s:openapi-gen=true
 // +kubebuilder:object:root=true
-// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 type InferenceGraphList struct {
 	metav1.TypeMeta `json:",inline"`
 	metav1.ListMeta `json:"metadata,omitempty"`

pkg/apis/serving/v1alpha1/inference_graph_validation.go

@@ -58,12 +58,11 @@ var (
 )
 
 // +kubebuilder:object:generate=false
-// +k8s:deepcopy-gen=false
 // +k8s:openapi-gen=false
 // InferenceGraphValidator is responsible for setting default values on the InferenceGraph resources
 // when created or updated.
 //
-// NOTE: The +kubebuilder:object:generate=false and +k8s:deepcopy-gen=false marker prevents controller-gen from generating DeepCopy methods,
+// NOTE: The +kubebuilder:object:generate=false marker prevents controller-gen from generating DeepCopy methods,
 // as it is used only for temporary operations and does not need to be deeply copied.
 type InferenceGraphValidator struct{}