diff --git a/helm-chart/kuberay-apiserver/Chart.yaml b/helm-chart/kuberay-apiserver/Chart.yaml index 30e5993..5bea86f 100644 --- a/helm-chart/kuberay-apiserver/Chart.yaml +++ b/helm-chart/kuberay-apiserver/Chart.yaml @@ -15,4 +15,4 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.2.2 +version: 1.3.0-alpha.0 diff --git a/helm-chart/kuberay-apiserver/values.yaml b/helm-chart/kuberay-apiserver/values.yaml index 49f2ced..78c80ef 100644 --- a/helm-chart/kuberay-apiserver/values.yaml +++ b/helm-chart/kuberay-apiserver/values.yaml @@ -7,7 +7,7 @@ replicaCount: 1 name: "kuberay-apiserver" image: repository: quay.io/kuberay/apiserver - tag: v1.2.2 + tag: v1.3.0-alpha.0 pullPolicy: IfNotPresent ## Install Default RBAC roles and bindings diff --git a/helm-chart/kuberay-operator/Chart.yaml b/helm-chart/kuberay-operator/Chart.yaml index 583a8a7..853b4f8 100644 --- a/helm-chart/kuberay-operator/Chart.yaml +++ b/helm-chart/kuberay-operator/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 description: A Helm chart for Kubernetes name: kuberay-operator -version: 1.2.2 +version: 1.3.0-alpha.0 icon: https://github.com/ray-project/ray/raw/master/doc/source/images/ray_header_logo.png type: application diff --git a/helm-chart/kuberay-operator/crds/ray.io_rayclusters.yaml b/helm-chart/kuberay-operator/crds/ray.io_rayclusters.yaml index 8174d48..bd062f6 100644 --- a/helm-chart/kuberay-operator/crds/ray.io_rayclusters.yaml +++ b/helm-chart/kuberay-operator/crds/ray.io_rayclusters.yaml @@ -296,6 +296,135 @@ spec: type: object enableInTreeAutoscaling: type: boolean + gcsFaultToleranceOptions: + properties: + externalStorageNamespace: + type: string + redisAddress: + type: string + redisPassword: + properties: + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + redisUsername: + properties: + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + required: + - redisAddress + type: object headGroupSpec: properties: enableIngress: @@ -4107,6 +4236,14 @@ spec: additionalProperties: type: string type: object + managedBy: + type: string + x-kubernetes-validations: + - message: the managedBy field is immutable + rule: self == oldSelf + - message: the managedBy field value must be either 'ray.io/kuberay-operator' + or 'kueue.x-k8s.io/multikueue' + rule: self in ['ray.io/kuberay-operator', 'kueue.x-k8s.io/multikueue'] rayVersion: type: string suspend: @@ -4116,6 +4253,9 @@ spec: properties: groupName: type: string + idleTimeoutSeconds: + format: int32 + type: integer maxReplicas: default: 2147483647 format: int32 @@ -4143,6 +4283,8 @@ spec: type: string type: array type: object + suspend: + type: boolean template: properties: metadata: diff --git a/helm-chart/kuberay-operator/crds/ray.io_rayjobs.yaml b/helm-chart/kuberay-operator/crds/ray.io_rayjobs.yaml index f9b06f7..580b50c 100644 --- a/helm-chart/kuberay-operator/crds/ray.io_rayjobs.yaml +++ b/helm-chart/kuberay-operator/crds/ray.io_rayjobs.yaml @@ -58,6 +58,12 @@ spec: additionalProperties: type: string type: object + deletionPolicy: + type: string + x-kubernetes-validations: + - message: the deletionPolicy field value must be either 'DeleteCluster', + 'DeleteWorkers', 'DeleteSelf', or 'DeleteNone' + rule: self in ['DeleteCluster', 'DeleteWorkers', 'DeleteSelf', 'DeleteNone'] entrypoint: type: string entrypointNumCpus: @@ -68,6 +74,14 @@ spec: type: string jobId: type: string + managedBy: + type: string + x-kubernetes-validations: + - message: the managedBy field is immutable + rule: self == oldSelf + - message: the managedBy field value must be either 'ray.io/kuberay-operator' + or 'kueue.x-k8s.io/multikueue' + rule: self in ['ray.io/kuberay-operator', 'kueue.x-k8s.io/multikueue'] metadata: additionalProperties: type: string @@ -308,6 +322,135 @@ spec: type: object enableInTreeAutoscaling: type: boolean + gcsFaultToleranceOptions: + properties: + externalStorageNamespace: + type: string + redisAddress: + type: string + redisPassword: + properties: + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + redisUsername: + properties: + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + required: + - redisAddress + type: object headGroupSpec: properties: enableIngress: @@ -4119,6 +4262,14 @@ spec: additionalProperties: type: string type: object + managedBy: + type: string + x-kubernetes-validations: + - message: the managedBy field is immutable + rule: self == oldSelf + - message: the managedBy field value must be either 'ray.io/kuberay-operator' + or 'kueue.x-k8s.io/multikueue' + rule: self in ['ray.io/kuberay-operator', 'kueue.x-k8s.io/multikueue'] rayVersion: type: string suspend: @@ -4128,6 +4279,9 @@ spec: properties: groupName: type: string + idleTimeoutSeconds: + format: int32 + type: integer maxReplicas: default: 2147483647 format: int32 @@ -4155,6 +4309,8 @@ spec: type: string type: array type: object + suspend: + type: boolean template: properties: metadata: diff --git a/helm-chart/kuberay-operator/crds/ray.io_rayservices.yaml b/helm-chart/kuberay-operator/crds/ray.io_rayservices.yaml index 4a45452..9d0fd96 100644 --- a/helm-chart/kuberay-operator/crds/ray.io_rayservices.yaml +++ b/helm-chart/kuberay-operator/crds/ray.io_rayservices.yaml @@ -38,6 +38,8 @@ spec: deploymentUnhealthySecondThreshold: format: int32 type: integer + excludeHeadPodFromServeSvc: + type: boolean rayClusterConfig: properties: autoscalerOptions: @@ -274,6 +276,135 @@ spec: type: object enableInTreeAutoscaling: type: boolean + gcsFaultToleranceOptions: + properties: + externalStorageNamespace: + type: string + redisAddress: + type: string + redisPassword: + properties: + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + redisUsername: + properties: + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + type: object + required: + - redisAddress + type: object headGroupSpec: properties: enableIngress: @@ -4085,6 +4216,14 @@ spec: additionalProperties: type: string type: object + managedBy: + type: string + x-kubernetes-validations: + - message: the managedBy field is immutable + rule: self == oldSelf + - message: the managedBy field value must be either 'ray.io/kuberay-operator' + or 'kueue.x-k8s.io/multikueue' + rule: self in ['ray.io/kuberay-operator', 'kueue.x-k8s.io/multikueue'] rayVersion: type: string suspend: @@ -4094,6 +4233,9 @@ spec: properties: groupName: type: string + idleTimeoutSeconds: + format: int32 + type: integer maxReplicas: default: 2147483647 format: int32 @@ -4121,6 +4263,8 @@ spec: type: string type: array type: object + suspend: + type: boolean template: properties: metadata: @@ -7930,6 +8074,11 @@ spec: serviceUnhealthySecondThreshold: format: int32 type: integer + upgradeStrategy: + properties: + type: + type: string + type: object type: object status: properties: diff --git a/helm-chart/kuberay-operator/templates/_helpers.tpl b/helm-chart/kuberay-operator/templates/_helpers.tpl index beaea3d..05b4c55 100644 --- a/helm-chart/kuberay-operator/templates/_helpers.tpl +++ b/helm-chart/kuberay-operator/templates/_helpers.tpl @@ -332,7 +332,7 @@ rules: - patch - update - watch -{{- if .batchSchedulerEnabled }} +{{- if or .batchSchedulerEnabled (eq .batchSchedulerName "volcano") }} - apiGroups: - scheduling.volcano.sh resources: diff --git a/helm-chart/kuberay-operator/templates/deployment.yaml b/helm-chart/kuberay-operator/templates/deployment.yaml index a9cd9b5..9c15f90 100644 --- a/helm-chart/kuberay-operator/templates/deployment.yaml +++ b/helm-chart/kuberay-operator/templates/deployment.yaml @@ -37,7 +37,12 @@ spec: {{- if and (.Values.logging.baseDir) (.Values.logging.fileName) }} volumes: - name: kuberay-logs + {{- if .Values.logging.sizeLimit }} + emptyDir: + sizeLimit: {{ .Values.logging.sizeLimit }} + {{- else }} emptyDir: {} + {{- end }} {{- end }} securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} @@ -53,7 +58,7 @@ spec: mountPath: "{{ .Values.logging.baseDir }}" {{- end }} command: - - /manager + - {{ .Values.operatorComand }} args: {{- $argList := list -}} {{- $argList = append $argList (include "kuberay.featureGates" . | trim) -}} diff --git a/helm-chart/kuberay-operator/templates/leader_election_role.yaml b/helm-chart/kuberay-operator/templates/leader_election_role.yaml index 9049e4d..134fdeb 100644 --- a/helm-chart/kuberay-operator/templates/leader_election_role.yaml +++ b/helm-chart/kuberay-operator/templates/leader_election_role.yaml @@ -17,14 +17,6 @@ rules: - update - patch - delete -- apiGroups: - - "" - resources: - - configmaps/status - verbs: - - get - - update - - patch - apiGroups: - "" resources: diff --git a/helm-chart/kuberay-operator/templates/role.yaml b/helm-chart/kuberay-operator/templates/role.yaml index 476be68..7ed91e2 100644 --- a/helm-chart/kuberay-operator/templates/role.yaml +++ b/helm-chart/kuberay-operator/templates/role.yaml @@ -6,5 +6,5 @@ metadata: labels: {{ include "kuberay-operator.labels" . | indent 4 }} name: {{ include "kuberay-operator.fullname" . }} -{{ include "role.consistentRules" (dict "batchSchedulerEnabled" .Values.batchScheduler.enabled) }} +{{ include "role.consistentRules" (dict "batchSchedulerEnabled" .Values.batchScheduler.enabled "batchSchedulerName" .Values.batchScheduler.name) }} {{- end }} diff --git a/helm-chart/kuberay-operator/values.yaml b/helm-chart/kuberay-operator/values.yaml index 6cde331..4ecc972 100644 --- a/helm-chart/kuberay-operator/values.yaml +++ b/helm-chart/kuberay-operator/values.yaml @@ -4,7 +4,7 @@ image: repository: quay.io/kuberay/operator - tag: v1.2.2 + tag: v1.3.0-alpha.0 pullPolicy: IfNotPresent nameOverride: "kuberay-operator" @@ -44,6 +44,8 @@ logging: baseDir: "" # File name for kuberay-operator log file fileName: "" + # EmptyDir volume size limit for kuberay-operator log file + sizeLimit: "" livenessProbe: initialDelaySeconds: 10 @@ -61,6 +63,8 @@ readinessProbe: # * "name" is the standard option, expecting a scheduler name, supported values are # "default", "volcano", and "yunikorn". # +# Note: "enabled" and "name" should not be set at the same time. If both are set, an error will be thrown. +# # Examples: # 1. Use volcano (deprecated) # batchScheduler: @@ -84,8 +88,12 @@ batchScheduler: featureGates: - name: RayClusterStatusConditions + enabled: true + - name: RayJobDeletionPolicy enabled: false +# Path to the operator binary +operatorComand: /manager # Set up `securityContext` to improve Pod security. # See https://github.com/ray-project/kuberay/blob/master/docs/guidance/pod-security.md for further guidance. diff --git a/helm-chart/ray-cluster/Chart.yaml b/helm-chart/ray-cluster/Chart.yaml index 5e5b3a3..45d19d9 100644 --- a/helm-chart/ray-cluster/Chart.yaml +++ b/helm-chart/ray-cluster/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v1 description: A Helm chart for Kubernetes name: ray-cluster -version: 1.2.2 +version: 1.3.0-alpha.0 icon: https://github.com/ray-project/ray/raw/master/doc/source/images/ray_header_logo.png diff --git a/helm-chart/ray-cluster/templates/raycluster-cluster.yaml b/helm-chart/ray-cluster/templates/raycluster-cluster.yaml index b04fdbc..61bb04e 100644 --- a/helm-chart/ray-cluster/templates/raycluster-cluster.yaml +++ b/helm-chart/ray-cluster/templates/raycluster-cluster.yaml @@ -38,6 +38,9 @@ spec: {{- end }} template: spec: + {{- if .Values.head.dnsConfig }} + dnsConfig: {{- toYaml .Values.head.dnsConfig | nindent 10 }} + {{- end }} imagePullSecrets: {{- toYaml .Values.imagePullSecrets | nindent 10 }} {{- if .Values.head.serviceAccountName }} serviceAccountName: {{ .Values.head.serviceAccountName }} @@ -89,6 +92,9 @@ spec: {{ if .Values.head.volumes }} volumes: {{- toYaml .Values.head.volumes | nindent 10 }} {{- end }} + {{- if .Values.head.topologySpreadConstraints.enabled }} + topologySpreadConstraints: {{- toYaml .Values.head.topologySpreadConstraints | nindent 10 }} + {{- end }} affinity: {{- toYaml .Values.head.affinity | nindent 10 }} {{ if .Values.head.priorityClassName }} priorityClassName: {{- toYaml .Values.head.priorityClassName | nindent 10 }} @@ -135,6 +141,9 @@ spec: groupName: {{ $groupName }} template: spec: + {{- if $values.dnsConfig }} + dnsConfig: {{- toYaml $values.dnsConfig | nindent 10 }} + {{- end }} imagePullSecrets: {{- toYaml $.Values.imagePullSecrets | nindent 10 }} {{- if $values.serviceAccountName }} serviceAccountName: {{ $values.serviceAccountName }} @@ -160,7 +169,7 @@ spec: resources: {{- toYaml $values.resources | nindent 14 }} securityContext: {{- toYaml $values.securityContext | nindent 14 }} - {{- with concat $.Values.common.containerEnv $values.containerEnv }} + {{- with concat $.Values.common.containerEnv ($values.containerEnv | default list) }} env: {{- toYaml . | nindent 14 }} {{- end }} @@ -186,6 +195,9 @@ spec: {{ if $values.volumes }} volumes: {{- toYaml $values.volumes | nindent 10 }} {{- end }} + {{- if $values.topologySpreadConstraints }} + topologySpreadConstraints: {{- toYaml $values.topologySpreadConstraints | nindent 10 }} + {{- end }} affinity: {{- toYaml $values.affinity | nindent 10 }} {{ if $values.priorityClassName }} priorityClassName: {{- toYaml $values.priorityClassName | nindent 10 }} @@ -232,6 +244,9 @@ spec: groupName: {{ .Values.worker.groupName }} template: spec: + {{- if .Values.worker.dnsConfig }} + dnsConfig: {{- toYaml .Values.worker.dnsConfig | nindent 10 }} + {{- end }} imagePullSecrets: {{- toYaml .Values.imagePullSecrets | nindent 10 }} {{- if .Values.worker.serviceAccountName }} serviceAccountName: {{ .Values.worker.serviceAccountName }} diff --git a/helm-chart/ray-cluster/values.yaml b/helm-chart/ray-cluster/values.yaml index ea430cf..5755ddc 100644 --- a/helm-chart/ray-cluster/values.yaml +++ b/helm-chart/ray-cluster/values.yaml @@ -116,6 +116,19 @@ head: # annotations: # prometheus.io/scrape: "true" + # Custom pod DNS configuration + # See https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-dns-config + # dnsConfig: + # nameservers: + # - 8.8.8.8 + # searches: + # - example.local + # options: + # - name: ndots + # value: "2" + # - name: edns0 + topologySpreadConstraints: {} + worker: # If you want to disable the default workergroup @@ -178,6 +191,20 @@ worker: # container command for worker Pod. command: [] args: [] + topologySpreadConstraints: {} + + + # Custom pod DNS configuration + # See https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-dns-config + # dnsConfig: + # nameservers: + # - 8.8.8.8 + # searches: + # - example.local + # options: + # - name: ndots + # value: "2" + # - name: edns0 # The map's key is used as the groupName. # For example, key:small-group in the map below @@ -241,6 +268,22 @@ additionalWorkerGroups: command: [] args: [] + # Topology Spread Constraints for worker pods + # See: https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/ + topologySpreadConstraints: {} + + # Custom pod DNS configuration + # See https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-dns-config + # dnsConfig: + # nameservers: + # - 8.8.8.8 + # searches: + # - example.local + # options: + # - name: ndots + # value: "2" + # - name: edns0 + # Configuration for Head's Kubernetes Service service: # This is optional, and the default is ClusterIP.