From 1a470e1314dbda2a82ded024443649d8f8dc2866 Mon Sep 17 00:00:00 2001 From: Jacob Salway Date: Sun, 29 Sep 2024 20:27:16 +1000 Subject: [PATCH] Upgrade to Spark 3.5.3 Signed-off-by: Jacob Salway --- Dockerfile | 2 +- README.md | 2 +- config/samples/v1beta1_sparkapplication.yaml | 4 ++-- config/samples/v1beta2_scheduledsparkapplication.yaml | 4 ++-- config/samples/v1beta2_sparkapplication.yaml | 4 ++-- examples/spark-pi-configmap.yaml | 4 ++-- examples/spark-pi-custom-resource.yaml | 4 ++-- examples/spark-pi-dynamic-allocation.yaml | 4 ++-- examples/spark-pi-kube-scheduler.yaml | 4 ++-- examples/spark-pi-prometheus.yaml | 4 ++-- examples/spark-pi-python.yaml | 2 +- examples/spark-pi-scheduled.yaml | 4 ++-- examples/spark-pi-ttl.yaml | 4 ++-- examples/spark-pi-volcano.yaml | 4 ++-- examples/spark-pi-yunikorn.yaml | 6 +++--- examples/spark-pi.yaml | 4 ++-- spark-docker/Dockerfile | 2 +- 17 files changed, 31 insertions(+), 31 deletions(-) diff --git a/Dockerfile b/Dockerfile index b372f4f7b..f66e2de72 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ # limitations under the License. # -ARG SPARK_IMAGE=spark:3.5.2 +ARG SPARK_IMAGE=spark:3.5.3 FROM golang:1.23.1 AS builder diff --git a/README.md b/README.md index 4feb67177..f249a0495 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ The following table lists the most recent few versions of the operator. | Operator Version | API Version | Kubernetes Version | Base Spark Version | | ------------- | ------------- | ------------- | ------------- | -| `v2.0.x` | `v1beta2` | 1.16+ | `3.5.2` | +| `v2.0.x` | `v1beta2` | 1.16+ | `3.5.3` | | `v1beta2-1.6.x-3.5.0` | `v1beta2` | 1.16+ | `3.5.0` | | `v1beta2-1.5.x-3.5.0` | `v1beta2` | 1.16+ | `3.5.0` | | `v1beta2-1.4.x-3.5.0` | `v1beta2` | 1.16+ | `3.5.0` | diff --git a/config/samples/v1beta1_sparkapplication.yaml b/config/samples/v1beta1_sparkapplication.yaml index a9b084bbd..25ff9d392 100644 --- a/config/samples/v1beta1_sparkapplication.yaml +++ b/config/samples/v1beta1_sparkapplication.yaml @@ -8,10 +8,10 @@ metadata: spec: type: Scala mode: cluster - image: spark:3.5.2 + image: spark:3.5.3 imagePullPolicy: IfNotPresent mainClass: org.apache.spark.examples.SparkPi - mainApplicationFile: local:///opt/spark/examples/jars/spark-examples_2.12-3.5.2.jar + mainApplicationFile: local:///opt/spark/examples/jars/spark-examples.jar sparkVersion: 3.5.2 driver: labels: diff --git a/config/samples/v1beta2_scheduledsparkapplication.yaml b/config/samples/v1beta2_scheduledsparkapplication.yaml index e6850801c..3f4786eb1 100644 --- a/config/samples/v1beta2_scheduledsparkapplication.yaml +++ b/config/samples/v1beta2_scheduledsparkapplication.yaml @@ -11,10 +11,10 @@ spec: template: type: Scala mode: cluster - image: spark:3.5.2 + image: spark:3.5.3 imagePullPolicy: IfNotPresent mainClass: org.apache.spark.examples.SparkPi - mainApplicationFile: local:///opt/spark/examples/jars/spark-examples_2.12-3.5.2.jar + mainApplicationFile: local:///opt/spark/examples/jars/spark-examples.jar sparkVersion: 3.5.2 restartPolicy: type: Never diff --git a/config/samples/v1beta2_sparkapplication.yaml b/config/samples/v1beta2_sparkapplication.yaml index f534a07e7..e664a6998 100644 --- a/config/samples/v1beta2_sparkapplication.yaml +++ b/config/samples/v1beta2_sparkapplication.yaml @@ -8,10 +8,10 @@ metadata: spec: type: Scala mode: cluster - image: spark:3.5.2 + image: spark:3.5.3 imagePullPolicy: IfNotPresent mainClass: org.apache.spark.examples.SparkPi - mainApplicationFile: local:///opt/spark/examples/jars/spark-examples_2.12-3.5.2.jar + mainApplicationFile: local:///opt/spark/examples/jars/spark-examples.jar sparkVersion: 3.5.2 driver: labels: diff --git a/examples/spark-pi-configmap.yaml b/examples/spark-pi-configmap.yaml index ddb812412..62cc9822d 100644 --- a/examples/spark-pi-configmap.yaml +++ b/examples/spark-pi-configmap.yaml @@ -21,10 +21,10 @@ metadata: spec: type: Scala mode: cluster - image: spark:3.5.2 + image: spark:3.5.3 imagePullPolicy: IfNotPresent mainClass: org.apache.spark.examples.SparkPi - mainApplicationFile: local:///opt/spark/examples/jars/spark-examples_2.12-3.5.2.jar + mainApplicationFile: local:///opt/spark/examples/jars/spark-examples.jar sparkVersion: 3.5.2 restartPolicy: type: Never diff --git a/examples/spark-pi-custom-resource.yaml b/examples/spark-pi-custom-resource.yaml index 1880cacfe..cd4c36d15 100644 --- a/examples/spark-pi-custom-resource.yaml +++ b/examples/spark-pi-custom-resource.yaml @@ -21,10 +21,10 @@ metadata: spec: type: Scala mode: cluster - image: spark:3.5.2 + image: spark:3.5.3 imagePullPolicy: IfNotPresent mainClass: org.apache.spark.examples.SparkPi - mainApplicationFile: local:///opt/spark/examples/jars/spark-examples_2.12-3.5.2.jar + mainApplicationFile: local:///opt/spark/examples/jars/spark-examples.jar sparkVersion: 3.5.2 restartPolicy: type: Never diff --git a/examples/spark-pi-dynamic-allocation.yaml b/examples/spark-pi-dynamic-allocation.yaml index c2480cb5a..e2b9dba03 100644 --- a/examples/spark-pi-dynamic-allocation.yaml +++ b/examples/spark-pi-dynamic-allocation.yaml @@ -21,10 +21,10 @@ metadata: spec: type: Scala mode: cluster - image: spark:3.5.2 + image: spark:3.5.3 imagePullPolicy: IfNotPresent mainClass: org.apache.spark.examples.SparkPi - mainApplicationFile: local:///opt/spark/examples/jars/spark-examples_2.12-3.5.2.jar + mainApplicationFile: local:///opt/spark/examples/jars/spark-examples.jar sparkVersion: 3.5.2 driver: cores: 1 diff --git a/examples/spark-pi-kube-scheduler.yaml b/examples/spark-pi-kube-scheduler.yaml index c30b6734e..f58d4b5ee 100644 --- a/examples/spark-pi-kube-scheduler.yaml +++ b/examples/spark-pi-kube-scheduler.yaml @@ -21,10 +21,10 @@ metadata: spec: type: Scala mode: cluster - image: spark:3.5.2 + image: spark:3.5.3 imagePullPolicy: IfNotPresent mainClass: org.apache.spark.examples.SparkPi - mainApplicationFile: local:///opt/spark/examples/jars/spark-examples_2.12-3.5.2.jar + mainApplicationFile: local:///opt/spark/examples/jars/spark-examples.jar sparkVersion: 3.5.2 driver: cores: 1 diff --git a/examples/spark-pi-prometheus.yaml b/examples/spark-pi-prometheus.yaml index 416cf527c..19847859d 100644 --- a/examples/spark-pi-prometheus.yaml +++ b/examples/spark-pi-prometheus.yaml @@ -22,10 +22,10 @@ metadata: spec: type: Scala mode: cluster - image: {IMAGE_REGISTRY}/{IMAGE_REPOSITORY}/spark:3.5.2-gcs-prometheus + image: {IMAGE_REGISTRY}/{IMAGE_REPOSITORY}/spark:3.5.3-gcs-prometheus imagePullPolicy: Always mainClass: org.apache.spark.examples.SparkPi - mainApplicationFile: local:///opt/spark/examples/jars/spark-examples_2.12-3.5.2.jar + mainApplicationFile: local:///opt/spark/examples/jars/spark-examples.jar arguments: - "100000" sparkVersion: 3.5.2 diff --git a/examples/spark-pi-python.yaml b/examples/spark-pi-python.yaml index 33b659eef..d9410fef4 100644 --- a/examples/spark-pi-python.yaml +++ b/examples/spark-pi-python.yaml @@ -22,7 +22,7 @@ spec: type: Python pythonVersion: "3" mode: cluster - image: spark:3.5.2 + image: spark:3.5.3 imagePullPolicy: IfNotPresent mainApplicationFile: local:///opt/spark/examples/src/main/python/pi.py sparkVersion: 3.5.2 diff --git a/examples/spark-pi-scheduled.yaml b/examples/spark-pi-scheduled.yaml index 3bcd053af..0a570f20a 100644 --- a/examples/spark-pi-scheduled.yaml +++ b/examples/spark-pi-scheduled.yaml @@ -25,10 +25,10 @@ spec: template: type: Scala mode: cluster - image: spark:3.5.2 + image: spark:3.5.3 imagePullPolicy: IfNotPresent mainClass: org.apache.spark.examples.SparkPi - mainApplicationFile: local:///opt/spark/examples/jars/spark-examples_2.12-3.5.2.jar + mainApplicationFile: local:///opt/spark/examples/jars/spark-examples.jar sparkVersion: 3.5.2 restartPolicy: type: Never diff --git a/examples/spark-pi-ttl.yaml b/examples/spark-pi-ttl.yaml index 68e6dd413..5c9f2f4d4 100644 --- a/examples/spark-pi-ttl.yaml +++ b/examples/spark-pi-ttl.yaml @@ -21,10 +21,10 @@ metadata: spec: type: Scala mode: cluster - image: spark:3.5.2 + image: spark:3.5.3 imagePullPolicy: IfNotPresent mainClass: org.apache.spark.examples.SparkPi - mainApplicationFile: local:///opt/spark/examples/jars/spark-examples_2.12-3.5.2.jar + mainApplicationFile: local:///opt/spark/examples/jars/spark-examples.jar sparkVersion: 3.5.2 timeToLiveSeconds: 30 driver: diff --git a/examples/spark-pi-volcano.yaml b/examples/spark-pi-volcano.yaml index 889d32c36..3ebe22288 100644 --- a/examples/spark-pi-volcano.yaml +++ b/examples/spark-pi-volcano.yaml @@ -21,10 +21,10 @@ metadata: spec: type: Scala mode: cluster - image: spark:3.5.2 + image: spark:3.5.3 imagePullPolicy: IfNotPresent mainClass: org.apache.spark.examples.SparkPi - mainApplicationFile: local:///opt/spark/examples/jars/spark-examples_2.12-3.5.2.jar + mainApplicationFile: local:///opt/spark/examples/jars/spark-examples.jar sparkVersion: 3.5.2 driver: cores: 1 diff --git a/examples/spark-pi-yunikorn.yaml b/examples/spark-pi-yunikorn.yaml index 0cffd6aaa..f1d0176ba 100644 --- a/examples/spark-pi-yunikorn.yaml +++ b/examples/spark-pi-yunikorn.yaml @@ -21,10 +21,10 @@ metadata: spec: type: Scala mode: cluster - image: spark:3.5.2 + image: spark:3.5.3 imagePullPolicy: IfNotPresent mainClass: org.apache.spark.examples.SparkPi - mainApplicationFile: local:///opt/spark/examples/jars/spark-examples_2.12-3.5.2.jar + mainApplicationFile: local:///opt/spark/examples/jars/spark-examples.jar sparkVersion: 3.5.2 driver: cores: 1 @@ -36,4 +36,4 @@ spec: memory: 512m batchScheduler: yunikorn batchSchedulerOptions: - queue: root.default \ No newline at end of file + queue: root.default diff --git a/examples/spark-pi.yaml b/examples/spark-pi.yaml index 034490e0c..b4179ee95 100644 --- a/examples/spark-pi.yaml +++ b/examples/spark-pi.yaml @@ -21,10 +21,10 @@ metadata: spec: type: Scala mode: cluster - image: spark:3.5.2 + image: spark:3.5.3 imagePullPolicy: IfNotPresent mainClass: org.apache.spark.examples.SparkPi - mainApplicationFile: local:///opt/spark/examples/jars/spark-examples_2.12-3.5.2.jar + mainApplicationFile: local:///opt/spark/examples/jars/spark-examples.jar arguments: - "5000" sparkVersion: 3.5.2 diff --git a/spark-docker/Dockerfile b/spark-docker/Dockerfile index 20ca69fbb..2a41a708a 100644 --- a/spark-docker/Dockerfile +++ b/spark-docker/Dockerfile @@ -14,7 +14,7 @@ # limitations under the License. # -ARG SPARK_IMAGE=spark:3.5.2 +ARG SPARK_IMAGE=spark:3.5.3 FROM ${SPARK_IMAGE} # Switch to user root so we can add additional jars and configuration files.