From d5a4fdd3d50e7644f35874e737298848045eaba5 Mon Sep 17 00:00:00 2001 From: Yikun Jiang Date: Thu, 7 Apr 2022 16:54:06 +0800 Subject: [PATCH 1/2] Add resource limited k8s it --- .github/workflows/k8s_integration_test.yml | 133 ++++++++++++++++++ .../k8s/integrationtest/DepsTestsSuite.scala | 3 +- 2 files changed, 135 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/k8s_integration_test.yml diff --git a/.github/workflows/k8s_integration_test.yml b/.github/workflows/k8s_integration_test.yml new file mode 100644 index 0000000000000..3bf90c33d543f --- /dev/null +++ b/.github/workflows/k8s_integration_test.yml @@ -0,0 +1,133 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Spark on Kubernetes Integration Test (Resource Limited) + +on: + # Note: + # - '**' if we want to trigger for each pr + # - 'master' if we only want to do post validation on master merged + # Enable push to do a post validation for each commits in current branch + push: + branches: + - '**' + # Enable workflow_dispatch to help validate below case manually: + # 1. Validate K8S IT for pull request: + # spark-ref: refs/pull/35786/head + # sync-current: true + # 2. Validate K8S IT for specific SHA: + # spark-ref: {Commit SHA} + # sync-current: true + # 3. Validate K8S IT for specific TAG/Branch using **master workflow**: + # spark-ref: branch-3.3 or v3.3.0 + # sync-current: false + workflow_dispatch: + inputs: + spark-ref: + description: 'Specified PR Ref/SHA/Branch/Tag:' + required: true + default: 'master' + sync-current: + type: choice + description: 'Sync the current branch:' + required: true + options: + - 'true' + - 'false' + +jobs: + k8s-integration-tests: + name: "Run Spark on Kubernetes Integration test (${{ github.event.inputs.spark-ref || github.event.ref }})" + runs-on: ubuntu-20.04 + steps: + # Checkout based branch + - name: Checkout Spark repository + uses: actions/checkout@v2 + with: + fetch-depth: 0 + repository: apache/spark + # Priority: + # 1. workflow_dispatch input + # 2. push.ref in apache projects for push event post validation + # 3. master, current branch for schedule or when we enable trigger for each pr + ref: >- + ${{ + github.event.inputs.spark-ref + || (github.repository == 'apache/spark' && github.event_name == 'push' && github.event.ref) + || 'master' + }} + - name: Sync the current branch with the latest in Apache Spark + # Do sync when workflow_dispatch(with sync-current=true) or other left events like push/schedule + if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.sync-current == 'true' }} + run: | + echo "APACHE_SPARK_REF=$(git rev-parse HEAD)" >> $GITHUB_ENV + git fetch https://github.com/$GITHUB_REPOSITORY.git ${GITHUB_REF#refs/heads/} + git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD + git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" + - name: Cache Scala, SBT and Maven + uses: actions/cache@v2 + with: + path: | + build/apache-maven-* + build/scala-* + build/*.jar + ~/.sbt + key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }} + restore-keys: | + build- + - name: Cache Coursier local repository + uses: actions/cache@v2 + with: + path: ~/.cache/coursier + key: k8s-integration-coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} + restore-keys: | + k8s-integration-coursier- + - name: Install Java 8 + uses: actions/setup-java@v1 + with: + java-version: 8 + - name: start minikube + run: | + # See more in "Installation" https://minikube.sigs.k8s.io/docs/start/ + curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 + sudo install minikube-linux-amd64 /usr/local/bin/minikube + # system limit cpu:2, memory: 6947MB + minikube start --cpus 2 --memory 6144 + - name: Print K8S pods and nodes info + run: | + kubectl get pods -A + kubectl describe node + - name: Run Spark on K8S integration test (With driver cpu 0.5, executor cpu 0.2 limited) + run: | + # Prepare PV test + PVC_TMP_DIR=$(mktemp -d) + export PVC_TESTS_HOST_PATH=$PVC_TMP_DIR + export PVC_TESTS_VM_PATH=$PVC_TMP_DIR + minikube mount ${PVC_TESTS_HOST_PATH}:${PVC_TESTS_VM_PATH} --9p-version=9p2000.L --gid=0 --uid=185 & + + kubectl create clusterrolebinding serviceaccounts-cluster-admin --clusterrole=cluster-admin --group=system:serviceaccounts || true + eval $(minikube docker-env) + # - Exclude Volcano test (-Pvolcano), batch jobs need more CPU resource + build/sbt -Psparkr -Pkubernetes -Pkubernetes-integration-tests -Dspark.kubernetes.test.minioRequestCores=0.25 -Dspark.kubernetes.test.driverRequestCores=0.5 -Dspark.kubernetes.test.executorRequestCores=0.2 "kubernetes-integration-tests/test" + - name: Upload Spark on K8S integration tests log files + if: failure() + uses: actions/upload-artifact@v2 + with: + name: spark-on-kubernetes-it-log + path: "**/target/integration-tests.log" diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala index 3f3c4ef14607c..e11309427ae3d 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala @@ -33,6 +33,7 @@ import org.scalatest.time.{Minutes, Span} import org.apache.spark.SparkException import org.apache.spark.deploy.k8s.integrationtest.DepsTestsSuite.{DEPS_TIMEOUT, FILE_CONTENTS, HOST_PATH} import org.apache.spark.deploy.k8s.integrationtest.KubernetesSuite.{INTERVAL, MinikubeTag, TIMEOUT} +import org.apache.spark.deploy.k8s.integrationtest.TestConstants.CONFIG_MINIO_REQUEST_CORES import org.apache.spark.deploy.k8s.integrationtest.Utils.getExamplesJarName import org.apache.spark.deploy.k8s.integrationtest.backend.minikube.Minikube import org.apache.spark.internal.config.{ARCHIVES, PYSPARK_DRIVER_PYTHON, PYSPARK_PYTHON} @@ -58,7 +59,7 @@ private[spark] trait DepsTestsSuite { k8sSuite: KubernetesSuite => ).toArray val resources = Map( - "cpu" -> new Quantity("1"), + "cpu" -> new Quantity(sys.props.get(CONFIG_MINIO_REQUEST_CORES).getOrElse("1")), "memory" -> new Quantity("512M") ).asJava From 8c96cdd9febd2bc60d88437cbc927be7573b999a Mon Sep 17 00:00:00 2001 From: Yikun Jiang Date: Fri, 8 Apr 2022 07:55:53 +0800 Subject: [PATCH 2/2] set minio cpu to 250m --- .github/workflows/k8s_integration_test.yml | 2 +- .../spark/deploy/k8s/integrationtest/DepsTestsSuite.scala | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/k8s_integration_test.yml b/.github/workflows/k8s_integration_test.yml index 3bf90c33d543f..fbed17ea1b2ed 100644 --- a/.github/workflows/k8s_integration_test.yml +++ b/.github/workflows/k8s_integration_test.yml @@ -124,7 +124,7 @@ jobs: kubectl create clusterrolebinding serviceaccounts-cluster-admin --clusterrole=cluster-admin --group=system:serviceaccounts || true eval $(minikube docker-env) # - Exclude Volcano test (-Pvolcano), batch jobs need more CPU resource - build/sbt -Psparkr -Pkubernetes -Pkubernetes-integration-tests -Dspark.kubernetes.test.minioRequestCores=0.25 -Dspark.kubernetes.test.driverRequestCores=0.5 -Dspark.kubernetes.test.executorRequestCores=0.2 "kubernetes-integration-tests/test" + build/sbt -Psparkr -Pkubernetes -Pkubernetes-integration-tests -Dspark.kubernetes.test.driverRequestCores=0.5 -Dspark.kubernetes.test.executorRequestCores=0.2 "kubernetes-integration-tests/test" - name: Upload Spark on K8S integration tests log files if: failure() uses: actions/upload-artifact@v2 diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala index e11309427ae3d..10ac197ec7715 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala @@ -33,7 +33,6 @@ import org.scalatest.time.{Minutes, Span} import org.apache.spark.SparkException import org.apache.spark.deploy.k8s.integrationtest.DepsTestsSuite.{DEPS_TIMEOUT, FILE_CONTENTS, HOST_PATH} import org.apache.spark.deploy.k8s.integrationtest.KubernetesSuite.{INTERVAL, MinikubeTag, TIMEOUT} -import org.apache.spark.deploy.k8s.integrationtest.TestConstants.CONFIG_MINIO_REQUEST_CORES import org.apache.spark.deploy.k8s.integrationtest.Utils.getExamplesJarName import org.apache.spark.deploy.k8s.integrationtest.backend.minikube.Minikube import org.apache.spark.internal.config.{ARCHIVES, PYSPARK_DRIVER_PYTHON, PYSPARK_PYTHON} @@ -59,7 +58,7 @@ private[spark] trait DepsTestsSuite { k8sSuite: KubernetesSuite => ).toArray val resources = Map( - "cpu" -> new Quantity(sys.props.get(CONFIG_MINIO_REQUEST_CORES).getOrElse("1")), + "cpu" -> new Quantity("250m"), "memory" -> new Quantity("512M") ).asJava