From ef0734393520424089304a4e6b36e47623734d52 Mon Sep 17 00:00:00 2001 From: Yikun Jiang Date: Thu, 7 Apr 2022 16:54:06 +0800 Subject: [PATCH] Add resource limited k8s it --- .github/workflows/k8s_integration_test.yml | 133 ++++++++++++++++++ .../kubernetes/integration-tests/README.md | 24 ++++ .../k8s/integrationtest/DepsTestsSuite.scala | 3 +- .../k8s/integrationtest/KubernetesSuite.scala | 6 + .../k8s/integrationtest/TestConstants.scala | 3 + 5 files changed, 168 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/k8s_integration_test.yml diff --git a/.github/workflows/k8s_integration_test.yml b/.github/workflows/k8s_integration_test.yml new file mode 100644 index 0000000000000..3bf90c33d543f --- /dev/null +++ b/.github/workflows/k8s_integration_test.yml @@ -0,0 +1,133 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Spark on Kubernetes Integration Test (Resource Limited) + +on: + # Note: + # - '**' if we want to trigger for each pr + # - 'master' if we only want to do post validation on master merged + # Enable push to do a post validation for each commits in current branch + push: + branches: + - '**' + # Enable workflow_dispatch to help validate below case manually: + # 1. Validate K8S IT for pull request: + # spark-ref: refs/pull/35786/head + # sync-current: true + # 2. Validate K8S IT for specific SHA: + # spark-ref: {Commit SHA} + # sync-current: true + # 3. Validate K8S IT for specific TAG/Branch using **master workflow**: + # spark-ref: branch-3.3 or v3.3.0 + # sync-current: false + workflow_dispatch: + inputs: + spark-ref: + description: 'Specified PR Ref/SHA/Branch/Tag:' + required: true + default: 'master' + sync-current: + type: choice + description: 'Sync the current branch:' + required: true + options: + - 'true' + - 'false' + +jobs: + k8s-integration-tests: + name: "Run Spark on Kubernetes Integration test (${{ github.event.inputs.spark-ref || github.event.ref }})" + runs-on: ubuntu-20.04 + steps: + # Checkout based branch + - name: Checkout Spark repository + uses: actions/checkout@v2 + with: + fetch-depth: 0 + repository: apache/spark + # Priority: + # 1. workflow_dispatch input + # 2. push.ref in apache projects for push event post validation + # 3. master, current branch for schedule or when we enable trigger for each pr + ref: >- + ${{ + github.event.inputs.spark-ref + || (github.repository == 'apache/spark' && github.event_name == 'push' && github.event.ref) + || 'master' + }} + - name: Sync the current branch with the latest in Apache Spark + # Do sync when workflow_dispatch(with sync-current=true) or other left events like push/schedule + if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.sync-current == 'true' }} + run: | + echo "APACHE_SPARK_REF=$(git rev-parse HEAD)" >> $GITHUB_ENV + git fetch https://github.com/$GITHUB_REPOSITORY.git ${GITHUB_REF#refs/heads/} + git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD + git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" + - name: Cache Scala, SBT and Maven + uses: actions/cache@v2 + with: + path: | + build/apache-maven-* + build/scala-* + build/*.jar + ~/.sbt + key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }} + restore-keys: | + build- + - name: Cache Coursier local repository + uses: actions/cache@v2 + with: + path: ~/.cache/coursier + key: k8s-integration-coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }} + restore-keys: | + k8s-integration-coursier- + - name: Install Java 8 + uses: actions/setup-java@v1 + with: + java-version: 8 + - name: start minikube + run: | + # See more in "Installation" https://minikube.sigs.k8s.io/docs/start/ + curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 + sudo install minikube-linux-amd64 /usr/local/bin/minikube + # system limit cpu:2, memory: 6947MB + minikube start --cpus 2 --memory 6144 + - name: Print K8S pods and nodes info + run: | + kubectl get pods -A + kubectl describe node + - name: Run Spark on K8S integration test (With driver cpu 0.5, executor cpu 0.2 limited) + run: | + # Prepare PV test + PVC_TMP_DIR=$(mktemp -d) + export PVC_TESTS_HOST_PATH=$PVC_TMP_DIR + export PVC_TESTS_VM_PATH=$PVC_TMP_DIR + minikube mount ${PVC_TESTS_HOST_PATH}:${PVC_TESTS_VM_PATH} --9p-version=9p2000.L --gid=0 --uid=185 & + + kubectl create clusterrolebinding serviceaccounts-cluster-admin --clusterrole=cluster-admin --group=system:serviceaccounts || true + eval $(minikube docker-env) + # - Exclude Volcano test (-Pvolcano), batch jobs need more CPU resource + build/sbt -Psparkr -Pkubernetes -Pkubernetes-integration-tests -Dspark.kubernetes.test.minioRequestCores=0.25 -Dspark.kubernetes.test.driverRequestCores=0.5 -Dspark.kubernetes.test.executorRequestCores=0.2 "kubernetes-integration-tests/test" + - name: Upload Spark on K8S integration tests log files + if: failure() + uses: actions/upload-artifact@v2 + with: + name: spark-on-kubernetes-it-log + path: "**/target/integration-tests.log" diff --git a/resource-managers/kubernetes/integration-tests/README.md b/resource-managers/kubernetes/integration-tests/README.md index 748664cf41b74..3a4e67a7700a7 100644 --- a/resource-managers/kubernetes/integration-tests/README.md +++ b/resource-managers/kubernetes/integration-tests/README.md @@ -268,6 +268,30 @@ to the wrapper scripts and using the wrapper scripts will simply set these appro + + spark.kubernetes.test.minioRequestCores + + Set cpu resource for minio pod in test, this is currently only for test on cpu resource limited cluster, + it's not recommended for other scenarios. + + + + + spark.kubernetes.test.driverRequestCores + + Set cpu resource for each driver pod in test, this is currently only for test on cpu resource limited cluster, + it's not recommended for other scenarios. + + + + + spark.kubernetes.test.executorRequestCores + + Set cpu resource for each executor pod in test, this is currently only for test on cpu resource limited cluster, + it's not recommended for other scenarios. + + + # Running the Kubernetes Integration Tests with SBT diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala index 3f3c4ef14607c..e11309427ae3d 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala @@ -33,6 +33,7 @@ import org.scalatest.time.{Minutes, Span} import org.apache.spark.SparkException import org.apache.spark.deploy.k8s.integrationtest.DepsTestsSuite.{DEPS_TIMEOUT, FILE_CONTENTS, HOST_PATH} import org.apache.spark.deploy.k8s.integrationtest.KubernetesSuite.{INTERVAL, MinikubeTag, TIMEOUT} +import org.apache.spark.deploy.k8s.integrationtest.TestConstants.CONFIG_MINIO_REQUEST_CORES import org.apache.spark.deploy.k8s.integrationtest.Utils.getExamplesJarName import org.apache.spark.deploy.k8s.integrationtest.backend.minikube.Minikube import org.apache.spark.internal.config.{ARCHIVES, PYSPARK_DRIVER_PYTHON, PYSPARK_PYTHON} @@ -58,7 +59,7 @@ private[spark] trait DepsTestsSuite { k8sSuite: KubernetesSuite => ).toArray val resources = Map( - "cpu" -> new Quantity("1"), + "cpu" -> new Quantity(sys.props.get(CONFIG_MINIO_REQUEST_CORES).getOrElse("1")), "memory" -> new Quantity("512M") ).asJava diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala index 3db51b2860023..5c572b62c003e 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala @@ -192,6 +192,12 @@ class KubernetesSuite extends SparkFunSuite .set("spark.kubernetes.driver.label.spark-app-locator", appLocator) .set("spark.kubernetes.executor.label.spark-app-locator", appLocator) .set(NETWORK_AUTH_ENABLED.key, "true") + sys.props.get(CONFIG_DRIVER_REQUEST_CORES).map { cpu => + sparkAppConf.set("spark.kubernetes.driver.request.cores", cpu) + } + sys.props.get(CONFIG_EXECUTOR_REQUEST_CORES).map { cpu => + sparkAppConf.set("spark.kubernetes.executor.request.cores", cpu) + } if (!kubernetesTestComponents.hasUserSpecifiedNamespace) { kubernetesTestComponents.createNamespace() } diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/TestConstants.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/TestConstants.scala index c46839f1dffcc..563868c03cb75 100644 --- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/TestConstants.scala +++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/TestConstants.scala @@ -34,4 +34,7 @@ object TestConstants { val CONFIG_KEY_IMAGE_TAG_FILE = "spark.kubernetes.test.imageTagFile" val CONFIG_KEY_IMAGE_REPO = "spark.kubernetes.test.imageRepo" val CONFIG_KEY_UNPACK_DIR = "spark.kubernetes.test.unpackSparkDir" + val CONFIG_MINIO_REQUEST_CORES = "spark.kubernetes.test.minioRequestCores" + val CONFIG_DRIVER_REQUEST_CORES = "spark.kubernetes.test.driverRequestCores" + val CONFIG_EXECUTOR_REQUEST_CORES = "spark.kubernetes.test.executorRequestCores" }