[POC][K8S][INFRA] Enable resource limited Spark on K8S integration tests in Github Action

.github/workflows/k8s_integration_test.yml

@@ -0,0 +1,133 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: Spark on Kubernetes Integration Test (Resource Limited)
+
+on:
+  # Note:
+  # - '**' if we want to trigger for each pr
+  # - 'master' if we only want to do post validation on master merged
+  # Enable push to do a post validation for each commits in current branch
+  push:
+    branches:
+      - '**'
+  # Enable workflow_dispatch to help validate below case manually:
+  # 1. Validate K8S IT for pull request:
+  #    spark-ref: refs/pull/35786/head
+  #    sync-current: true
+  # 2. Validate K8S IT for specific SHA:
+  #    spark-ref: {Commit SHA}
+  #    sync-current: true
+  # 3. Validate K8S IT for specific TAG/Branch using **master workflow**:
+  #    spark-ref: branch-3.3 or v3.3.0
+  #    sync-current: false
+  workflow_dispatch:
+    inputs:
+      spark-ref:
+        description: 'Specified PR Ref/SHA/Branch/Tag:'
+        required: true
+        default: 'master'
+      sync-current:
+        type: choice
+        description: 'Sync the current branch:'
+        required: true
+        options:
+          - 'true'
+          - 'false'
+
+jobs:
+  k8s-integration-tests:
+    name: "Run Spark on Kubernetes Integration test (${{ github.event.inputs.spark-ref || github.event.ref }})"
+    runs-on: ubuntu-20.04
+    steps:
+      # Checkout based branch
+      - name: Checkout Spark repository
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          repository: apache/spark
+          # Priority:
+          # 1. workflow_dispatch input
+          # 2. push.ref in apache projects for push event post validation
+          # 3. master, current branch for schedule or when we enable trigger for each pr
+          ref: >-
+            ${{
+              github.event.inputs.spark-ref
+              || (github.repository == 'apache/spark' && github.event_name == 'push' && github.event.ref)
+              || 'master'
+            }}
+      - name: Sync the current branch with the latest in Apache Spark
+        # Do sync when workflow_dispatch(with sync-current=true) or other left events like push/schedule
+        if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.sync-current == 'true' }}
+        run: |
+          echo "APACHE_SPARK_REF=$(git rev-parse HEAD)" >> $GITHUB_ENV
+          git fetch https://github.com/$GITHUB_REPOSITORY.git ${GITHUB_REF#refs/heads/}
+          git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD
+          git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit"
+      - name: Cache Scala, SBT and Maven
+        uses: actions/cache@v2
+        with:
+          path: |
+            build/apache-maven-*
+            build/scala-*
+            build/*.jar
+            ~/.sbt
+          key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }}
+          restore-keys: |
+            build-
+      - name: Cache Coursier local repository
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/coursier
+          key: k8s-integration-coursier-${{ hashFiles('**/pom.xml', '**/plugins.sbt') }}
+          restore-keys: |
+            k8s-integration-coursier-
+      - name: Install Java 8
+        uses: actions/setup-java@v1
+        with:
+          java-version: 8
+      - name: start minikube
+        run: |
+          # See more in "Installation" https://minikube.sigs.k8s.io/docs/start/
+          curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64
+          sudo install minikube-linux-amd64 /usr/local/bin/minikube
+          # system limit cpu:2, memory: 6947MB
+          minikube start --cpus 2 --memory 6144
+      - name: Print K8S pods and nodes info
+        run: |
+          kubectl get pods -A
+          kubectl describe node
+      - name: Run Spark on K8S integration test (With driver cpu 0.5, executor cpu 0.2 limited)
+        run: |
+          # Prepare PV test
+          PVC_TMP_DIR=$(mktemp -d)
+          export PVC_TESTS_HOST_PATH=$PVC_TMP_DIR
+          export PVC_TESTS_VM_PATH=$PVC_TMP_DIR
+          minikube mount ${PVC_TESTS_HOST_PATH}:${PVC_TESTS_VM_PATH} --9p-version=9p2000.L --gid=0 --uid=185 &
+
+          kubectl create clusterrolebinding serviceaccounts-cluster-admin --clusterrole=cluster-admin --group=system:serviceaccounts || true
+          eval $(minikube docker-env)
+          # - Exclude Volcano test (-Pvolcano), batch jobs need more CPU resource
+          build/sbt -Psparkr -Pkubernetes -Pkubernetes-integration-tests -Dspark.kubernetes.test.driverRequestCores=0.5 -Dspark.kubernetes.test.executorRequestCores=0.2 "kubernetes-integration-tests/test"
+      - name: Upload Spark on K8S integration tests log files
+        if: failure()
+        uses: actions/upload-artifact@v2
+        with:
+          name: spark-on-kubernetes-it-log
+          path: "**/target/integration-tests.log"

resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala

@@ -58,7 +58,7 @@ private[spark] trait DepsTestsSuite { k8sSuite: KubernetesSuite =>
     ).toArray
 
     val resources = Map(
-      "cpu" -> new Quantity("1"),
+      "cpu" -> new Quantity("250m"),
       "memory" -> new Quantity("512M")
     ).asJava