Update e2e tests

.github/workflows/integration.yaml

@@ -72,23 +72,8 @@ jobs:
       - name: Run unit tests
         run: make unit-test
 
-      - name: Build Spark-Operator Docker Image
-        run: make docker-build IMAGE_TAG=latest
-
-      - name: Check changes in resources used in docker file
-        run: |
-          DOCKERFILE_RESOURCES=$(cat Dockerfile | grep -P -o "COPY [a-zA-Z0-9].*? " | cut -c6-)
-          for resource in $DOCKERFILE_RESOURCES; do
-            # If the resource is different
-            if ! git diff  --quiet origin/master -- $resource; then
-              ## And the appVersion hasn't been updated
-              if ! git diff origin/master -- charts/spark-operator-chart/Chart.yaml | grep +appVersion; then
-                echo "resource used in docker.io/kubeflow/spark-operator has changed in $resource, need to update the appVersion in charts/spark-operator-chart/Chart.yaml"
-                git diff origin/master -- $resource;
-                echo "failing the build... " && false
-              fi
-            fi
-          done
+      - name: Build Spark operator
+        run: make build-operator
 
   build-helm-chart:
     runs-on: ubuntu-latest
@@ -131,7 +116,7 @@ jobs:
       - name: Run chart-testing (lint)
         if: steps.list-changed.outputs.changed == 'true'
         env:
-            BRANCH: ${{ steps.get_branch.outputs.BRANCH }}
+          BRANCH: ${{ steps.get_branch.outputs.BRANCH }}
         run: ct lint --check-version-increment=false --target-branch $BRANCH
 
       - name: Detect CRDs drift between chart and manifest

test/e2e/sparkapplication_test.go

@@ -26,11 +26,14 @@ import (
 	. "github.com/onsi/gomega"
 
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/equality"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/yaml"
 
 	"github.com/kubeflow/spark-operator/api/v1beta2"
+	"github.com/kubeflow/spark-operator/pkg/common"
 	"github.com/kubeflow/spark-operator/pkg/util"
 )
 
@@ -130,13 +133,43 @@ var _ = Describe("Example SparkApplication", func() {
 			}
 		})
 
-		It("Should complete successfully", func() {
+		It("Should complete successfully with configmap mounted", func() {
 			By("Waiting for SparkApplication to complete")
 			key := types.NamespacedName{Namespace: app.Namespace, Name: app.Name}
 			Expect(waitForSparkApplicationCompleted(ctx, key)).NotTo(HaveOccurred())
 
-			By("Checking out driver logs")
+			By("Checking out whether volumes are mounted to driver pod")
 			driverPodName := util.GetDriverPodName(app)
+			driverPodKey := types.NamespacedName{Namespace: app.Namespace, Name: driverPodName}
+			driverPod := &corev1.Pod{}
+			Expect(k8sClient.Get(ctx, driverPodKey, driverPod)).NotTo(HaveOccurred())
+			hasVolumes := false
+			hasVolumeMounts := false
+			for _, volume := range app.Spec.Volumes {
+				for _, podVolume := range driverPod.Spec.Volumes {
+					if volume.Name == podVolume.Name {
+						hasVolumes = true
+						break
+					}
+				}
+			}
+			for _, volumeMount := range app.Spec.Driver.VolumeMounts {
+				for _, container := range driverPod.Spec.Containers {
+					if container.Name != common.SparkDriverContainerName {
+						continue
+					}
+					for _, podVolumeMount := range container.VolumeMounts {
+						if equality.Semantic.DeepEqual(volumeMount, podVolumeMount) {
+							hasVolumeMounts = true
+							break
+						}
+					}
+				}
+			}
+			Expect(hasVolumes).To(BeTrue())
+			Expect(hasVolumeMounts).To(BeTrue())
+
+			By("Checking out driver logs")
 			bytes, err := clientset.CoreV1().Pods(app.Namespace).GetLogs(driverPodName, &corev1.PodLogOptions{}).Do(ctx).Raw()
 			Expect(err).NotTo(HaveOccurred())
 			Expect(bytes).NotTo(BeEmpty())
@@ -176,8 +209,26 @@ var _ = Describe("Example SparkApplication", func() {
 			key := types.NamespacedName{Namespace: app.Namespace, Name: app.Name}
 			Expect(waitForSparkApplicationCompleted(ctx, key)).NotTo(HaveOccurred())
 
-			By("Checking out driver logs")
+			By("Checking out whether resource requests and limits of driver pod are set")
 			driverPodName := util.GetDriverPodName(app)
+			driverPodKey := types.NamespacedName{Namespace: app.Namespace, Name: driverPodName}
+			driverPod := &corev1.Pod{}
+			Expect(k8sClient.Get(ctx, driverPodKey, driverPod)).NotTo(HaveOccurred())
+			for _, container := range driverPod.Spec.Containers {
+				if container.Name != common.SparkDriverContainerName {
+					continue
+				}
+				if app.Spec.Driver.CoreRequest != nil {
+					Expect(container.Resources.Requests.Cpu().Equal(resource.MustParse(*app.Spec.Driver.CoreRequest))).To(BeTrue())
+				}
+				if app.Spec.Driver.CoreLimit != nil {
+					Expect(container.Resources.Limits.Cpu().Equal(resource.MustParse(*app.Spec.Driver.CoreLimit))).To(BeTrue())
+				}
+				Expect(container.Resources.Requests.Memory).NotTo(BeNil())
+				Expect(container.Resources.Limits.Memory).NotTo(BeNil())
+			}
+
+			By("Checking out driver logs")
 			bytes, err := clientset.CoreV1().Pods(app.Namespace).GetLogs(driverPodName, &corev1.PodLogOptions{}).Do(ctx).Raw()
 			Expect(err).NotTo(HaveOccurred())
 			Expect(bytes).NotTo(BeEmpty())

test/e2e/suit_test.go

@@ -149,6 +149,8 @@ var _ = BeforeSuite(func() {
 	validatingWebhookKey := types.NamespacedName{Name: ValidatingWebhookName}
 	Expect(waitForMutatingWebhookReady(context.Background(), mutatingWebhookKey)).NotTo(HaveOccurred())
 	Expect(waitForValidatingWebhookReady(context.Background(), validatingWebhookKey)).NotTo(HaveOccurred())
+	// TODO: Remove this when there is a better way to ensure the webhooks are ready before running the e2e tests.
+	time.Sleep(10 * time.Second)
 })
 
 var _ = AfterSuite(func() {