From 07e22811bfcb57c5ccabdfd83976f3611a793eb7 Mon Sep 17 00:00:00 2001 From: panbingkun Date: Mon, 19 Feb 2024 10:55:36 -0800 Subject: [PATCH] [SPARK-47089][BUILD][TESTS] Migrate `mockito 4` to `mockito 5` ### What changes were proposed in this pull request? The pr aims to migrate `mockito 4` to `mockito 5`. ### Why are the changes needed? This is the prerequisite behavior for upgrading `scalatest mockito` to version `3.2.18`. `Scalatest mockito 3.2.18` no longer supports `mockito 4`. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Pass GA. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #45158 from panbingkun/SPARK-47089. Authored-by: panbingkun Signed-off-by: Dongjoon Hyun --- mllib-local/pom.xml | 5 ----- mllib/pom.xml | 5 ----- pom.xml | 16 +++++--------- resource-managers/kubernetes/core/pom.xml | 11 +++++++++- .../spark/deploy/k8s/submit/ClientSuite.scala | 12 +++++----- .../k8s/ExecutorPodsAllocatorSuite.scala | 22 ++++++++++--------- sql/core/pom.xml | 5 ----- 7 files changed, 33 insertions(+), 43 deletions(-) diff --git a/mllib-local/pom.xml b/mllib-local/pom.xml index 408aec1ff276b..ecfe45f046f2b 100644 --- a/mllib-local/pom.xml +++ b/mllib-local/pom.xml @@ -52,11 +52,6 @@ mockito-core test - - org.mockito - mockito-inline - test - org.apache.spark spark-tags_${scala.binary.version} diff --git a/mllib/pom.xml b/mllib/pom.xml index 88400e7ba6ac4..49a53e9a994fe 100644 --- a/mllib/pom.xml +++ b/mllib/pom.xml @@ -113,11 +113,6 @@ mockito-core test - - org.mockito - mockito-inline - test - org.apache.spark spark-streaming_${scala.binary.version} diff --git a/pom.xml b/pom.xml index 410cefb0e8d97..427a68f29e146 100644 --- a/pom.xml +++ b/pom.xml @@ -412,7 +412,7 @@ org.scalatestplus - mockito-4-11_${scala.binary.version} + mockito-5-8_${scala.binary.version} test @@ -1156,7 +1156,7 @@ org.scalatestplus - mockito-4-11_${scala.binary.version} + mockito-5-8_${scala.binary.version} 3.2.17.0 test @@ -1175,25 +1175,19 @@ org.mockito mockito-core - 4.11.0 - test - - - org.mockito - mockito-inline - 4.11.0 + 5.8.0 test net.bytebuddy byte-buddy - 1.14.5 + 1.14.10 test net.bytebuddy byte-buddy-agent - 1.14.5 + 1.14.10 test diff --git a/resource-managers/kubernetes/core/pom.xml b/resource-managers/kubernetes/core/pom.xml index afd42f6a82a06..fa0fd454ccc44 100644 --- a/resource-managers/kubernetes/core/pom.xml +++ b/resource-managers/kubernetes/core/pom.xml @@ -120,7 +120,16 @@ mockito-core test - + + net.bytebuddy + byte-buddy + test + + + net.bytebuddy + byte-buddy-agent + test + org.jmock diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/ClientSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/ClientSuite.scala index de5244be95e28..c98654d6c02aa 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/ClientSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/ClientSuite.scala @@ -167,7 +167,7 @@ class ClientSuite extends SparkFunSuite with BeforeAndAfter { private var kconf: KubernetesDriverConf = _ private var createdPodArgumentCaptor: ArgumentCaptor[Pod] = _ - private var createdResourcesArgumentCaptor: ArgumentCaptor[HasMetadata] = _ + private var createdResourcesArgumentCaptor: ArgumentCaptor[Array[HasMetadata]] = _ before { MockitoAnnotations.openMocks(this).close() @@ -179,7 +179,7 @@ class ClientSuite extends SparkFunSuite with BeforeAndAfter { when(podsWithNamespace.withName(POD_NAME)).thenReturn(namedPods) createdPodArgumentCaptor = ArgumentCaptor.forClass(classOf[Pod]) - createdResourcesArgumentCaptor = ArgumentCaptor.forClass(classOf[HasMetadata]) + createdResourcesArgumentCaptor = ArgumentCaptor.forClass(classOf[Array[HasMetadata]]) when(podsWithNamespace.resource(fullExpectedPod())).thenReturn(namedPods) when(resourceList.forceConflicts()).thenReturn(resourceList) when(namedPods.serverSideApply()).thenReturn(podWithOwnerReference()) @@ -189,7 +189,7 @@ class ClientSuite extends SparkFunSuite with BeforeAndAfter { when(loggingPodStatusWatcher.watchOrStop(sId)).thenReturn(true) doReturn(resourceList) .when(kubernetesClient) - .resourceList(createdResourcesArgumentCaptor.capture()) + .resourceList(createdResourcesArgumentCaptor.capture(): _*) } test("The client should configure the pod using the builder.") { @@ -210,7 +210,7 @@ class ClientSuite extends SparkFunSuite with BeforeAndAfter { kubernetesClient, loggingPodStatusWatcher) submissionClient.run() - val otherCreatedResources = createdResourcesArgumentCaptor.getAllValues + val otherCreatedResources = createdResourcesArgumentCaptor.getAllValues.asScala.flatten assert(otherCreatedResources.size === 2) val secrets = otherCreatedResources.toArray.filter(_.isInstanceOf[Secret]).toSeq assert(secrets === ADDITIONAL_RESOURCES_WITH_OWNER_REFERENCES) @@ -246,7 +246,7 @@ class ClientSuite extends SparkFunSuite with BeforeAndAfter { kubernetesClient, loggingPodStatusWatcher) submissionClient.run() - val otherCreatedResources = createdResourcesArgumentCaptor.getAllValues + val otherCreatedResources = createdResourcesArgumentCaptor.getAllValues.asScala.flatten // 2 for pre-resource creation/update, 1 for resource creation, 1 for config map assert(otherCreatedResources.size === 4) @@ -326,7 +326,7 @@ class ClientSuite extends SparkFunSuite with BeforeAndAfter { kubernetesClient, loggingPodStatusWatcher) submissionClient.run() - val otherCreatedResources = createdResourcesArgumentCaptor.getAllValues + val otherCreatedResources = createdResourcesArgumentCaptor.getAllValues.asScala.flatten val configMaps = otherCreatedResources.toArray .filter(_.isInstanceOf[ConfigMap]).map(_.asInstanceOf[ConfigMap]) diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocatorSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocatorSuite.scala index eb9246a2a5f2a..f1ef96b942fe1 100644 --- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocatorSuite.scala +++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocatorSuite.scala @@ -124,10 +124,12 @@ class ExecutorPodsAllocatorSuite extends SparkFunSuite with BeforeAndAfter { when(podsWithNamespace.withName(driverPodName)).thenReturn(driverPodOperations) when(podsWithNamespace.resource(any())).thenReturn(podResource) when(podsWithNamespace.withLabel(anyString(), anyString())).thenReturn(labeledPods) - when(podsWithNamespace.withLabelIn(anyString(), any())).thenReturn(labeledPods) + when(podsWithNamespace.withLabelIn( + anyString(), any(classOf[Array[String]]): _*)).thenReturn(labeledPods) when(podsWithNamespace.withField(anyString(), anyString())).thenReturn(labeledPods) when(labeledPods.withLabel(anyString(), anyString())).thenReturn(labeledPods) - when(labeledPods.withLabelIn(anyString(), any())).thenReturn(labeledPods) + when(labeledPods.withLabelIn( + anyString(), any(classOf[Array[String]]): _*)).thenReturn(labeledPods) when(labeledPods.withField(anyString(), anyString())).thenReturn(labeledPods) when(driverPodOperations.get).thenReturn(driverPod) when(driverPodOperations.waitUntilReady(any(), any())).thenReturn(driverPod) @@ -212,7 +214,7 @@ class ExecutorPodsAllocatorSuite extends SparkFunSuite with BeforeAndAfter { .withLabel(SPARK_ROLE_LABEL, SPARK_POD_EXECUTOR_ROLE)) .thenReturn(podOperations) when(podOperations - .withLabelIn(meq(SPARK_EXECUTOR_ID_LABEL), any())) + .withLabelIn(meq(SPARK_EXECUTOR_ID_LABEL), any(classOf[Array[String]]): _*)) .thenReturn(podOperations) val startTime = Instant.now.toEpochMilli @@ -362,7 +364,7 @@ class ExecutorPodsAllocatorSuite extends SparkFunSuite with BeforeAndAfter { .withLabel(SPARK_ROLE_LABEL, SPARK_POD_EXECUTOR_ROLE)) .thenReturn(labeledPods) when(labeledPods - .withLabelIn(meq(SPARK_EXECUTOR_ID_LABEL), any())) + .withLabelIn(meq(SPARK_EXECUTOR_ID_LABEL), any(classOf[Array[String]]): _*)) .thenReturn(labeledPods) val startTime = Instant.now.toEpochMilli @@ -432,7 +434,7 @@ class ExecutorPodsAllocatorSuite extends SparkFunSuite with BeforeAndAfter { .withLabel(SPARK_ROLE_LABEL, SPARK_POD_EXECUTOR_ROLE)) .thenReturn(labeledPods) when(labeledPods - .withLabelIn(meq(SPARK_EXECUTOR_ID_LABEL), any())) + .withLabelIn(meq(SPARK_EXECUTOR_ID_LABEL), any(classOf[Array[String]]): _*)) .thenReturn(labeledPods) val startTime = Instant.now.toEpochMilli @@ -477,7 +479,7 @@ class ExecutorPodsAllocatorSuite extends SparkFunSuite with BeforeAndAfter { .withLabel(SPARK_ROLE_LABEL, SPARK_POD_EXECUTOR_ROLE)) .thenReturn(labeledPods) when(labeledPods - .withLabelIn(meq(SPARK_EXECUTOR_ID_LABEL), any())) + .withLabelIn(meq(SPARK_EXECUTOR_ID_LABEL), any(classOf[Array[String]]): _*)) .thenReturn(labeledPods) val startTime = Instant.now.toEpochMilli @@ -560,7 +562,7 @@ class ExecutorPodsAllocatorSuite extends SparkFunSuite with BeforeAndAfter { .withLabel(SPARK_ROLE_LABEL, SPARK_POD_EXECUTOR_ROLE)) .thenReturn(labeledPods) when(labeledPods - .withLabelIn(meq(SPARK_EXECUTOR_ID_LABEL), any())) + .withLabelIn(meq(SPARK_EXECUTOR_ID_LABEL), any(classOf[Array[String]]): _*)) .thenReturn(labeledPods) val startTime = Instant.now.toEpochMilli @@ -642,7 +644,7 @@ class ExecutorPodsAllocatorSuite extends SparkFunSuite with BeforeAndAfter { .withLabel(SPARK_ROLE_LABEL, SPARK_POD_EXECUTOR_ROLE)) .thenReturn(labeledPods) when(labeledPods - .withLabelIn(meq(SPARK_EXECUTOR_ID_LABEL), any())) + .withLabelIn(meq(SPARK_EXECUTOR_ID_LABEL), any(classOf[Array[String]]): _*)) .thenReturn(labeledPods) val startTime = Instant.now.toEpochMilli @@ -794,7 +796,7 @@ class ExecutorPodsAllocatorSuite extends SparkFunSuite with BeforeAndAfter { .withLabel(SPARK_ROLE_LABEL, SPARK_POD_EXECUTOR_ROLE)) .thenReturn(labeledPods) when(labeledPods - .withLabelIn(meq(SPARK_EXECUTOR_ID_LABEL), any())) + .withLabelIn(meq(SPARK_EXECUTOR_ID_LABEL), any(classOf[Array[String]]): _*)) .thenReturn(labeledPods) val startTime = Instant.now.toEpochMilli @@ -890,7 +892,7 @@ class ExecutorPodsAllocatorSuite extends SparkFunSuite with BeforeAndAfter { .withLabel(SPARK_ROLE_LABEL, SPARK_POD_EXECUTOR_ROLE)) .thenReturn(labeledPods) when(labeledPods - .withLabelIn(meq(SPARK_EXECUTOR_ID_LABEL), any())) + .withLabelIn(meq(SPARK_EXECUTOR_ID_LABEL), any(classOf[Array[String]]): _*)) .thenReturn(labeledPods) val startTime = Instant.now.toEpochMilli diff --git a/sql/core/pom.xml b/sql/core/pom.xml index 1ef41f9b43daf..8b1b51352a201 100644 --- a/sql/core/pom.xml +++ b/sql/core/pom.xml @@ -213,11 +213,6 @@ mockito-core test - - org.mockito - mockito-inline - test - org.seleniumhq.selenium selenium-java