Support java-library dependency declaration in SparkDockerPlugin (apa…

…che#630)

* Try using runtimeElements

* Add tests

* Use path instead of name parameter

* Add newline

Co-authored-by: Willi Raschkowski <wraschkowski@palantir.com>

spark-docker-image-generator/src/main/java/org/apache/spark/deploy/kubernetes/docker/gradle/SparkDockerPlugin.java

@@ -56,7 +56,7 @@ public void apply(Project project) {
                 project.getConfigurations().maybeCreate(SPARK_DOCKER_RUNTIME_CONFIGURATION_NAME);
         File dockerFile = new File(dockerBuildDirectory, "Dockerfile");
         project.getPluginManager().withPlugin("java", plugin -> {
-            Configuration runtimeConfiguration = project.getConfigurations().findByName("runtime");
+            Configuration runtimeConfiguration = project.getConfigurations().findByName("runtimeElements");
             if (runtimeConfiguration == null) {
                 log.warn("No runtime configuration was found for a reference configuration for building"
                         + " your Spark application's docker images.");

spark-docker-image-generator/src/test/java/org/apache/spark/deploy/kubernetes/docker/gradle/SparkDockerPluginSuite.java

@@ -38,6 +38,8 @@
 public final class SparkDockerPluginSuite {
 
     private static final File TEST_PROJECT_DIR = Paths.get("src/test/resources/plugin-test-project").toFile();
+    private static final File TEST_LIBRARY_PROJECT_DIR =
+            Paths.get("src/test/resources/plugin-test-library-project").toFile();
 
     private final String version;
     private final String expectedRegistry;
@@ -73,6 +75,11 @@ public void after() throws Exception {
 
     @Test
     public void testSetupProject() throws Exception {
+        runSetupProjectTest(TEST_PROJECT_DIR);
+        runSetupProjectTest(TEST_LIBRARY_PROJECT_DIR);
+    }
+
+    private void runSetupProjectTest(File testProjectDir) throws Exception {
         GradleRunner runner = GradleRunner.create()
                 .withPluginClasspath()
                 .withArguments(
@@ -82,7 +89,7 @@ public void testSetupProject() throws Exception {
                         String.format("-Dtest-project-version=%s", version),
                         "--stacktrace",
                         "--info")
-                .withProjectDir(TEST_PROJECT_DIR)
+                .withProjectDir(testProjectDir)
                 .forwardOutput();
         runner.build();
 
@@ -103,6 +110,7 @@ public void testSetupProject() throws Exception {
                         containerId,
                         "/opt/spark/jars",
                         "guava-21.0.jar",
+                        "commons-compress-1.18.jar",
                         "commons-io-2.4.jar",
                         String.format("plugin-test-project-%s.jar", version));
                 expectFilesInDir(

spark-docker-image-generator/src/test/resources/plugin-test-library-project/.gitignore

@@ -0,0 +1 @@
+build

spark-docker-image-generator/src/test/resources/plugin-test-library-project/build.gradle

@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+buildscript {
+  repositories {
+    jcenter()
+    maven { url "http://palantir.bintray.com/releases" }
+  }
+}
+
+plugins {
+  id 'org.apache.spark.deploy.kubernetes.docker.gradle'
+}
+
+repositories {
+  jcenter()
+  maven { url "http://palantir.bintray.com/releases" }
+  maven { url System.env.MAVEN_REPO }
+}
+
+apply plugin: 'java-library'
+version System.getProperty("test-project-version")
+group 'org.apache.spark'
+sourceCompatibility = 1.8
+
+dependencies {
+  api 'org.apache.commons:commons-compress:1.18'
+  implementation 'com.google.guava:guava:21.0'
+  sparkDockerRuntime 'commons-io:commons-io:2.4'
+}
+
+sparkDocker {
+  baseImage 'openjdk:8-jdk-slim'
+  imagePath 'spark/spark-test-app'
+  snapshotRegistry 'docker.palantir.test.snapshot'
+  releaseRegistry 'docker.palantir.test.release'
+  tags System.getProperty('docker-tag')
+}

spark-docker-image-generator/src/test/resources/plugin-test-library-project/settings.gradle

@@ -0,0 +1,18 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+rootProject.name = "plugin-test-project"

spark-docker-image-generator/src/test/resources/plugin-test-library-project/src/main/java/org/apache/spark/test/HelloWorld.java

@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.test;
+
+import java.util.List;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableList;
+
+public final class HelloWorld {
+
+    private HelloWorld() { }
+
+    public static void main(String[] args) {
+        List<String> strs = ImmutableList.of("Hello", "World");
+        System.out.println(Joiner.on(",").join(strs));
+    }
+}

spark-docker-image-generator/src/test/resources/plugin-test-project/build.gradle

@@ -39,6 +39,7 @@ sourceCompatibility = 1.8
 
 dependencies {
   compile 'com.google.guava:guava:21.0'
+  compile 'org.apache.commons:commons-compress:1.18'
   sparkDockerRuntime 'commons-io:commons-io:2.4'
 }
 

spark-docker-image-generator/src/test/resources/plugin-test-project/settings.gradle

@@ -15,4 +15,4 @@
  * limitations under the License.
  */
 
-rootProject.name = "plugin-test-project"
+rootProject.name = "plugin-test-project"