Extract deephaven/runtime-base (#1557)

DB-test/build.gradle

@@ -2,7 +2,7 @@ plugins {
   id 'com.bmuschko.docker-remote-api'
 }
 
-evaluationDependsOn ':Integrations' // runtimeBase
+evaluationDependsOn ':docker-runtime-base' // runtimeBase
 
 configurations {
   //compile.extendsFrom dhDb
@@ -58,7 +58,7 @@ tasks.getByName('check').dependsOn Docker.registerDockerTask(project, 'test-in-d
       into 'classes'
     }
   }
-  parentContainers = [project(':Integrations').tasks.findByName('buildDeephavenPython')] // deephaven/runtime-base
+  parentContainers = [project(':docker-runtime-base').tasks.findByName('buildDocker')] // deephaven/runtime-base
   dockerfile {
     // base image with default java, python, wheels
     from 'deephaven/runtime-base:local-build'

Integrations/build.gradle

@@ -1,14 +1,9 @@
-import com.bmuschko.gradle.docker.tasks.image.Dockerfile
-
 plugins {
     id 'com.bmuschko.docker-remote-api'
     id 'com.avast.gradle.docker-compose'
 }
 
-evaluationDependsOn ':docker-java-and-python'
-evaluationDependsOn ':deephaven-jpy'
-evaluationDependsOn ':deephaven-wheel'
-evaluationDependsOn ':deephaven2-wheel'
+evaluationDependsOn ':docker-runtime-base'
 
 configurations {
     compile.extendsFrom dhIntegrations
@@ -66,45 +61,6 @@ idea {
         }
 }
 
-def dockerContext = project.layout.buildDirectory.dir('context')
-
-def prepareDocker = project.tasks.register('prepareDocker', Sync) {
-    // deephaven-jpy.whl
-    def deephavenJpyWheel = project(':deephaven-jpy').tasks.getByName('buildWheel')
-
-    // deephaven.whl
-    def deephavenWheel = project(':deephaven-wheel').tasks.getByName('buildWheel')
-
-    // deephaven2.whl
-    def deephaven2Wheel = project(':deephaven2-wheel').tasks.getByName('buildWheel')
-
-    from (deephavenJpyWheel.outputs.files) {
-        into 'deephaven-jpy-wheel'
-    }
-    from (deephavenWheel.outputs.files) {
-        into 'deephaven-wheel'
-    }
-    from (deephaven2Wheel.outputs.files) {
-        into 'deephaven2-wheel'
-    }
-
-    from 'src/main/docker'
-    into dockerContext
-}
-
-Docker.registerDockerImage(project, 'buildDeephavenPython') {
-    // deephaven/java-and-python:local-build
-    def javaAndPython = project(':docker-java-and-python').tasks.getByName('buildDocker')
-
-    dependsOn prepareDocker
-
-    inputs.files javaAndPython.outputs.files
-
-    inputDir.set dockerContext
-
-    images.add('deephaven/runtime-base:local-build')
-}
-
 JavaPluginConvention java = project.convention.plugins.get('java') as JavaPluginConvention
 SourceSet test = java.sourceSets.maybeCreate('test')
 
@@ -125,7 +81,7 @@ def runInDocker = { String name, String sourcePath, List<String> command ->
                 into 'python/configs'
             }
         }
-        parentContainers = [tasks.findByName('buildDeephavenPython')] // deephaven/runtime-base
+        parentContainers = [project(':docker-runtime-base').tasks.findByName('buildDocker')] // deephaven/runtime-base
 
         imageName = 'deephaven/py-integrations:local-build'
 

build.gradle

@@ -97,7 +97,7 @@ ext.globalVersion = "0.7.0"
 // If you have fishlib open in IDE, also set includeFish=true to get the
 // intellij projects wired up across repositories.
 
-Set<Project> modsAreBasic = subprojects.findAll {it.name in ['deephaven-wheel', 'deephaven2-wheel', 'envoy', 'grpc-proxy', 'web-client-ui', 'protoc', 'pyclient', 'sphinx', 'docker-java-and-python'] }
+Set<Project> modsAreBasic = subprojects.findAll {it.name in ['deephaven-wheel', 'deephaven2-wheel', 'envoy', 'grpc-proxy', 'web-client-ui', 'protoc', 'pyclient', 'sphinx', 'docker-java-and-python', 'docker-runtime-base'] }
 
 Set<Project> modsAreBin = subprojects.findAll {it.name in ['bin', 'configs', 'test-configs'] }
 

docker/java-and-python/src/main/docker/Dockerfile

@@ -23,7 +23,7 @@ RUN set -eux; \
 ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8'
 ENV JAVA_HOME=/usr/lib/jvm/zulu11-ca-amd64
 
-FROM java_base
+FROM java_base as python_install
 RUN set -eux; \
     apt-get -qq update; \
     apt-get -qq -y --no-install-recommends install \
@@ -33,7 +33,27 @@ RUN set -eux; \
       python3-setuptools \
       python3-wheel \
       ; \
-    rm -rf /var/lib/apt/lists/*; \
-    python3 -m pip install -q --no-cache-dir --upgrade pip; \
-    python3 -m pip install -q --no-cache-dir --upgrade setuptools; \
-    python3 -m pip install -q --no-cache-dir --upgrade wheel
+    rm -rf /var/lib/apt/lists/*
+
+# To update the requirements.txt:
+# 1. Comment out BLOCK 1, and uncomment BLOCK 2
+# 2. Run `./gradlew docker-java-and-python:build`
+# 3. Run `docker run --rm -it deephaven/java-and-python:local-build pip freeze --all > docker/java-and-python/src/main/docker/requirements.txt`
+# 4. Uncomment BLOCK 1, and comment out BLOCK 2
+
+# START BLOCK 1
+COPY requirements.txt requirements.txt
+RUN set -eux; \
+    python3 -m pip install --no-cache-dir -r requirements.txt; \
+    rm requirements.txt
+# END BLOCK 1
+
+# START BLOCK 2
+#RUN set -eux; \
+#    python3 -m pip install --no-cache-dir --upgrade pip; \
+#    python3 -m pip install --no-cache-dir --upgrade setuptools; \
+#    python3 -m pip install --no-cache-dir --upgrade wheel
+# END BLOCK 2
+
+FROM java_base
+COPY --from=python_install / /

docker/java-and-python/src/main/docker/requirements.txt

@@ -0,0 +1,3 @@
+pip==21.3.1
+setuptools==59.0.1
+wheel==0.37.0

docker/runtime-base/build.gradle

@@ -0,0 +1,49 @@
+plugins {
+    id 'com.bmuschko.docker-remote-api'
+}
+
+evaluationDependsOn ':docker-java-and-python'
+evaluationDependsOn ':deephaven-jpy'
+evaluationDependsOn ':deephaven-wheel'
+evaluationDependsOn ':deephaven2-wheel'
+
+def dockerContext = project.layout.buildDirectory.dir('context')
+
+def prepareDocker = project.tasks.register('prepareDocker', Sync) {
+    // deephaven-jpy.whl
+    def deephavenJpyWheel = project(':deephaven-jpy').tasks.getByName('buildWheel')
+
+    // deephaven.whl
+    def deephavenWheel = project(':deephaven-wheel').tasks.getByName('buildWheel')
+
+    // deephaven2.whl
+    def deephaven2Wheel = project(':deephaven2-wheel').tasks.getByName('buildWheel')
+
+    from (deephavenJpyWheel.outputs.files) {
+        into 'deephaven-jpy-wheel'
+    }
+    from (deephavenWheel.outputs.files) {
+        into 'deephaven-wheel'
+    }
+    from (deephaven2Wheel.outputs.files) {
+        into 'deephaven2-wheel'
+    }
+
+    from 'src/main/docker'
+    into dockerContext
+}
+
+Docker.registerDockerImage(project, 'buildDocker') {
+    // deephaven/java-and-python:local-build
+    def javaAndPython = project(':docker-java-and-python').tasks.getByName('buildDocker')
+
+    dependsOn prepareDocker
+
+    inputs.files javaAndPython.outputs.files
+
+    inputDir.set dockerContext
+
+    images.add('deephaven/runtime-base:local-build')
+}
+
+assemble.dependsOn buildDocker

docker/runtime-base/src/main/docker/Dockerfile

@@ -0,0 +1,39 @@
+FROM deephaven/java-and-python:local-build as install-wheels
+
+RUN set -eux; \
+    apt-get -qq update; \
+    apt-get -qq -y --no-install-recommends install liblzo2-2 curl; \
+    rm -rf /var/lib/apt/lists/*
+
+# To update the requirements.txt:
+# 1. Comment out BLOCK 1
+# 2. Remove the --no-index arguments from the RUNs
+# 3. Run `./gradlew docker-runtime-base:build`
+# 4. Run `docker run --rm -it deephaven/runtime-base:local-build pip freeze | grep -v " @ file" > docker/runtime-base/src/main/docker/requirements.txt`
+# 5. Replace the --no-index arguments from the RUNs
+# 6. Uncomment BLOCK 1
+
+# START BLOCK 1
+COPY requirements.txt requirements.txt
+RUN set -eux; \
+    python3 -m pip install --no-cache-dir -r requirements.txt; \
+    rm requirements.txt
+# END BLOCK 1
+
+COPY deephaven-jpy-wheel/ /deephaven-jpy-wheel
+RUN set -eux; \
+    python3 -m pip install -q --no-index --no-cache-dir /deephaven-jpy-wheel/*.whl; \
+    rm -r /deephaven-jpy-wheel
+
+COPY deephaven-wheel/ /deephaven-wheel
+RUN set -eux; \
+    python3 -m pip install -q --no-index --no-cache-dir /deephaven-wheel/*.whl; \
+    rm -r /deephaven-wheel
+
+COPY deephaven2-wheel/ /deephaven2-wheel
+RUN set -eux; \
+    python3 -m pip install -q --no-index --no-cache-dir /deephaven2-wheel/*.whl; \
+    rm -r /deephaven2-wheel
+
+FROM deephaven/java-and-python:local-build
+COPY --from=install-wheels / /

docker/runtime-base/src/main/docker/requirements.txt

@@ -0,0 +1,9 @@
+dill==0.3.4
+llvmlite==0.37.0
+numba==0.54.1
+numpy==1.20.3
+pandas==1.3.4
+python-dateutil==2.8.2
+pytz==2021.3
+six==1.16.0
+wrapt==1.13.3

grpc-api/server/docker/build.gradle

@@ -5,7 +5,7 @@ plugins {
   id 'com.bmuschko.docker-java-application'
 }
 
-evaluationDependsOn(":Integrations")
+evaluationDependsOn(':docker-runtime-base')
 
 dependencies {
   implementation(project(':grpc-api')) {
@@ -101,7 +101,7 @@ docker {
 
 
 // deephaven/runtime-base
-def grpcApiBase = project(':Integrations').tasks.findByName('buildDeephavenPython')
+def grpcApiBase = project(':docker-runtime-base').tasks.findByName('buildDocker')
 dockerBuildImage.dependsOn grpcApiBase
 dockerBuildImage.inputs.file grpcApiBase.outputs.files.singleFile
 // Currently, GH build-ci.yml calls dockerCreateDockerfile, and relies on buildx; so we need to

py/jpy-integration/build.gradle

@@ -7,7 +7,7 @@ plugins {
 }
 
 evaluationDependsOn ':deephaven-jpy'
-evaluationDependsOn ':Integrations'
+evaluationDependsOn ':docker-runtime-base'
 
 sourceSets {
   // All sourcesets here are used for testing; we'll inform the idea{} plugin, below
@@ -134,7 +134,7 @@ Closure<TaskProvider<Task>> gradleTestInDocker = { String taskName, SourceSet so
         into 'classes'
       }
     }
-    parentContainers = [project(':Integrations').tasks.findByName('buildDeephavenPython')] // deephaven/runtime-base
+    parentContainers = [project(':docker-runtime-base').tasks.findByName('buildDocker')] // deephaven/runtime-base
     dockerfile {
       // base image with default java, python, wheels
       from 'deephaven/runtime-base:local-build'
@@ -166,7 +166,7 @@ Closure<TaskProvider<Task>> javaMainInDocker = { String taskName, String javaMai
         into 'classpath'
       }
     }
-    parentContainers = [project(':Integrations').tasks.findByName('buildDeephavenPython')] // deephaven/runtime-base
+    parentContainers = [project(':docker-runtime-base').tasks.findByName('buildDocker')] // deephaven/runtime-base
     imageName = 'deephaven/jpy-integration-java-to-python-tests:local-build'
     dockerfile {
       from 'deephaven/runtime-base:local-build'
@@ -217,7 +217,7 @@ Closure<TaskProvider<Task>> pyExec = { String taskName, List<String> command, bo
         into 'python'
       }
     }
-    parentContainers = [project(':Integrations').tasks.findByName('buildDeephavenPython')] // deephaven/runtime-base
+    parentContainers = [project(':docker-runtime-base').tasks.findByName('buildDocker')] // deephaven/runtime-base
     imageName = 'deephaven/jpy-integration-python-to-java-tests:local-build'
     dockerfile {
       // set up the container, env vars - things that aren't likely to change

settings.gradle

@@ -186,6 +186,9 @@ project(':deephaven2-wheel').projectDir = file('py/deephaven2-wheel')
 include(':docker-java-and-python')
 project(':docker-java-and-python').projectDir = file('docker/java-and-python')
 
+include(':docker-runtime-base')
+project(':docker-runtime-base').projectDir = file('docker/runtime-base')
+
 // Apply "vanity naming" (look for .gradle files matching ProjectName/ProjectName.gradle)
 File root = settings.rootDir
 mods.each {

sphinx/sphinx.gradle

@@ -6,7 +6,7 @@ plugins {
 
 description = 'Generates docs for the python libraries provided in Deephaven Core'
 
-evaluationDependsOn ':Integrations'
+evaluationDependsOn ':docker-runtime-base'
 evaluationDependsOn ':pyclient'
 
 def copyPyClientWhl = tasks.register('copyPyClientWhl', Sync) {
@@ -37,7 +37,7 @@ def sphinxImage = Docker.registerDockerImage(project, 'sphinx') {
     inputs.files copyPyClientWhl.get().outputs.files
     inputs.files copySphinxLib.get().outputs.files
     inputDir.set layout.buildDirectory.dir('sphinx-image')
-    inputs.files project(':Integrations').tasks.findByName('buildDeephavenPython').outputs.files // deephaven/runtime-base
+    inputs.files project(':docker-runtime-base').tasks.findByName('buildDocker').outputs.files // deephaven/runtime-base
     images.add('deephaven/sphinx:local-build')
 }
 
@@ -101,7 +101,7 @@ def cppClientDoxygenTask = Docker.registerDockerTask(project, 'cppClientDoxygen'
                       doxygen
                       ''')
     }
-    parentContainers = [project(':Integrations').tasks.findByName('buildDeephavenPython')] // deephaven/runtime-base
+    parentContainers = [project(':docker-runtime-base').tasks.findByName('buildDocker')] // deephaven/runtime-base
     containerOutPath = '/project/doc/doxygenoutput'
     copyOut {
         into "$buildDir/cppClientDoxygen"