diff --git a/DB-test/build.gradle b/DB-test/build.gradle index b2d596b996f..ae9fa2ecaf3 100644 --- a/DB-test/build.gradle +++ b/DB-test/build.gradle @@ -2,7 +2,7 @@ plugins { id 'com.bmuschko.docker-remote-api' } -evaluationDependsOn ':Integrations' // runtimeBase +evaluationDependsOn ':docker-runtime-base' // runtimeBase configurations { //compile.extendsFrom dhDb @@ -58,7 +58,7 @@ tasks.getByName('check').dependsOn Docker.registerDockerTask(project, 'test-in-d into 'classes' } } - parentContainers = [project(':Integrations').tasks.findByName('buildDeephavenPython')] // deephaven/runtime-base + parentContainers = [project(':docker-runtime-base').tasks.findByName('buildDocker')] // deephaven/runtime-base dockerfile { // base image with default java, python, wheels from 'deephaven/runtime-base:local-build' diff --git a/Integrations/build.gradle b/Integrations/build.gradle index 730bf20ea55..9bfe6dd995b 100644 --- a/Integrations/build.gradle +++ b/Integrations/build.gradle @@ -1,14 +1,9 @@ -import com.bmuschko.gradle.docker.tasks.image.Dockerfile - plugins { id 'com.bmuschko.docker-remote-api' id 'com.avast.gradle.docker-compose' } -evaluationDependsOn ':docker-java-and-python' -evaluationDependsOn ':deephaven-jpy' -evaluationDependsOn ':deephaven-wheel' -evaluationDependsOn ':deephaven2-wheel' +evaluationDependsOn ':docker-runtime-base' configurations { compile.extendsFrom dhIntegrations @@ -66,45 +61,6 @@ idea { } } -def dockerContext = project.layout.buildDirectory.dir('context') - -def prepareDocker = project.tasks.register('prepareDocker', Sync) { - // deephaven-jpy.whl - def deephavenJpyWheel = project(':deephaven-jpy').tasks.getByName('buildWheel') - - // deephaven.whl - def deephavenWheel = project(':deephaven-wheel').tasks.getByName('buildWheel') - - // deephaven2.whl - def deephaven2Wheel = project(':deephaven2-wheel').tasks.getByName('buildWheel') - - from (deephavenJpyWheel.outputs.files) { - into 'deephaven-jpy-wheel' - } - from (deephavenWheel.outputs.files) { - into 'deephaven-wheel' - } - from (deephaven2Wheel.outputs.files) { - into 'deephaven2-wheel' - } - - from 'src/main/docker' - into dockerContext -} - -Docker.registerDockerImage(project, 'buildDeephavenPython') { - // deephaven/java-and-python:local-build - def javaAndPython = project(':docker-java-and-python').tasks.getByName('buildDocker') - - dependsOn prepareDocker - - inputs.files javaAndPython.outputs.files - - inputDir.set dockerContext - - images.add('deephaven/runtime-base:local-build') -} - JavaPluginConvention java = project.convention.plugins.get('java') as JavaPluginConvention SourceSet test = java.sourceSets.maybeCreate('test') @@ -125,7 +81,7 @@ def runInDocker = { String name, String sourcePath, List command -> into 'python/configs' } } - parentContainers = [tasks.findByName('buildDeephavenPython')] // deephaven/runtime-base + parentContainers = [project(':docker-runtime-base').tasks.findByName('buildDocker')] // deephaven/runtime-base imageName = 'deephaven/py-integrations:local-build' diff --git a/Integrations/src/main/docker/Dockerfile b/Integrations/src/main/docker/Dockerfile deleted file mode 100644 index 23499d77aaa..00000000000 --- a/Integrations/src/main/docker/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -FROM deephaven/java-and-python:local-build as install-wheels - -RUN set -eux; \ - apt-get -qq update; \ - apt-get -qq -y --no-install-recommends install liblzo2-2 curl; \ - rm -rf /var/lib/apt/lists/* - -COPY deephaven-jpy-wheel/ /deephaven-jpy-wheel -RUN set -eux; \ - python3 -m pip install -q --no-cache-dir /deephaven-jpy-wheel/*.whl; \ - rm -r /deephaven-jpy-wheel - -COPY deephaven-wheel/ /deephaven-wheel -RUN set -eux; \ - python3 -m pip install -q --no-cache-dir /deephaven-wheel/*.whl; \ - rm -r /deephaven-wheel - -COPY deephaven2-wheel/ /deephaven2-wheel -RUN set -eux; \ - python3 -m pip install -q --no-cache-dir /deephaven2-wheel/*.whl; \ - rm -r /deephaven2-wheel - -FROM deephaven/java-and-python:local-build -COPY --from=install-wheels / / \ No newline at end of file diff --git a/build.gradle b/build.gradle index 853e89b358f..dd0e76b60e6 100644 --- a/build.gradle +++ b/build.gradle @@ -97,7 +97,7 @@ ext.globalVersion = "0.7.0" // If you have fishlib open in IDE, also set includeFish=true to get the // intellij projects wired up across repositories. -Set modsAreBasic = subprojects.findAll {it.name in ['deephaven-wheel', 'deephaven2-wheel', 'envoy', 'grpc-proxy', 'web-client-ui', 'protoc', 'pyclient', 'sphinx', 'docker-java-and-python'] } +Set modsAreBasic = subprojects.findAll {it.name in ['deephaven-wheel', 'deephaven2-wheel', 'envoy', 'grpc-proxy', 'web-client-ui', 'protoc', 'pyclient', 'sphinx', 'docker-java-and-python', 'docker-runtime-base'] } Set modsAreBin = subprojects.findAll {it.name in ['bin', 'configs', 'test-configs'] } diff --git a/docker/java-and-python/src/main/docker/Dockerfile b/docker/java-and-python/src/main/docker/Dockerfile index 468f8dd84f2..ec1dcca90b7 100644 --- a/docker/java-and-python/src/main/docker/Dockerfile +++ b/docker/java-and-python/src/main/docker/Dockerfile @@ -23,7 +23,7 @@ RUN set -eux; \ ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8' ENV JAVA_HOME=/usr/lib/jvm/zulu11-ca-amd64 -FROM java_base +FROM java_base as python_install RUN set -eux; \ apt-get -qq update; \ apt-get -qq -y --no-install-recommends install \ @@ -33,7 +33,27 @@ RUN set -eux; \ python3-setuptools \ python3-wheel \ ; \ - rm -rf /var/lib/apt/lists/*; \ - python3 -m pip install -q --no-cache-dir --upgrade pip; \ - python3 -m pip install -q --no-cache-dir --upgrade setuptools; \ - python3 -m pip install -q --no-cache-dir --upgrade wheel + rm -rf /var/lib/apt/lists/* + +# To update the requirements.txt: +# 1. Comment out BLOCK 1, and uncomment BLOCK 2 +# 2. Run `./gradlew docker-java-and-python:build` +# 3. Run `docker run --rm -it deephaven/java-and-python:local-build pip freeze --all > docker/java-and-python/src/main/docker/requirements.txt` +# 4. Uncomment BLOCK 1, and comment out BLOCK 2 + +# START BLOCK 1 +COPY requirements.txt requirements.txt +RUN set -eux; \ + python3 -m pip install --no-cache-dir -r requirements.txt; \ + rm requirements.txt +# END BLOCK 1 + +# START BLOCK 2 +#RUN set -eux; \ +# python3 -m pip install --no-cache-dir --upgrade pip; \ +# python3 -m pip install --no-cache-dir --upgrade setuptools; \ +# python3 -m pip install --no-cache-dir --upgrade wheel +# END BLOCK 2 + +FROM java_base +COPY --from=python_install / / \ No newline at end of file diff --git a/docker/java-and-python/src/main/docker/requirements.txt b/docker/java-and-python/src/main/docker/requirements.txt new file mode 100644 index 00000000000..51f0125d2b1 --- /dev/null +++ b/docker/java-and-python/src/main/docker/requirements.txt @@ -0,0 +1,3 @@ +pip==21.3.1 +setuptools==59.0.1 +wheel==0.37.0 diff --git a/docker/runtime-base/build.gradle b/docker/runtime-base/build.gradle new file mode 100644 index 00000000000..99cc61f8077 --- /dev/null +++ b/docker/runtime-base/build.gradle @@ -0,0 +1,49 @@ +plugins { + id 'com.bmuschko.docker-remote-api' +} + +evaluationDependsOn ':docker-java-and-python' +evaluationDependsOn ':deephaven-jpy' +evaluationDependsOn ':deephaven-wheel' +evaluationDependsOn ':deephaven2-wheel' + +def dockerContext = project.layout.buildDirectory.dir('context') + +def prepareDocker = project.tasks.register('prepareDocker', Sync) { + // deephaven-jpy.whl + def deephavenJpyWheel = project(':deephaven-jpy').tasks.getByName('buildWheel') + + // deephaven.whl + def deephavenWheel = project(':deephaven-wheel').tasks.getByName('buildWheel') + + // deephaven2.whl + def deephaven2Wheel = project(':deephaven2-wheel').tasks.getByName('buildWheel') + + from (deephavenJpyWheel.outputs.files) { + into 'deephaven-jpy-wheel' + } + from (deephavenWheel.outputs.files) { + into 'deephaven-wheel' + } + from (deephaven2Wheel.outputs.files) { + into 'deephaven2-wheel' + } + + from 'src/main/docker' + into dockerContext +} + +Docker.registerDockerImage(project, 'buildDocker') { + // deephaven/java-and-python:local-build + def javaAndPython = project(':docker-java-and-python').tasks.getByName('buildDocker') + + dependsOn prepareDocker + + inputs.files javaAndPython.outputs.files + + inputDir.set dockerContext + + images.add('deephaven/runtime-base:local-build') +} + +assemble.dependsOn buildDocker \ No newline at end of file diff --git a/docker/runtime-base/src/main/docker/Dockerfile b/docker/runtime-base/src/main/docker/Dockerfile new file mode 100644 index 00000000000..46f12bd6016 --- /dev/null +++ b/docker/runtime-base/src/main/docker/Dockerfile @@ -0,0 +1,39 @@ +FROM deephaven/java-and-python:local-build as install-wheels + +RUN set -eux; \ + apt-get -qq update; \ + apt-get -qq -y --no-install-recommends install liblzo2-2 curl; \ + rm -rf /var/lib/apt/lists/* + +# To update the requirements.txt: +# 1. Comment out BLOCK 1 +# 2. Remove the --no-index arguments from the RUNs +# 3. Run `./gradlew docker-runtime-base:build` +# 4. Run `docker run --rm -it deephaven/runtime-base:local-build pip freeze | grep -v " @ file" > docker/runtime-base/src/main/docker/requirements.txt` +# 5. Replace the --no-index arguments from the RUNs +# 6. Uncomment BLOCK 1 + +# START BLOCK 1 +COPY requirements.txt requirements.txt +RUN set -eux; \ + python3 -m pip install --no-cache-dir -r requirements.txt; \ + rm requirements.txt +# END BLOCK 1 + +COPY deephaven-jpy-wheel/ /deephaven-jpy-wheel +RUN set -eux; \ + python3 -m pip install -q --no-index --no-cache-dir /deephaven-jpy-wheel/*.whl; \ + rm -r /deephaven-jpy-wheel + +COPY deephaven-wheel/ /deephaven-wheel +RUN set -eux; \ + python3 -m pip install -q --no-index --no-cache-dir /deephaven-wheel/*.whl; \ + rm -r /deephaven-wheel + +COPY deephaven2-wheel/ /deephaven2-wheel +RUN set -eux; \ + python3 -m pip install -q --no-index --no-cache-dir /deephaven2-wheel/*.whl; \ + rm -r /deephaven2-wheel + +FROM deephaven/java-and-python:local-build +COPY --from=install-wheels / / \ No newline at end of file diff --git a/docker/runtime-base/src/main/docker/requirements.txt b/docker/runtime-base/src/main/docker/requirements.txt new file mode 100644 index 00000000000..8db84f4d2bc --- /dev/null +++ b/docker/runtime-base/src/main/docker/requirements.txt @@ -0,0 +1,9 @@ +dill==0.3.4 +llvmlite==0.37.0 +numba==0.54.1 +numpy==1.20.3 +pandas==1.3.4 +python-dateutil==2.8.2 +pytz==2021.3 +six==1.16.0 +wrapt==1.13.3 diff --git a/grpc-api/server/docker/build.gradle b/grpc-api/server/docker/build.gradle index dc7bc3e329d..557a77cd308 100644 --- a/grpc-api/server/docker/build.gradle +++ b/grpc-api/server/docker/build.gradle @@ -5,7 +5,7 @@ plugins { id 'com.bmuschko.docker-java-application' } -evaluationDependsOn(":Integrations") +evaluationDependsOn(':docker-runtime-base') dependencies { implementation(project(':grpc-api')) { @@ -101,7 +101,7 @@ docker { // deephaven/runtime-base -def grpcApiBase = project(':Integrations').tasks.findByName('buildDeephavenPython') +def grpcApiBase = project(':docker-runtime-base').tasks.findByName('buildDocker') dockerBuildImage.dependsOn grpcApiBase dockerBuildImage.inputs.file grpcApiBase.outputs.files.singleFile // Currently, GH build-ci.yml calls dockerCreateDockerfile, and relies on buildx; so we need to diff --git a/py/jpy-integration/build.gradle b/py/jpy-integration/build.gradle index b74e4c124fd..21bd359d5aa 100644 --- a/py/jpy-integration/build.gradle +++ b/py/jpy-integration/build.gradle @@ -7,7 +7,7 @@ plugins { } evaluationDependsOn ':deephaven-jpy' -evaluationDependsOn ':Integrations' +evaluationDependsOn ':docker-runtime-base' sourceSets { // All sourcesets here are used for testing; we'll inform the idea{} plugin, below @@ -134,7 +134,7 @@ Closure> gradleTestInDocker = { String taskName, SourceSet so into 'classes' } } - parentContainers = [project(':Integrations').tasks.findByName('buildDeephavenPython')] // deephaven/runtime-base + parentContainers = [project(':docker-runtime-base').tasks.findByName('buildDocker')] // deephaven/runtime-base dockerfile { // base image with default java, python, wheels from 'deephaven/runtime-base:local-build' @@ -166,7 +166,7 @@ Closure> javaMainInDocker = { String taskName, String javaMai into 'classpath' } } - parentContainers = [project(':Integrations').tasks.findByName('buildDeephavenPython')] // deephaven/runtime-base + parentContainers = [project(':docker-runtime-base').tasks.findByName('buildDocker')] // deephaven/runtime-base imageName = 'deephaven/jpy-integration-java-to-python-tests:local-build' dockerfile { from 'deephaven/runtime-base:local-build' @@ -217,7 +217,7 @@ Closure> pyExec = { String taskName, List command, bo into 'python' } } - parentContainers = [project(':Integrations').tasks.findByName('buildDeephavenPython')] // deephaven/runtime-base + parentContainers = [project(':docker-runtime-base').tasks.findByName('buildDocker')] // deephaven/runtime-base imageName = 'deephaven/jpy-integration-python-to-java-tests:local-build' dockerfile { // set up the container, env vars - things that aren't likely to change diff --git a/settings.gradle b/settings.gradle index d7c921f2aa7..7afde85e88b 100644 --- a/settings.gradle +++ b/settings.gradle @@ -186,6 +186,9 @@ project(':deephaven2-wheel').projectDir = file('py/deephaven2-wheel') include(':docker-java-and-python') project(':docker-java-and-python').projectDir = file('docker/java-and-python') +include(':docker-runtime-base') +project(':docker-runtime-base').projectDir = file('docker/runtime-base') + // Apply "vanity naming" (look for .gradle files matching ProjectName/ProjectName.gradle) File root = settings.rootDir mods.each { diff --git a/sphinx/sphinx.gradle b/sphinx/sphinx.gradle index 863c867b61a..b16bdc03e82 100644 --- a/sphinx/sphinx.gradle +++ b/sphinx/sphinx.gradle @@ -6,7 +6,7 @@ plugins { description = 'Generates docs for the python libraries provided in Deephaven Core' -evaluationDependsOn ':Integrations' +evaluationDependsOn ':docker-runtime-base' evaluationDependsOn ':pyclient' def copyPyClientWhl = tasks.register('copyPyClientWhl', Sync) { @@ -37,7 +37,7 @@ def sphinxImage = Docker.registerDockerImage(project, 'sphinx') { inputs.files copyPyClientWhl.get().outputs.files inputs.files copySphinxLib.get().outputs.files inputDir.set layout.buildDirectory.dir('sphinx-image') - inputs.files project(':Integrations').tasks.findByName('buildDeephavenPython').outputs.files // deephaven/runtime-base + inputs.files project(':docker-runtime-base').tasks.findByName('buildDocker').outputs.files // deephaven/runtime-base images.add('deephaven/sphinx:local-build') } @@ -101,7 +101,7 @@ def cppClientDoxygenTask = Docker.registerDockerTask(project, 'cppClientDoxygen' doxygen ''') } - parentContainers = [project(':Integrations').tasks.findByName('buildDeephavenPython')] // deephaven/runtime-base + parentContainers = [project(':docker-runtime-base').tasks.findByName('buildDocker')] // deephaven/runtime-base containerOutPath = '/project/doc/doxygenoutput' copyOut { into "$buildDir/cppClientDoxygen"