diff --git a/.github/workflows/build-ci.yml b/.github/workflows/build-ci.yml index f8e76b11d07..d2dffbe6b4e 100644 --- a/.github/workflows/build-ci.yml +++ b/.github/workflows/build-ci.yml @@ -483,7 +483,7 @@ jobs: uses: burrunan/gradle-cache-action@v1 with: job-id: build-web - arguments: --scan outputVersion createCraneTagScript web-client-ide:prepareDocker + arguments: --scan outputVersion createCraneTagScript docker-web:prepareDocker gradle-version: wrapper - name: Get Deephaven Version @@ -498,7 +498,7 @@ jobs: DEEPHAVEN_VERSION=${{ steps.deephaven_version.outputs.deephaven_version }} tags: ${{ steps.docker_meta.outputs.tags }} builder: ${{ steps.buildx.outputs.name }} - context: ./web/client-ide/build/docker/ + context: ./docker/web/build/docker/ push: ${{ github.event_name != 'pull_request' }} - name: Tag upstream images diff --git a/ClientSupport/src/main/java/io/deephaven/clientsupport/plotdownsampling/RunChartDownsample.java b/ClientSupport/src/main/java/io/deephaven/clientsupport/plotdownsampling/RunChartDownsample.java index eeaf22e57f8..705051be608 100644 --- a/ClientSupport/src/main/java/io/deephaven/clientsupport/plotdownsampling/RunChartDownsample.java +++ b/ClientSupport/src/main/java/io/deephaven/clientsupport/plotdownsampling/RunChartDownsample.java @@ -7,6 +7,7 @@ import io.deephaven.engine.table.TableUpdate; import io.deephaven.engine.table.impl.TableUpdateImpl; import io.deephaven.engine.rowset.chunkattributes.OrderedRowKeys; +import io.deephaven.internal.log.LoggerFactory; import io.deephaven.time.DateTime; import io.deephaven.hash.KeyedLongObjectHash; import io.deephaven.hash.KeyedLongObjectHashMap; @@ -14,7 +15,6 @@ import io.deephaven.base.verify.Assert; import io.deephaven.configuration.Configuration; import io.deephaven.io.logger.Logger; -import io.deephaven.util.process.ProcessEnvironment; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.impl.perf.QueryPerformanceRecorder; import io.deephaven.engine.table.impl.*; @@ -50,7 +50,7 @@ * make nulls result in fewer items in the result table */ public class RunChartDownsample implements Function { - private static final Logger log = ProcessEnvironment.getDefaultLog(RunChartDownsample.class); + private static final Logger log = LoggerFactory.getLogger(RunChartDownsample.class); public static final int CHUNK_SIZE = Configuration.getInstance().getIntegerWithDefault("chunkSize", 1 << 14); diff --git a/FishUtil/src/main/java/io/deephaven/util/process/BaseProcessEnvironment.java b/FishUtil/src/main/java/io/deephaven/util/process/BaseProcessEnvironment.java index 36166f1b448..5bc29ca99cc 100644 --- a/FishUtil/src/main/java/io/deephaven/util/process/BaseProcessEnvironment.java +++ b/FishUtil/src/main/java/io/deephaven/util/process/BaseProcessEnvironment.java @@ -63,14 +63,4 @@ public final FatalErrorReporter getFatalErrorReporter() { public final String getMainClassName() { return mainClassName; } - - @Override - public final Logger getLog() { - if (!Boolean.getBoolean("LoggerFactory.silenceOnProcessEnvironment")) { - log.warn(new RuntimeException("Trace")) - .append("Logger being fetched via ProcessEnvironment instead of io.deephaven.internal.log.LoggerFactory") - .endl(); - } - return log; - } } diff --git a/FishUtil/src/main/java/io/deephaven/util/process/ProcessEnvironment.java b/FishUtil/src/main/java/io/deephaven/util/process/ProcessEnvironment.java index fdf921a053c..0f171f84f6d 100644 --- a/FishUtil/src/main/java/io/deephaven/util/process/ProcessEnvironment.java +++ b/FishUtil/src/main/java/io/deephaven/util/process/ProcessEnvironment.java @@ -8,8 +8,6 @@ import io.deephaven.configuration.Configuration; import io.deephaven.configuration.PropertyException; import io.deephaven.io.logger.Logger; -import io.deephaven.io.logger.StreamLoggerImpl; -import io.deephaven.internal.log.LoggerFactory; import org.jetbrains.annotations.NotNull; /** @@ -42,13 +40,6 @@ public interface ProcessEnvironment { */ String getMainClassName(); - /** - * Access a shared Logger. - * - * @return The Logger - */ - Logger getLog(); - /** * Hook for setting up an installation-specific environment for a given process. */ @@ -126,32 +117,6 @@ static String getGlobalMainClassName() { return get().getMainClassName(); } - static Logger getGlobalLog() { - return get().getLog(); - } - - /** - * Get the global log if a global process environment has been installed, or else a logger that will output to - * {@link System#out}. - * - * @return A logger that can safely be used by code that doesn't otherwise have access to one - */ - static Logger getDefaultLog() { - final ProcessEnvironment processEnvironment = ProcessEnvironment.tryGet(); - return processEnvironment != null ? processEnvironment.getLog() : new StreamLoggerImpl(); - } - - /** - * Get the global log if a global process environment has been installed, or else a - * {@link LoggerFactory#getLogger(Class)} for the given class. - * - * @return A logger that can safely be used by code that doesn't otherwise have access to one - */ - static Logger getDefaultLog(Class clazz) { - final ProcessEnvironment processEnvironment = ProcessEnvironment.tryGet(); - return processEnvironment != null ? processEnvironment.getLog() : LoggerFactory.getLogger(clazz); - } - /** * Setter for the global instance. It is an error to invoke this without allowReplace if the global instance may * already have been set. diff --git a/ModelFarm/src/main/java/io/deephaven/modelfarm/ConditionalModels.java b/ModelFarm/src/main/java/io/deephaven/modelfarm/ConditionalModels.java index acc0585d4d2..436e4d2e864 100644 --- a/ModelFarm/src/main/java/io/deephaven/modelfarm/ConditionalModels.java +++ b/ModelFarm/src/main/java/io/deephaven/modelfarm/ConditionalModels.java @@ -6,7 +6,7 @@ import io.deephaven.base.verify.Require; import io.deephaven.configuration.Configuration; -import io.deephaven.util.process.ProcessEnvironment; +import io.deephaven.internal.log.LoggerFactory; import io.deephaven.modelfarm.ModelMultiExec.Models; import java.util.ArrayList; @@ -26,7 +26,7 @@ */ public class ConditionalModels implements Models { - private static final io.deephaven.io.logger.Logger log = ProcessEnvironment.getDefaultLog(ConditionalModels.class); + private static final io.deephaven.io.logger.Logger log = LoggerFactory.getLogger(ConditionalModels.class); private static final boolean LOG_PERF = Configuration.getInstance().getBooleanWithDefault("ModelFarm.logConditionalModelsPerformance", false); diff --git a/ModelFarm/src/main/java/io/deephaven/modelfarm/ModelFarmOnDemand.java b/ModelFarm/src/main/java/io/deephaven/modelfarm/ModelFarmOnDemand.java index 0194ed8b699..5f72b028a62 100644 --- a/ModelFarm/src/main/java/io/deephaven/modelfarm/ModelFarmOnDemand.java +++ b/ModelFarm/src/main/java/io/deephaven/modelfarm/ModelFarmOnDemand.java @@ -9,8 +9,8 @@ import io.deephaven.engine.table.Table; import io.deephaven.engine.updategraph.UpdateGraphProcessor; import io.deephaven.engine.rowset.RowSet; +import io.deephaven.internal.log.LoggerFactory; import io.deephaven.io.logger.Logger; -import io.deephaven.util.process.ProcessEnvironment; import io.deephaven.engine.table.impl.NotificationStepSource; import io.deephaven.util.FunctionalInterfaces; @@ -33,7 +33,7 @@ public class ModelFarmOnDemand DO_LOCKED_FUNCTION = getDoLockedConsumer(GetDataLockType.UGP_READ_LOCK); diff --git a/Net/src/main/java/io/deephaven/net/impl/nio/NIODriver.java b/Net/src/main/java/io/deephaven/net/impl/nio/NIODriver.java index f011cc51ed7..d48ac67c519 100644 --- a/Net/src/main/java/io/deephaven/net/impl/nio/NIODriver.java +++ b/Net/src/main/java/io/deephaven/net/impl/nio/NIODriver.java @@ -18,7 +18,6 @@ import io.deephaven.io.sched.Scheduler; import io.deephaven.io.sched.TimedJob; import io.deephaven.io.sched.YASchedulerImpl; -import io.deephaven.util.process.ProcessEnvironment; public class NIODriver implements Runnable { private static Logger log; @@ -94,14 +93,7 @@ public static String summary() { */ public static void init() { if (!initialized) { - final Logger log; - final ProcessEnvironment processEnvironment = ProcessEnvironment.tryGet(); - if (processEnvironment == null) { - log = LoggerFactory.getLogger(NIODriver.class); - } else { - log = processEnvironment.getLog(); - } - init(log); + init(LoggerFactory.getLogger(NIODriver.class)); } } diff --git a/build.gradle b/build.gradle index 59d2f3abdd0..e3dda0a1805 100644 --- a/build.gradle +++ b/build.gradle @@ -98,6 +98,7 @@ Set modsAreBasic = subprojects.findAll {it.name in [ 'deephaven2-wheel', 'envoy', 'grpc-proxy', + 'web', 'web-client-ui', 'pyclient', 'sphinx', @@ -480,7 +481,7 @@ tasks.register('prepareCompose') { it.description 'A lifecycle task that prepares prerequisites for local docker-compose builds' it.dependsOn project(':docker-server-slim').tasks.findByName('buildDocker'), project(':docker-server').tasks.findByName('buildDocker-server'), - project(':web-client-ide').tasks.findByName('buildDocker'), + project(':docker-web').tasks.findByName('buildDocker'), project(':envoy').tasks.findByName('buildDocker'), project(':grpc-proxy').tasks.findByName('buildDocker') } @@ -496,7 +497,7 @@ tasks.register('smoke') { it.dependsOn project(':grpc-api').tasks.findByName(LifecycleBasePlugin.CHECK_TASK_NAME) it.dependsOn project(':docker-server-slim').tasks.findByName('prepareDocker') it.dependsOn project(':docker-server').tasks.findByName('prepareDocker') - it.dependsOn project(':web-client-ide').tasks.findByName('prepareDocker') + it.dependsOn project(':web').tasks.findByName(LifecycleBasePlugin.ASSEMBLE_TASK_NAME) it.dependsOn project(':Generators').tasks.findByName(LifecycleBasePlugin.CHECK_TASK_NAME) } diff --git a/buildSrc/src/main/groovy/Docker.groovy b/buildSrc/src/main/groovy/Docker.groovy index 7e464f332fb..22c4f38b56f 100644 --- a/buildSrc/src/main/groovy/Docker.groovy +++ b/buildSrc/src/main/groovy/Docker.groovy @@ -12,6 +12,7 @@ import com.bmuschko.gradle.docker.tasks.image.Dockerfile import com.github.dockerjava.api.command.InspectImageResponse import com.github.dockerjava.api.exception.DockerException import groovy.transform.CompileStatic +import io.deephaven.tools.docker.Architecture import org.gradle.api.Action import org.gradle.api.GradleException import org.gradle.api.Project @@ -158,6 +159,11 @@ class Docker { */ Map buildArgs; + /** + * Optional platform + */ + String platform; + /** * Logs are always printed from the build task when it runs, but entrypoint logs are only printed * when it fails. Set this flag to always show logs, even when entrypoint is successful. @@ -270,6 +276,11 @@ class Docker { if (cfg.buildArgs) { buildArgs.putAll(cfg.buildArgs) } + + // the platform, if provided + if (cfg.platform) { + platform.set(cfg.platform) + } } } @@ -426,6 +437,12 @@ class Docker { // Produce a docker image from the copied inputs and provided dockerfile, and tag it TaskProvider makeImage = project.tasks.register(taskName, DockerBuildImage) { buildImage -> action.execute(buildImage) + if (!buildImage.platform.isPresent()) { + def targetArch = Architecture.targetArchitecture(project).toString() + buildImage.platform.set "linux/${targetArch}".toString() + // Use the same environment variable that buildkit uses + buildImage.buildArgs.put('TARGETARCH', targetArch) + } if (buildImage.images) { buildImage.images.get().forEach { String imageName -> validateImageName(imageName) } diff --git a/buildSrc/src/main/groovy/GwtTools.groovy b/buildSrc/src/main/groovy/GwtTools.groovy index 7a8a7115e85..db5fbe69ae6 100644 --- a/buildSrc/src/main/groovy/GwtTools.groovy +++ b/buildSrc/src/main/groovy/GwtTools.groovy @@ -6,10 +6,7 @@ import groovy.transform.CompileStatic import org.gradle.api.Project import org.gradle.api.artifacts.ProjectDependency import org.gradle.api.file.ConfigurableFileCollection -import org.gradle.api.file.CopySpec import org.gradle.api.plugins.JavaPlugin -import org.gradle.api.tasks.TaskOutputs -import org.gradle.api.tasks.bundling.Jar import org.gradle.api.tasks.compile.JavaCompile /** @@ -72,37 +69,6 @@ class GwtTools { gwtc.logger.quiet('Running in gwt dev mode; saving source to {}/dh/src', extras) } - // Add a jar task to create an artifact containing our compiled application - p.tasks.register "jsJar", Jar, { - Jar j -> - j.group = '~Deephaven' - j.description = description - // make a task dependency, AND make sure we rebuild this jar if the gwtc task output files change. - j.inputs.files gwtc.outputs.files - j.from("$gwt.compile.war/dhapi") { - CopySpec c-> - c.exclude '**/extra' // don't put our source files in exported jar, thx - c.into 'dhapi' - } - // let's also pull in the ide client's build output - TaskOutputs uiOutputs = p.project(':web-client-ui').tasks.getByName('ideClient').outputs - j.from(uiOutputs.files) { - CopySpec c -> - c.into 'dhide' - } - - TaskOutputs internalOpenapiOutputs = p.project(':proto:raw-js-openapi').tasks.getByName('webpackSources').outputs - j.from(internalOpenapiOutputs.files) { - CopySpec c -> - c.into 'dhapi' - } - - j.classifier = 'js' - j.doFirst { - j.logger.info "Merging $gwt.compile.war and $uiOutputs into $j.archivePath" - } - } - p.tasks.findByName('gwtCheck')?.enabled = false } diff --git a/buildSrc/src/main/groovy/io/deephaven/tools/docker/Architecture.groovy b/buildSrc/src/main/groovy/io/deephaven/tools/docker/Architecture.groovy new file mode 100644 index 00000000000..0611f3c07d8 --- /dev/null +++ b/buildSrc/src/main/groovy/io/deephaven/tools/docker/Architecture.groovy @@ -0,0 +1,44 @@ +package io.deephaven.tools.docker + +import org.gradle.api.Project +import org.gradle.nativeplatform.platform.internal.DefaultNativePlatform + +enum Architecture { + AMD64("amd64"), + ARM64("arm64"); + + static Architecture targetArchitecture(Project project) { + if (project.hasProperty('docker.targetArch')) { + return fromDockerName(project.property('docker.targetArch') as String) + } else { + return fromHost() + } + } + + static Architecture fromHost() { + String archName = DefaultNativePlatform.host().getArchitecture().getName() + switch (archName) { + case "x86-64": + return AMD64 + case "arm-v8": + return ARM64 + default: + throw new IllegalStateException("Unable to determine proper docker architecture for " + archName) + } + } + + static Architecture fromDockerName(String dockerName) { + values().find { a -> a.dockerName == dockerName } + } + + private final String dockerName + + Architecture(String dockerName) { + this.dockerName = dockerName + } + + @Override + String toString() { + dockerName + } +} diff --git a/docker/server-slim/src/main/docker/image-bootstrap.properties b/docker/server-slim/src/main/docker/image-bootstrap.properties index 6d71923e88d..947975840e8 100644 --- a/docker/server-slim/src/main/docker/image-bootstrap.properties +++ b/docker/server-slim/src/main/docker/image-bootstrap.properties @@ -1,9 +1,6 @@ Configuration.rootFile=grpc-api.prop -io.deephaven.configuration.PropertyInputStreamLoader.override=io.deephaven.configuration.PropertyInputStreamLoaderTraditional -LoggerFactory.silenceOnProcessEnvironment=true workspace=. devroot=. -stdout.toLogBuffer=true -stderr.toLogBuffer=true -deephaven.console.type=groovy \ No newline at end of file +deephaven.console.type=groovy +deephaven.cache.dir=/cache diff --git a/docker/server/build.gradle b/docker/server/build.gradle index aa75e660166..c866e47154a 100644 --- a/docker/server/build.gradle +++ b/docker/server/build.gradle @@ -1,17 +1,26 @@ +import io.deephaven.tools.docker.Architecture import io.deephaven.tools.License plugins { id 'com.bmuschko.docker-remote-api' } -def baseMap = [ - 'server-base': 'server', - 'nltk-base': 'server-nltk', - 'pytorch-base': 'server-pytorch', - 'sklearn-base': 'server-sklearn', - 'tensorflow-base': 'server-tensorflow', +def targetArch = Architecture.targetArchitecture(project) + +def baseMapAmd64 = [ + 'server-base': 'server', + 'nltk-base': 'server-nltk', + 'pytorch-base': 'server-pytorch', + 'sklearn-base': 'server-sklearn', + 'tensorflow-base': 'server-tensorflow', ] +// Only the server image is supported on arm64 +// TODO(deephaven-core#1702): Support arm64 builds for applicable extended images +def baseMapArm64 = baseMapAmd64.findAll { key, value -> value == 'server' } + +def baseMap = targetArch == Architecture.AMD64 ? baseMapAmd64 : baseMapArm64 + baseMap.keySet().each {base -> evaluationDependsOn Docker.registryProject(base) } @@ -49,12 +58,18 @@ def prepareDocker = project.tasks.register('prepareDocker', Sync) { // deephaven2.whl def deephaven2Wheel = project(':deephaven2-wheel').tasks.getByName('buildWheel') - from 'src/main/common' + from 'src/main/docker' baseMap.values().each { server -> from("src/main/${server}") { into server } } + from ('src/main/arch') { + into 'arch' + } + from ('src/main/configure') { + into 'configure' + } from (deephavenJpyWheel.outputs.files) { into 'deephaven-jpy-wheel' } @@ -86,4 +101,3 @@ def buildDockerClosure = { String base, String server -> } assemble.dependsOn baseMap.collect { base, server -> buildDockerClosure(base, server) } - diff --git a/docker/server/src/main/common/image-bootstrap.properties b/docker/server/src/main/common/image-bootstrap.properties deleted file mode 100644 index 88ce35f6c00..00000000000 --- a/docker/server/src/main/common/image-bootstrap.properties +++ /dev/null @@ -1,11 +0,0 @@ -Configuration.rootFile=grpc-api.prop -io.deephaven.configuration.PropertyInputStreamLoader.override=io.deephaven.configuration.PropertyInputStreamLoaderTraditional -LoggerFactory.silenceOnProcessEnvironment=true -workspace=. -devroot=. -stdout.toLogBuffer=true -stderr.toLogBuffer=true - -jpy.jpyLib=/usr/local/lib/python3.7/dist-packages/jpy.cpython-37m-x86_64-linux-gnu.so -jpy.jdlLib=/usr/local/lib/python3.7/dist-packages/jdl.cpython-37m-x86_64-linux-gnu.so -jpy.pythonLib=/usr/lib/x86_64-linux-gnu/libpython3.7m.so.1.0 diff --git a/docker/server/src/main/configure/image-bootstrap.properties b/docker/server/src/main/configure/image-bootstrap.properties new file mode 100644 index 00000000000..039e7d6b754 --- /dev/null +++ b/docker/server/src/main/configure/image-bootstrap.properties @@ -0,0 +1,5 @@ +Configuration.rootFile=grpc-api.prop +workspace=. +devroot=. + +deephaven.cache.dir=/cache diff --git a/docker/server/src/main/configure/properties.py b/docker/server/src/main/configure/properties.py new file mode 100644 index 00000000000..9d13a9fcd1b --- /dev/null +++ b/docker/server/src/main/configure/properties.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 + +import sys +import jpyutil + +properties = { + 'jpy.pythonPrefix': sys.prefix, + 'jpy.programName': sys.executable, + 'jpy.pythonLib': jpyutil._find_python_dll_file(fail=True), + 'jpy.jpyLib': jpyutil._get_module_path('jpy', fail=True), + 'jpy.jdlLib': jpyutil._get_module_path('jdl', fail=True) +} + +for key, value in properties.items(): + print(f'{key}={value}') diff --git a/docker/server/src/main/common/Dockerfile b/docker/server/src/main/docker/Dockerfile similarity index 86% rename from docker/server/src/main/common/Dockerfile rename to docker/server/src/main/docker/Dockerfile index 0ee0d098323..359f5644f20 100644 --- a/docker/server/src/main/common/Dockerfile +++ b/docker/server/src/main/docker/Dockerfile @@ -29,13 +29,19 @@ RUN set -eux; \ rm -r /deephaven2-wheel ARG DEEPHAVEN_VERSION - ADD serverApplicationDist/server-${DEEPHAVEN_VERSION}.tar /opt/deephaven RUN set -eux; \ ln -s /opt/deephaven/server-${DEEPHAVEN_VERSION} /opt/deephaven/server -COPY image-bootstrap.properties image-bootstrap.properties +COPY configure/ /configure + +# TODO(deephaven-core#1703): Native-compatible python startup improvements +RUN set -eux; \ + cp /configure/image-bootstrap.properties image-bootstrap.properties; \ + python3 /configure/properties.py >> image-bootstrap.properties; \ + cat image-bootstrap.properties; \ + rm -r /configure FROM $BASE COPY --from=install / / diff --git a/docker/web/build.gradle b/docker/web/build.gradle new file mode 100644 index 00000000000..c7532fd517a --- /dev/null +++ b/docker/web/build.gradle @@ -0,0 +1,45 @@ +import io.deephaven.tools.License + +plugins { + id 'com.bmuschko.docker-remote-api' +} + +configurations { + js +} +dependencies { + js project(path: ':web', targetConfiguration: 'js') +} + +evaluationDependsOn Docker.registryProject('nginx-base') + +def dockerLicenses = License.createFrom(project).syncDockerLicense() + +def syncDir = layout.buildDirectory.dir('docker') +def prepareDocker = project.tasks.register('prepareDocker', Sync) { + // TODO(deephaven-core#1596): Remove extra dependencies for build-ci.yml + // Currently, GH build-ci.yml calls dockerCreateDockerfile, and relies on buildx; so we need to + // make sure that dockerCreateDockerfile has the proper dependencies. + // If we change how our build works in the future (so that gradle is responsible for pushing the + // image), we can remove this extra dependency. + dependsOn Docker.registryTask(project, 'nginx-base') + + from layout.projectDirectory.dir('src/main/docker') + from(configurations.js) { + into 'static' + } + from (dockerLicenses.get().outputs) { + into 'licenses' + } + into syncDir +} + +Docker.registerDockerImage(project, 'buildDocker') { + inputDir.set syncDir + inputs.files Docker.registryFiles(project, 'nginx-base') + inputs.files prepareDocker.get().outputs.files + buildArgs.put('DEEPHAVEN_VERSION', project.version) + images.add('deephaven/web:local-build') +} + +assemble.dependsOn buildDocker diff --git a/web/client-ide/docker/Dockerfile b/docker/web/src/main/docker/Dockerfile similarity index 52% rename from web/client-ide/docker/Dockerfile rename to docker/web/src/main/docker/Dockerfile index 4afcfbd8f85..2b8a51d12fe 100644 --- a/web/client-ide/docker/Dockerfile +++ b/docker/web/src/main/docker/Dockerfile @@ -1,16 +1,11 @@ -FROM deephaven/dhide:local-build as dhide - -FROM deephaven/js-out:local-build as js-out - FROM deephaven/nginx-base:local-build -COPY --from=dhide /usr/src/app/package/build /usr/share/nginx/html/ide -COPY --from=js-out /usr/src/app/raw-js-openapi/build/js-out /usr/share/nginx/html/jsapi COPY licenses/ / -COPY dhapi /usr/share/nginx/html/jsapi -COPY default.conf /etc/nginx/conf.d/ -COPY nginx.conf /etc/nginx/ -COPY 99-init-notebooks.sh /docker-entrypoint.d +COPY nginx/default.conf /etc/nginx/conf.d/ +COPY nginx/nginx.conf /etc/nginx/ +COPY nginx/99-init-notebooks.sh /docker-entrypoint.d + +COPY static/ /usr/share/nginx/html VOLUME /tmp ARG DEEPHAVEN_VERSION diff --git a/web/client-ide/nginx/99-init-notebooks.sh b/docker/web/src/main/docker/nginx/99-init-notebooks.sh similarity index 100% rename from web/client-ide/nginx/99-init-notebooks.sh rename to docker/web/src/main/docker/nginx/99-init-notebooks.sh diff --git a/web/client-ide/nginx/default.conf b/docker/web/src/main/docker/nginx/default.conf similarity index 100% rename from web/client-ide/nginx/default.conf rename to docker/web/src/main/docker/nginx/default.conf diff --git a/web/client-ide/nginx/nginx.conf b/docker/web/src/main/docker/nginx/nginx.conf similarity index 100% rename from web/client-ide/nginx/nginx.conf rename to docker/web/src/main/docker/nginx/nginx.conf diff --git a/engine/rowset/src/main/java/io/deephaven/engine/rowset/impl/rsp/RspArray.java b/engine/rowset/src/main/java/io/deephaven/engine/rowset/impl/rsp/RspArray.java index 025dc963fb6..5747918f3d4 100644 --- a/engine/rowset/src/main/java/io/deephaven/engine/rowset/impl/rsp/RspArray.java +++ b/engine/rowset/src/main/java/io/deephaven/engine/rowset/impl/rsp/RspArray.java @@ -141,8 +141,9 @@ *
  • Our version of RB Container supports a "shared" boolean flag that is used to implement copy-on-write (COW) * semantics and allow operation results to share containers in COW fashion.
    • *
  • We extended the Container class hierarchy to include specializations for empty, single value, single range, and - * two values containers. These are immutable; empty is used only as a way to return empty results, and are never actual - * stored in the spans array. For details, please see the Container class definition and derived class hierarchy.
    • + * two values containers. These are immutable; empty and singleton are used only as a way to return empty and singleton + * results, and are never actually stored in the spans array. For details, please see the Container class definition and + * derived class hierarchy. * */ public abstract class RspArray extends RefCountedCow { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/SourceTable.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/SourceTable.java index 9c27e2d56ef..6ef3c89a97b 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/SourceTable.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/SourceTable.java @@ -35,12 +35,6 @@ */ public abstract class SourceTable extends RedefinableTable { - /** - * Static log instance for use in trace. - */ - private static final Logger log = ProcessEnvironment.tryGet() == null ? LoggerFactory.getLogger(SourceTable.class) - : ProcessEnvironment.getGlobalLog(); - /** * Component factory. Mostly held for redefinitions. */ diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractConditionFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractConditionFilter.java index 92e5cc01ba1..95d89ad12e7 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractConditionFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractConditionFilter.java @@ -7,6 +7,7 @@ import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.lang.QueryLanguageParser; import io.deephaven.engine.table.lang.QueryScopeParam; +import io.deephaven.internal.log.LoggerFactory; import io.deephaven.time.DateTimeUtils; import io.deephaven.vector.ObjectVector; import io.deephaven.engine.table.lang.QueryLibrary; @@ -14,7 +15,6 @@ import io.deephaven.engine.table.impl.select.python.DeephavenCompatibleFunction; import io.deephaven.engine.rowset.RowSet; import io.deephaven.io.logger.Logger; -import io.deephaven.util.process.ProcessEnvironment; import org.jetbrains.annotations.NotNull; import java.lang.reflect.InvocationTargetException; @@ -27,7 +27,7 @@ import static io.deephaven.engine.table.impl.select.DhFormulaColumn.COLUMN_SUFFIX; public abstract class AbstractConditionFilter extends WhereFilterImpl { - private static final Logger log = ProcessEnvironment.getDefaultLog(AbstractConditionFilter.class); + private static final Logger log = LoggerFactory.getLogger(AbstractConditionFilter.class); final Map outerToInnerNames; final Map innerToOuterNames; @NotNull diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AutoTuningIncrementalReleaseFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AutoTuningIncrementalReleaseFilter.java index ffa6c8aa36b..07003fdfa52 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AutoTuningIncrementalReleaseFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AutoTuningIncrementalReleaseFilter.java @@ -5,10 +5,10 @@ package io.deephaven.engine.table.impl.select; import io.deephaven.engine.updategraph.UpdateGraphProcessor; +import io.deephaven.internal.log.LoggerFactory; import io.deephaven.time.DateTime; import io.deephaven.io.logger.Logger; import io.deephaven.util.clock.RealTimeClock; -import io.deephaven.util.process.ProcessEnvironment; import io.deephaven.time.ClockTimeProvider; import io.deephaven.engine.updategraph.TerminalNotification; import io.deephaven.time.TimeProvider; @@ -96,6 +96,8 @@ * */ public class AutoTuningIncrementalReleaseFilter extends BaseIncrementalReleaseFilter { + private static final Logger log = LoggerFactory.getLogger(AutoTuningIncrementalReleaseFilter.class); + @NotNull private final TimeProvider timeProvider; private final long initialRelease; @@ -190,7 +192,7 @@ private static ClockTimeProvider getRealTimeProvider() { @ScriptApi public AutoTuningIncrementalReleaseFilter(long initialSize, long initialRelease, double targetFactor, boolean verbose, TimeProvider timeProvider) { - this(ProcessEnvironment.getDefaultLog(), initialSize, initialRelease, targetFactor, verbose, timeProvider); + this(log, initialSize, initialRelease, targetFactor, verbose, timeProvider); } /** diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java index 4db5f823811..cc4fa45866a 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterFactory.java @@ -6,10 +6,10 @@ import io.deephaven.base.Pair; import io.deephaven.engine.table.lang.QueryScope; +import io.deephaven.internal.log.LoggerFactory; import io.deephaven.time.DateTime; import io.deephaven.time.DateTimeUtils; import io.deephaven.io.logger.Logger; -import io.deephaven.util.process.ProcessEnvironment; import io.deephaven.engine.table.Table; import io.deephaven.engine.util.AbstractExpressionFactory; import io.deephaven.engine.util.ExpressionParser; @@ -37,7 +37,7 @@ */ public class WhereFilterFactory { - private static final Logger log = ProcessEnvironment.getDefaultLog(WhereFilterFactory.class); + private static final Logger log = LoggerFactory.getLogger(WhereFilterFactory.class); private static final ExpressionParser parser = new ExpressionParser<>(); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/util/RowSetShiftDataExpander.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/util/RowSetShiftDataExpander.java index 11cedbd0988..0dd6635f74a 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/util/RowSetShiftDataExpander.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/util/RowSetShiftDataExpander.java @@ -10,10 +10,10 @@ import io.deephaven.engine.table.impl.BaseTable; import io.deephaven.engine.table.TableUpdateListener; import io.deephaven.engine.table.ModifiedColumnSet; +import io.deephaven.internal.log.LoggerFactory; import io.deephaven.io.log.impl.LogOutputStringImpl; import io.deephaven.io.logger.Logger; import io.deephaven.util.SafeCloseable; -import io.deephaven.util.process.ProcessEnvironment; import java.io.ByteArrayOutputStream; import java.io.ObjectOutputStream; @@ -27,6 +27,8 @@ */ public class RowSetShiftDataExpander implements SafeCloseable { + private static final Logger log = LoggerFactory.getLogger(RowSetShiftDataExpander.class); + private final RowSet added; private final RowSet removed; private final WritableRowSet modified; @@ -224,7 +226,6 @@ public void validate(final TableUpdate update, final TrackingRowSet sourceRowSet .append(LogOutput::nl).append("\tremovedIntersectCurrent=").append(removedIntersectCurrent) .append(LogOutput::nl).append("\t modifiedMinusCurrent=").append(modifiedMinusCurrent).toString(); - final Logger log = ProcessEnvironment.getDefaultLog(); log.error().append(indexUpdateErrorMessage).endl(); if (serializedIndices != null) { diff --git a/engine/table/src/main/java/io/deephaven/engine/util/AbstractScriptSession.java b/engine/table/src/main/java/io/deephaven/engine/util/AbstractScriptSession.java index a675ecc2eff..d3264ed00ae 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/AbstractScriptSession.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/AbstractScriptSession.java @@ -9,7 +9,6 @@ import io.deephaven.api.util.NameValidator; import io.deephaven.base.FileUtils; import io.deephaven.compilertools.CompilerTools; -import io.deephaven.configuration.Configuration; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.lang.QueryLibrary; @@ -31,11 +30,11 @@ * evaluateScript which handles liveness and diffs in a consistent way. */ public abstract class AbstractScriptSession extends LivenessScope implements ScriptSession, VariableProvider { - public static final String CLASS_CACHE_LOCATION = Configuration.getInstance() - .getStringWithDefault("ScriptSession.classCacheDirectory", "/tmp/dh_class_cache"); + + private static final Path CLASS_CACHE_LOCATION = CacheDir.get().resolve("script-session-classes"); public static void createScriptCache() { - final File classCacheDirectory = new File(CLASS_CACHE_LOCATION); + final File classCacheDirectory = CLASS_CACHE_LOCATION.toFile(); createOrClearDirectory(classCacheDirectory); } @@ -60,8 +59,9 @@ private static void createOrClearDirectory(final File directory) { protected AbstractScriptSession(@Nullable Listener changeListener, boolean isDefaultScriptSession) { this.changeListener = changeListener; + // TODO(deephaven-core#1713): Introduce instance-id concept final UUID scriptCacheId = UuidCreator.getRandomBased(); - classCacheDirectory = new File(CLASS_CACHE_LOCATION, UuidCreator.toString(scriptCacheId)); + classCacheDirectory = CLASS_CACHE_LOCATION.resolve(UuidCreator.toString(scriptCacheId)).toFile(); createOrClearDirectory(classCacheDirectory); queryScope = newQueryScope(); diff --git a/engine/table/src/main/java/io/deephaven/engine/util/CacheDir.java b/engine/table/src/main/java/io/deephaven/engine/util/CacheDir.java new file mode 100644 index 00000000000..bc6e9c011e1 --- /dev/null +++ b/engine/table/src/main/java/io/deephaven/engine/util/CacheDir.java @@ -0,0 +1,39 @@ +package io.deephaven.engine.util; + +import io.deephaven.configuration.Configuration; + +import java.nio.file.Path; + +/** + * The cache directory is a directory that the application may use for storing data with "cache-like" semantics. + * Cache-like data is data that may be preserved across restarts, but the application logic should not make the + * assumptions that the data will be available. + */ +public final class CacheDir { + private static final String DEEPHAVEN_CACHE_DIR = "deephaven.cache.dir"; + private static final String JAVA_IO_TMPDIR = "java.io.tmpdir"; + + private static final Path cacheDir; + + static { + final Configuration config = Configuration.getInstance(); + if (config.hasProperty(DEEPHAVEN_CACHE_DIR)) { + cacheDir = Path.of(config.getProperty(DEEPHAVEN_CACHE_DIR)); + } else { + cacheDir = Path.of(System.getProperty(JAVA_IO_TMPDIR), "deephaven", "cache"); + } + } + + /** + * Return the value for the configuration {@value DEEPHAVEN_CACHE_DIR} if it is present. + * + *

    + * Otherwise, return "%s/deephaven/cache", parameterized from the value of the system property + * {@value JAVA_IO_TMPDIR}. + * + * @return the cache dir + */ + public static Path get() { + return cacheDir; + } +} diff --git a/engine/table/src/main/java/io/deephaven/engine/util/TableTools.java b/engine/table/src/main/java/io/deephaven/engine/util/TableTools.java index 2538ec79a1b..20908ad0cfa 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/TableTools.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/TableTools.java @@ -16,6 +16,7 @@ import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.perf.QueryPerformanceRecorder; +import io.deephaven.internal.log.LoggerFactory; import io.deephaven.time.DateTime; import io.deephaven.time.DateTimeUtils; import io.deephaven.time.TimeProvider; @@ -36,7 +37,6 @@ import io.deephaven.io.logger.Logger; import io.deephaven.io.util.NullOutputStream; import io.deephaven.util.annotations.ScriptApi; -import io.deephaven.util.process.ProcessEnvironment; import io.deephaven.util.type.ArrayTypeUtils; import java.io.*; @@ -58,7 +58,7 @@ @SuppressWarnings("unused") public class TableTools { - private static final Logger staticLog_ = ProcessEnvironment.getDefaultLog(TableTools.class); + private static final Logger staticLog_ = LoggerFactory.getLogger(TableTools.class); // Public so it can be used from user scripts @SuppressWarnings("WeakerAccess") diff --git a/engine/table/src/main/java/io/deephaven/engine/util/WorkerPythonEnvironment.java b/engine/table/src/main/java/io/deephaven/engine/util/WorkerPythonEnvironment.java index c537c2ea159..c9a5f520f47 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/WorkerPythonEnvironment.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/WorkerPythonEnvironment.java @@ -3,8 +3,8 @@ import io.deephaven.base.FileUtils; import io.deephaven.base.verify.Assert; import io.deephaven.configuration.Configuration; +import io.deephaven.internal.log.LoggerFactory; import io.deephaven.io.logger.Logger; -import io.deephaven.util.process.ProcessEnvironment; import io.deephaven.engine.table.lang.QueryScope; import io.deephaven.engine.util.jpy.JpyInit; import org.jpy.PyObject; @@ -36,7 +36,7 @@ public enum WorkerPythonEnvironment { * Runs the init script specified by WorkerPythonEnvironment.initScript. */ WorkerPythonEnvironment() { - log = ProcessEnvironment.getGlobalLog(); + log = LoggerFactory.getLogger(WorkerPythonEnvironment.class); JpyInit.init(log); PythonEvaluatorJpy jpy = PythonEvaluatorJpy.withGlobalCopy(); diff --git a/engine/table/src/main/java/io/deephaven/engine/util/config/InputTableStatusListener.java b/engine/table/src/main/java/io/deephaven/engine/util/config/InputTableStatusListener.java index 7df24f3d9d3..043aaee7bde 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/config/InputTableStatusListener.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/config/InputTableStatusListener.java @@ -1,7 +1,7 @@ package io.deephaven.engine.util.config; +import io.deephaven.internal.log.LoggerFactory; import io.deephaven.io.logger.Logger; -import io.deephaven.util.process.ProcessEnvironment; import java.util.concurrent.CompletableFuture; @@ -13,7 +13,7 @@ public interface InputTableStatusListener { * A Simple implementation that does nothing on success, and logs an error on failure */ InputTableStatusListener DEFAULT = new InputTableStatusListener() { - final Logger log = ProcessEnvironment.getDefaultLog(InputTableStatusListener.class); + final Logger log = LoggerFactory.getLogger(InputTableStatusListener.class); @Override public void onError(Throwable t) { diff --git a/engine/table/src/main/java/io/deephaven/engine/util/parametrized/TableSupplier.java b/engine/table/src/main/java/io/deephaven/engine/util/parametrized/TableSupplier.java index 4008945b7ac..4baa95048b5 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/parametrized/TableSupplier.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/parametrized/TableSupplier.java @@ -6,9 +6,9 @@ import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.TableMap; import io.deephaven.engine.util.TableTools; +import io.deephaven.internal.log.LoggerFactory; import io.deephaven.io.logger.Logger; import io.deephaven.util.annotations.ScriptApi; -import io.deephaven.util.process.ProcessEnvironment; import org.jetbrains.annotations.NotNull; import java.lang.ref.WeakReference; @@ -75,7 +75,7 @@ private static boolean isFilterOperation(Method method) { return method.getName().startsWith(FILTER_OPERATION_PREFIX); } - private final Logger log = ProcessEnvironment.getDefaultLog(TableSupplier.class); + private static final Logger log = LoggerFactory.getLogger(TableSupplier.class); /** * The source table used to generate the supplied table diff --git a/engine/time/src/main/java/io/deephaven/time/calendar/Calendars.java b/engine/time/src/main/java/io/deephaven/time/calendar/Calendars.java index 87366a97ac0..48bbb573188 100644 --- a/engine/time/src/main/java/io/deephaven/time/calendar/Calendars.java +++ b/engine/time/src/main/java/io/deephaven/time/calendar/Calendars.java @@ -131,7 +131,7 @@ private void load(final String businessCalendarConfig) final File calendarFile = inputStreamToFile(inputStream); final BusinessCalendar businessCalendar = DefaultBusinessCalendar.getInstance(calendarFile); addCalendar(businessCalendar); - calendarFile.deleteOnExit(); + calendarFile.delete(); } else { logger.warn("Could not open " + filePath + " from classpath"); throw new RuntimeException("Could not open " + filePath + " from classpath"); diff --git a/engine/updategraph/src/main/java/io/deephaven/engine/liveness/RetainedReferenceTracker.java b/engine/updategraph/src/main/java/io/deephaven/engine/liveness/RetainedReferenceTracker.java index b9bb689c1c5..1733f13a872 100644 --- a/engine/updategraph/src/main/java/io/deephaven/engine/liveness/RetainedReferenceTracker.java +++ b/engine/updategraph/src/main/java/io/deephaven/engine/liveness/RetainedReferenceTracker.java @@ -4,9 +4,10 @@ import io.deephaven.base.reference.WeakCleanupReference; import io.deephaven.engine.util.reference.CleanupReferenceProcessorInstance; import io.deephaven.hash.KeyedObjectHashSet; +import io.deephaven.internal.log.LoggerFactory; +import io.deephaven.io.logger.Logger; import io.deephaven.util.Utils; import io.deephaven.util.datastructures.hash.IdentityKeyedObjectKey; -import io.deephaven.util.process.ProcessEnvironment; import org.jetbrains.annotations.NotNull; import java.lang.ref.SoftReference; @@ -44,6 +45,8 @@ final class RetainedReferenceTracker extends WeakC private static final ThreadLocal>>> tlSavedQueueReference = new ThreadLocal<>(); + private static final Logger log = LoggerFactory.getLogger(RetainedReferenceTracker.class); + private final Impl impl; @SuppressWarnings("FieldMayBeFinal") // We are using an AtomicIntegerFieldUpdater (via reflection) to change this @@ -60,8 +63,10 @@ final class RetainedReferenceTracker extends WeakC impl = enforceStrongReachability ? new StrongImpl() : new WeakImpl(); outstandingCount.getAndIncrement(); if (Liveness.DEBUG_MODE_ENABLED) { - ProcessEnvironment.getDefaultLog().info().append("Creating ").append(Utils.REFERENT_FORMATTER, this) - .append(" at ").append(new LivenessDebugException()).endl(); + log.info() + .append("Creating ").append(Utils.REFERENT_FORMATTER, this) + .append(" at ").append(new LivenessDebugException()) + .endl(); } } diff --git a/engine/updategraph/src/main/java/io/deephaven/engine/updategraph/LogicalClock.java b/engine/updategraph/src/main/java/io/deephaven/engine/updategraph/LogicalClock.java index 648c05ef628..c8b222e2458 100644 --- a/engine/updategraph/src/main/java/io/deephaven/engine/updategraph/LogicalClock.java +++ b/engine/updategraph/src/main/java/io/deephaven/engine/updategraph/LogicalClock.java @@ -5,7 +5,8 @@ package io.deephaven.engine.updategraph; import io.deephaven.base.verify.Assert; -import io.deephaven.util.process.ProcessEnvironment; +import io.deephaven.internal.log.LoggerFactory; +import io.deephaven.io.logger.Logger; import io.deephaven.util.annotations.TestUseOnly; import java.util.concurrent.atomic.AtomicLong; @@ -28,6 +29,8 @@ public enum LogicalClock { DEFAULT; + private static final Logger log = LoggerFactory.getLogger(LogicalClock.class); + /** * The state component of a logical timestamp. */ @@ -142,7 +145,7 @@ public final void ensureUpdateCycleCompleted(final long updatingCycleValue) { return; } if (value == updatingCycleValue) { - ProcessEnvironment.getDefaultLog(LogicalClock.class).warn() + log.warn() .append("LogicalClock cycle was not completed in normal operation, value=").append(value).endl(); completeUpdateCycle(); return; diff --git a/grpc-api-client/console/src/main/java/io/deephaven/grpc_api/example/ConsoleClient.java b/grpc-api-client/console/src/main/java/io/deephaven/grpc_api/example/ConsoleClient.java index f7afbb851f1..f74eb830ac5 100644 --- a/grpc-api-client/console/src/main/java/io/deephaven/grpc_api/example/ConsoleClient.java +++ b/grpc-api-client/console/src/main/java/io/deephaven/grpc_api/example/ConsoleClient.java @@ -47,8 +47,6 @@ private RemoteStderr() {} public static void main(final String[] args) throws Exception { // Assign properties that need to be set to even turn on System.setProperty("Configuration.rootFile", "grpc-api.prop"); - System.setProperty("io.deephaven.configuration.PropertyInputStreamLoader.override", - "io.deephaven.configuration.PropertyInputStreamLoaderTraditional"); final String sessionType = System.getProperty("console.sessionType", "groovy"); log.info().append("Session type ").append(sessionType).endl(); diff --git a/grpc-api/server/native/src/main/resources/bootstrap.properties b/grpc-api/server/native/src/main/resources/bootstrap.properties index 70bc7dccc8f..1b704679a58 100644 --- a/grpc-api/server/native/src/main/resources/bootstrap.properties +++ b/grpc-api/server/native/src/main/resources/bootstrap.properties @@ -1,7 +1,3 @@ Configuration.rootFile=grpc-api-native.prop -io.deephaven.configuration.PropertyInputStreamLoader.override=io.deephaven.configuration.PropertyInputStreamLoaderTraditional -LoggerFactory.silenceOnProcessEnvironment=true workspace=. devroot=. -stdout.toLogBuffer=true -stderr.toLogBuffer=true diff --git a/grpc-api/src/main/java/io/deephaven/grpc_api/log/LogModule.java b/grpc-api/src/main/java/io/deephaven/grpc_api/log/LogModule.java index dc3c5d20436..497c31612c7 100644 --- a/grpc-api/src/main/java/io/deephaven/grpc_api/log/LogModule.java +++ b/grpc-api/src/main/java/io/deephaven/grpc_api/log/LogModule.java @@ -48,8 +48,8 @@ static Set providesLoggerSinkSetups() { @Provides @Singleton // StreamToLogBuffer maintains state static StreamToLogBuffer providesStreamToLogBuffer(LogBuffer logBuffer) { - final boolean toStdout = Boolean.getBoolean("stdout.toLogBuffer"); - final boolean toStderr = Boolean.getBoolean("stderr.toLogBuffer"); + final boolean toStdout = Boolean.parseBoolean(System.getProperty("stdout.toLogBuffer", "true")); + final boolean toStderr = Boolean.parseBoolean(System.getProperty("stderr.toLogBuffer", "true")); // TODO (core#90): Add configuration and documentation for LogBufferOutputStream return new StreamToLogBuffer(logBuffer, toStdout, toStderr, 256, 1 << 19); // 512 KiB } diff --git a/grpc-proxy/build.gradle b/grpc-proxy/build.gradle index 83c3d0751e2..093946d8f3e 100644 --- a/grpc-proxy/build.gradle +++ b/grpc-proxy/build.gradle @@ -1,4 +1,5 @@ import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage +import io.deephaven.tools.docker.Architecture import io.deephaven.tools.License plugins { @@ -9,21 +10,27 @@ evaluationDependsOn Docker.registryProject('alpine') def dockerLicenses = License.createFrom(project).syncDockerLicense() +def context = project.layout.buildDirectory.dir('docker') + def prepareDocker = project.tasks.register('prepareDocker', Sync) { // TODO(deephaven-core#1596): Remove extra dependencies for build-ci.yml inputs.files Docker.registryFiles(project, 'alpine') - from 'Dockerfile' - from ('contents') { + from 'src/main/docker' + from('src/main/arch/') { + into 'arch' + } + from('src/main/common') { into 'contents' } from (dockerLicenses.get().outputs) { into 'contents' } - into 'build/docker' + into context } def buildDocker = Docker.registerDockerImage(project, 'buildDocker') { + inputDir.set context inputs.files prepareDocker.get().outputs.files inputs.files Docker.registryFiles(project, 'alpine') buildArgs.put('DEEPHAVEN_VERSION', project.version) diff --git a/grpc-proxy/contents/checksums.txt b/grpc-proxy/src/main/arch/amd64/checksums.txt similarity index 100% rename from grpc-proxy/contents/checksums.txt rename to grpc-proxy/src/main/arch/amd64/checksums.txt diff --git a/grpc-proxy/src/main/arch/amd64/install.sh b/grpc-proxy/src/main/arch/amd64/install.sh new file mode 100755 index 00000000000..ad9266705a1 --- /dev/null +++ b/grpc-proxy/src/main/arch/amd64/install.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env sh + +set -o errexit +set -o nounset +set -o xtrace + +wget -q "https://github.com/improbable-eng/grpc-web/releases/download/${GRPCWEBPROXY_VERSION}/grpcwebproxy-${GRPCWEBPROXY_VERSION}-linux-x86_64.zip" +sha256sum -c /arch/checksums.txt + +mkdir /app +unzip -d /app "grpcwebproxy-${GRPCWEBPROXY_VERSION}-linux-x86_64.zip" +rm "grpcwebproxy-${GRPCWEBPROXY_VERSION}-linux-x86_64.zip" +mv "/app/dist/grpcwebproxy-${GRPCWEBPROXY_VERSION}-linux-x86_64" /app/dist/grpcwebproxy + +apk add --no-cache tini diff --git a/grpc-proxy/src/main/arch/arm64/checksums.txt b/grpc-proxy/src/main/arch/arm64/checksums.txt new file mode 100644 index 00000000000..3d809628ff4 --- /dev/null +++ b/grpc-proxy/src/main/arch/arm64/checksums.txt @@ -0,0 +1 @@ +be1628a1f79f0e7d1e96b2ef55001c0143f762732867bfbd35d55f5b0634d5b8 grpcwebproxy-v0.13.0-arm64.zip \ No newline at end of file diff --git a/grpc-proxy/src/main/arch/arm64/install.sh b/grpc-proxy/src/main/arch/arm64/install.sh new file mode 100755 index 00000000000..558e8f57a82 --- /dev/null +++ b/grpc-proxy/src/main/arch/arm64/install.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env sh + +set -o errexit +set -o nounset +set -o xtrace + +wget -q "https://github.com/improbable-eng/grpc-web/releases/download/${GRPCWEBPROXY_VERSION}/grpcwebproxy-${GRPCWEBPROXY_VERSION}-arm64.zip" +sha256sum -c /arch/checksums.txt + +mkdir /app /app/dist +unzip -d /app/dist "grpcwebproxy-${GRPCWEBPROXY_VERSION}-arm64.zip" +rm "grpcwebproxy-${GRPCWEBPROXY_VERSION}-arm64.zip" +mv "/app/dist/grpcwebproxy-${GRPCWEBPROXY_VERSION}-arm64" /app/dist/grpcwebproxy + +apk add --no-cache tini diff --git a/grpc-proxy/contents/docker-entrypoint.sh b/grpc-proxy/src/main/common/docker-entrypoint.sh similarity index 100% rename from grpc-proxy/contents/docker-entrypoint.sh rename to grpc-proxy/src/main/common/docker-entrypoint.sh diff --git a/grpc-proxy/Dockerfile b/grpc-proxy/src/main/docker/Dockerfile similarity index 55% rename from grpc-proxy/Dockerfile rename to grpc-proxy/src/main/docker/Dockerfile index 456837a996f..1e696831975 100644 --- a/grpc-proxy/Dockerfile +++ b/grpc-proxy/src/main/docker/Dockerfile @@ -1,21 +1,18 @@ -FROM deephaven/alpine:local-build - -COPY contents/ . - -ENV GRPCWEBPROXY_VERSION=v0.13.0 +FROM deephaven/alpine:local-build as install # Note(deephaven-core#599): Consider moving grpcwebproxy DL into gradle + +ARG TARGETARCH +COPY arch/${TARGETARCH} /arch RUN set -eux; \ - mkdir /app; \ - apk add --no-cache tini; \ - wget -q "https://github.com/improbable-eng/grpc-web/releases/download/${GRPCWEBPROXY_VERSION}/grpcwebproxy-${GRPCWEBPROXY_VERSION}-linux-x86_64.zip"; \ - sha256sum -c checksums.txt; \ - unzip -d /app "grpcwebproxy-${GRPCWEBPROXY_VERSION}-linux-x86_64.zip"; \ - rm "grpcwebproxy-${GRPCWEBPROXY_VERSION}-linux-x86_64.zip"; \ - mv "/app/dist/grpcwebproxy-${GRPCWEBPROXY_VERSION}-linux-x86_64" /app/dist/grpcwebproxy + GRPCWEBPROXY_VERSION=v0.13.0 /arch/install.sh; \ + rm -r /arch -EXPOSE 8080 8443 +COPY contents/ . +FROM deephaven/alpine:local-build +COPY --from=install / / +EXPOSE 8080 8443 ENTRYPOINT ["/sbin/tini", "--", "/docker-entrypoint.sh"] ARG DEEPHAVEN_VERSION diff --git a/log-factory/src/main/java/io/deephaven/internal/log/LoggerFactory.java b/log-factory/src/main/java/io/deephaven/internal/log/LoggerFactory.java index 6ac059fad59..67524b96e32 100644 --- a/log-factory/src/main/java/io/deephaven/internal/log/LoggerFactory.java +++ b/log-factory/src/main/java/io/deephaven/internal/log/LoggerFactory.java @@ -8,7 +8,7 @@ static Logger getLogger(String name) { } static Logger getLogger(Class clazz) { - return getInstance().create(clazz.getName()); // todo + return getInstance().create(clazz.getName()); } static LoggerFactory getInstance() { diff --git a/open-api/lang-tools/src/test/groovy/io/deephaven/lang/completion/ChunkerCompleterMixin.groovy b/open-api/lang-tools/src/test/groovy/io/deephaven/lang/completion/ChunkerCompleterMixin.groovy index d3052123d2a..dee84d5d6ca 100644 --- a/open-api/lang-tools/src/test/groovy/io/deephaven/lang/completion/ChunkerCompleterMixin.groovy +++ b/open-api/lang-tools/src/test/groovy/io/deephaven/lang/completion/ChunkerCompleterMixin.groovy @@ -1,9 +1,9 @@ package io.deephaven.lang.completion import io.deephaven.engine.util.VariableProvider +import io.deephaven.internal.log.LoggerFactory import io.deephaven.io.logger.Logger import io.deephaven.proto.backplane.script.grpc.CompletionItem -import io.deephaven.util.process.ProcessEnvironment import io.deephaven.lang.parse.ParsedDocument /** @@ -12,7 +12,7 @@ trait ChunkerCompleterMixin extends ChunkerParseTestMixin { Set performSearch(ParsedDocument doc, int from, VariableProvider vars) { - Logger log = ProcessEnvironment.getDefaultLog(CompletionHandler) + Logger log = LoggerFactory.getLogger(CompletionHandler) ChunkerCompleter completer = new ChunkerCompleter(log, vars) diff --git a/open-api/lang-tools/src/test/groovy/io/deephaven/lang/completion/ChunkerCompletionHandlerTest.groovy b/open-api/lang-tools/src/test/groovy/io/deephaven/lang/completion/ChunkerCompletionHandlerTest.groovy index 07da3f6da22..6319dc15266 100644 --- a/open-api/lang-tools/src/test/groovy/io/deephaven/lang/completion/ChunkerCompletionHandlerTest.groovy +++ b/open-api/lang-tools/src/test/groovy/io/deephaven/lang/completion/ChunkerCompletionHandlerTest.groovy @@ -3,10 +3,10 @@ package io.deephaven.lang.completion import io.deephaven.engine.table.ColumnDefinition import io.deephaven.engine.table.TableDefinition import io.deephaven.engine.util.VariableProvider +import io.deephaven.internal.log.LoggerFactory import io.deephaven.io.logger.Logger import io.deephaven.proto.backplane.script.grpc.ChangeDocumentRequest.TextDocumentContentChangeEvent import io.deephaven.proto.backplane.script.grpc.CompletionItem -import io.deephaven.util.process.ProcessEnvironment import io.deephaven.engine.table.Table import io.deephaven.lang.parse.CompletionParser import spock.lang.Specification @@ -125,7 +125,7 @@ u = t.''' String src = "t =" doc = p.parse(src) - ProcessEnvironment.getDefaultLog(CompletionHandler) + LoggerFactory.getLogger(CompletionHandler) VariableProvider variables = Mock(VariableProvider) { _ * getVariableNames() >> ['emptyTable'] 0 * _ @@ -141,7 +141,7 @@ u = t.''' def "Completion should work correctly after making edits"() { given: - Logger log = ProcessEnvironment.getDefaultLog(CompletionHandler) + Logger log = LoggerFactory.getLogger(CompletionHandler) CompletionParser p = new CompletionParser() String uri = "testing://" String src1 = """a = 1 diff --git a/open-api/lang-tools/src/test/groovy/io/deephaven/lang/completion/ChunkerParseTestMixin.groovy b/open-api/lang-tools/src/test/groovy/io/deephaven/lang/completion/ChunkerParseTestMixin.groovy index ee450bcc460..c77bf009ebe 100644 --- a/open-api/lang-tools/src/test/groovy/io/deephaven/lang/completion/ChunkerParseTestMixin.groovy +++ b/open-api/lang-tools/src/test/groovy/io/deephaven/lang/completion/ChunkerParseTestMixin.groovy @@ -3,6 +3,7 @@ package io.deephaven.lang.completion import groovy.transform.CompileDynamic import groovy.transform.CompileStatic import io.deephaven.engine.util.VariableProvider +import io.deephaven.internal.log.LoggerFactory import io.deephaven.io.logger.Logger import io.deephaven.lang.generated.Node import io.deephaven.lang.generated.Token @@ -11,7 +12,6 @@ import io.deephaven.lang.parse.ParsedDocument import io.deephaven.proto.backplane.script.grpc.CompletionItem import io.deephaven.proto.backplane.script.grpc.CompletionItemOrBuilder import io.deephaven.proto.backplane.script.grpc.Position -import io.deephaven.util.process.ProcessEnvironment import spock.lang.Specification /** @@ -55,7 +55,7 @@ trait ChunkerParseTestMixin { @CompileDynamic String doCompletion(String command, int completionPos, int resultIndex) { - Logger log = ProcessEnvironment.getDefaultLog(CompletionHandler) + Logger log = LoggerFactory.getLogger(CompletionHandler) ChunkerCompleter completer = new ChunkerCompleter(log, variables) parse(command) diff --git a/open-api/lang-tools/src/test/groovy/io/deephaven/lang/completion/ColumnExpressionCompletionHandlerTest.groovy b/open-api/lang-tools/src/test/groovy/io/deephaven/lang/completion/ColumnExpressionCompletionHandlerTest.groovy index f876ff8a81e..5525e9e6a3d 100644 --- a/open-api/lang-tools/src/test/groovy/io/deephaven/lang/completion/ColumnExpressionCompletionHandlerTest.groovy +++ b/open-api/lang-tools/src/test/groovy/io/deephaven/lang/completion/ColumnExpressionCompletionHandlerTest.groovy @@ -1,9 +1,9 @@ package io.deephaven.lang.completion import io.deephaven.engine.util.VariableProvider +import io.deephaven.internal.log.LoggerFactory import io.deephaven.io.logger.Logger import io.deephaven.proto.backplane.script.grpc.CompletionItem -import io.deephaven.util.process.ProcessEnvironment import io.deephaven.engine.table.Table import io.deephaven.engine.table.TableDefinition import io.deephaven.time.DateTime @@ -26,7 +26,7 @@ class ColumnExpressionCompletionHandlerTest extends Specification implements Chu CompletionParser p = new CompletionParser() doc = p.parse(src) - Logger log = ProcessEnvironment.getDefaultLog(CompletionHandler) + Logger log = LoggerFactory.getLogger(CompletionHandler) VariableProvider variables = Mock(VariableProvider) { (0..1) * getVariableNames() >> ['t'] (0..1) * getVariableType('t') >> Table @@ -74,7 +74,7 @@ t = t.updateView ( 'D """ doc = p.parse(src) - Logger log = ProcessEnvironment.getDefaultLog(CompletionHandler) + Logger log = LoggerFactory.getLogger(CompletionHandler) VariableProvider variables = Mock() { (0..1) * getTableDefinition('t') >> new TableDefinition([String, Long, Integer], ['Date', 'Delta', 'NotMeThough']) (0..1) * getVariableType('t') >> Table @@ -113,7 +113,7 @@ t = t.update('A=') .update( 'B=') """ doc = p.parse(src) - Logger log = ProcessEnvironment.getDefaultLog(CompletionHandler) + Logger log = LoggerFactory.getLogger(CompletionHandler) VariableProvider variables = Mock(VariableProvider) { _ * getTableDefinition('t') >> new TableDefinition([Long, Integer], ['A1', 'A2']) 0 * _ @@ -170,7 +170,7 @@ t = newTable( t.where('""" doc = p.parse(src) - Logger log = ProcessEnvironment.getDefaultLog(CompletionHandler) + Logger log = LoggerFactory.getLogger(CompletionHandler) VariableProvider variables = Mock(VariableProvider) { _ * getTableDefinition('t') >> null 0 * _ diff --git a/proto/proto-backplane-grpc/build.gradle b/proto/proto-backplane-grpc/build.gradle index e3e45168577..5ea040ad2f6 100644 --- a/proto/proto-backplane-grpc/build.gradle +++ b/proto/proto-backplane-grpc/build.gradle @@ -33,7 +33,11 @@ TaskProvider generateProtobuf = Docker.registerDockerTask(project, 'genera } } } + + // protoc-base is only provided via linux/amd64 parentContainers = [ Docker.registryTask(project, 'protoc-base') ] + platform = 'linux/amd64' + containerOutPath = '/generated' imageName = 'deephaven/proto-backplane-grpc:local-build' copyOut { diff --git a/proto/raw-js-openapi/build.gradle b/proto/raw-js-openapi/build.gradle index 12cc8b52e3b..cde0fdb7f33 100644 --- a/proto/raw-js-openapi/build.gradle +++ b/proto/raw-js-openapi/build.gradle @@ -7,7 +7,7 @@ evaluationDependsOn ':proto:proto-backplane-grpc' evaluationDependsOn Docker.registryProject('node') def backplaneProject = project(':proto:proto-backplane-grpc') -def webpackSourcesLocation = "${buildDir}/dhapi" +def webpackSourcesLocation = layout.buildDirectory.dir("${buildDir}/dhapi") Docker.registerDockerTask(project, 'webpackSources') { copyIn { @@ -30,12 +30,22 @@ Docker.registerDockerTask(project, 'webpackSources') { from ('Dockerfile') from ('../package.json') from ('../package-lock.json') - } parentContainers = [ Docker.registryTask(project, 'node') ] imageName = 'deephaven/js-out:local-build' containerOutPath = '/usr/src/app/raw-js-openapi/build/js-out' copyOut { - into(webpackSourcesLocation) + include 'dh-internal.js' + into(webpackSourcesLocation.get().dir('jsapi')) + } +} + +configurations { + js +} + +artifacts { + js(webpackSourcesLocation) { + builtBy webpackSources } } diff --git a/settings.gradle b/settings.gradle index a1c152583e5..c01fb821954 100644 --- a/settings.gradle +++ b/settings.gradle @@ -17,6 +17,7 @@ String[] mods = [ 'Util', 'Numerics', 'TableLogger', 'Plot', // new web projects; these modules are intended to form a complete, modular web application, // with heavy dependency isolation that should enable very fast rebuilds. String[] webMods = [ + 'web', 'web-client-api', // compiled javascript api client 'web-client-ide', // IDE-only additions to open-api javascript client 'web-client-ui', // React IDE client app @@ -245,6 +246,9 @@ project(':docker-runtime-base').projectDir = file('docker/runtime-base') include(':docker-server') project(':docker-server').projectDir = file('docker/server') +include(':docker-web') +project(':docker-web').projectDir = file('docker/web') + include(':docker-server-slim') project(':docker-server-slim').projectDir = file('docker/server-slim') diff --git a/web/client-ide/client-ide.gradle b/web/client-ide/client-ide.gradle index d1c9a29a9b1..07e70372112 100644 --- a/web/client-ide/client-ide.gradle +++ b/web/client-ide/client-ide.gradle @@ -1,57 +1,37 @@ -import io.deephaven.tools.License - plugins { id 'com.bmuschko.docker-remote-api' } -evaluationDependsOn ':web-client-ui' -evaluationDependsOn ':proto:raw-js-openapi' evaluationDependsOn Docker.registryProject('nginx-base') apply from: "$rootDir/gradle/web-client.gradle" +configurations { + js +} + dependencies { compile project(':web-client-api') compile project(':open-api-lang-parser') + + js project(path: ':proto:raw-js-openapi', configuration: 'js') } GwtTools.gwtCompile project, 'io.deephaven.ide.DeephavenIde', 'Create a jar of javascript for web IDE' -// deephaven/dhide -Task dhide = project(':web-client-ui').tasks.findByName('ideClientMakeImage') - -// deephaven/js-out -Task jsOut = project(':proto:raw-js-openapi').tasks.findByName('webpackSourcesMakeImage') - -Task gwtc = tasks.getByName('gwtCompile') - -def dockerLicenses = License.createFrom(project).syncDockerLicense() - -def prepareDocker = project.tasks.register('prepareDocker', Sync) { - // TODO(deephaven-core#1596): Remove extra dependencies for build-ci.yml - // Note: GH actions currently prepares docker via gradle, but does its own building via - // a native GH action. As such, we need to make sure that the prepare also invokes the - // necessarily build pre-reqs. If GH actions eventually calls buildDocker instead, we - // can remove these dependencies. - inputs.files([dhide, jsOut].each { t -> t.outputs.files }) - inputs.files Docker.registryFiles(project, 'nginx-base') - - into "${buildDir}/docker" - from(gwtc.outputs.files) { - include("dhapi/**") +def jsOutput = layout.buildDirectory.dir('js') +def gwtOutput = tasks.register('gwtOutput', Sync) { + includeEmptyDirs = false + from(tasks.getByName('gwtCompile').outputs.files) { + // only copy the dhapi module, and don't give it a wrapper directory + include 'dhapi/**' + eachFile { it.path = 'jsapi/' + it.path.substring('dhapi/'.length()) } } - from (dockerLicenses.get().outputs) { - into 'licenses' - } - from 'docker/Dockerfile' - from 'nginx/default.conf' - from 'nginx/nginx.conf' - from 'nginx/99-init-notebooks.sh' + into jsOutput } -Docker.registerDockerImage(project, 'buildDocker') { - inputs.files([prepareDocker.get(), dhide, jsOut].each { t -> t.outputs.files }) - inputs.files Docker.registryFiles(project, 'nginx-base') - buildArgs.put('DEEPHAVEN_VERSION', project.version) - images.add('deephaven/web:local-build') +artifacts { + js(jsOutput) { + builtBy gwtOutput + } } diff --git a/web/client-ui/client-ui.gradle b/web/client-ui/client-ui.gradle index 99aab5c764b..8847ebcbcd4 100644 --- a/web/client-ui/client-ui.gradle +++ b/web/client-ui/client-ui.gradle @@ -4,7 +4,8 @@ plugins { evaluationDependsOn Docker.registryProject('node') -Docker.registerDockerTask(project, 'ideClient') { +def dhUi = layout.buildDirectory.dir('dhide') +def ui = Docker.registerDockerTask(project, 'ui') { copyIn { from file('Dockerfile') } @@ -12,6 +13,16 @@ Docker.registerDockerTask(project, 'ideClient') { containerOutPath = '/usr/src/app/package/build' imageName = 'deephaven/dhide:local-build' copyOut { - into "${buildDir}/dhide" + into dhUi.get().dir('ide') + } +} + +configurations { + js +} + +artifacts { + js(dhUi) { + builtBy ui } } diff --git a/web/web.gradle b/web/web.gradle new file mode 100644 index 00000000000..4098cc3e984 --- /dev/null +++ b/web/web.gradle @@ -0,0 +1,25 @@ +configurations { + js + jsJar { + transitive = false + } +} + +archivesBaseName = "deephaven-web" + +dependencies { + js project(path: ':web-client-ui', configuration: 'js') + js project(path: ':web-client-ide', configuration: 'js') +} + +/** + * Provides a jar full of static js/html/css that can be served from jetty + */ +def ideClientJsJar = tasks.register('ideClientJsJar', Jar) { + from configurations.js +} + +artifacts { + jsJar ideClientJsJar + js layout.buildDirectory.dir('empty') +}