Merge remote-tracking branch 'upstream/main' into build/docker/labels

deephaven · Dec 17, 2021 · e18b5f8 · e18b5f8
2 parents b8bb872 + 170e336
commit e18b5f8
Show file tree

Hide file tree

Showing 58 changed files with 398 additions and 249 deletions.
diff --git a/.github/workflows/build-ci.yml b/.github/workflows/build-ci.yml
@@ -483,7 +483,7 @@ jobs:
         uses: burrunan/gradle-cache-action@v1
         with:
           job-id: build-web
-          arguments: --scan outputVersion createCraneTagScript web-client-ide:prepareDocker
+          arguments: --scan outputVersion createCraneTagScript docker-web:prepareDocker
           gradle-version: wrapper
 
       - name: Get Deephaven Version
@@ -498,7 +498,7 @@ jobs:
             DEEPHAVEN_VERSION=${{ steps.deephaven_version.outputs.deephaven_version }}
           tags: ${{ steps.docker_meta.outputs.tags }}
           builder: ${{ steps.buildx.outputs.name }}
-          context: ./web/client-ide/build/docker/
+          context: ./docker/web/build/docker/
           push: ${{ github.event_name != 'pull_request' }}
 
       - name: Tag upstream images

diff --git a/...Support/src/main/java/io/deephaven/clientsupport/plotdownsampling/RunChartDownsample.java b/...Support/src/main/java/io/deephaven/clientsupport/plotdownsampling/RunChartDownsample.java
@@ -7,14 +7,14 @@
 import io.deephaven.engine.table.TableUpdate;
 import io.deephaven.engine.table.impl.TableUpdateImpl;
 import io.deephaven.engine.rowset.chunkattributes.OrderedRowKeys;
+import io.deephaven.internal.log.LoggerFactory;
 import io.deephaven.time.DateTime;
 import io.deephaven.hash.KeyedLongObjectHash;
 import io.deephaven.hash.KeyedLongObjectHashMap;
 import io.deephaven.hash.KeyedLongObjectKey;
 import io.deephaven.base.verify.Assert;
 import io.deephaven.configuration.Configuration;
 import io.deephaven.io.logger.Logger;
-import io.deephaven.util.process.ProcessEnvironment;
 import io.deephaven.engine.table.Table;
 import io.deephaven.engine.table.impl.perf.QueryPerformanceRecorder;
 import io.deephaven.engine.table.impl.*;
@@ -50,7 +50,7 @@
  * make nulls result in fewer items in the result table
  */
 public class RunChartDownsample implements Function<Table, Table> {
-    private static final Logger log = ProcessEnvironment.getDefaultLog(RunChartDownsample.class);
+    private static final Logger log = LoggerFactory.getLogger(RunChartDownsample.class);
 
     public static final int CHUNK_SIZE = Configuration.getInstance().getIntegerWithDefault("chunkSize", 1 << 14);
 

diff --git a/FishUtil/src/main/java/io/deephaven/util/process/BaseProcessEnvironment.java b/FishUtil/src/main/java/io/deephaven/util/process/BaseProcessEnvironment.java
@@ -63,14 +63,4 @@ public final FatalErrorReporter getFatalErrorReporter() {
     public final String getMainClassName() {
         return mainClassName;
     }
-
-    @Override
-    public final Logger getLog() {
-        if (!Boolean.getBoolean("LoggerFactory.silenceOnProcessEnvironment")) {
-            log.warn(new RuntimeException("Trace"))
-                    .append("Logger being fetched via ProcessEnvironment instead of io.deephaven.internal.log.LoggerFactory")
-                    .endl();
-        }
-        return log;
-    }
 }
diff --git a/FishUtil/src/main/java/io/deephaven/util/process/ProcessEnvironment.java b/FishUtil/src/main/java/io/deephaven/util/process/ProcessEnvironment.java
@@ -8,8 +8,6 @@
 import io.deephaven.configuration.Configuration;
 import io.deephaven.configuration.PropertyException;
 import io.deephaven.io.logger.Logger;
-import io.deephaven.io.logger.StreamLoggerImpl;
-import io.deephaven.internal.log.LoggerFactory;
 import org.jetbrains.annotations.NotNull;
 
 /**
@@ -42,13 +40,6 @@ public interface ProcessEnvironment {
      */
     String getMainClassName();
 
-    /**
-     * Access a shared Logger.
-     *
-     * @return The Logger
-     */
-    Logger getLog();
-
     /**
      * Hook for setting up an installation-specific environment for a given process.
      */
@@ -126,32 +117,6 @@ static String getGlobalMainClassName() {
         return get().getMainClassName();
     }
 
-    static Logger getGlobalLog() {
-        return get().getLog();
-    }
-
-    /**
-     * Get the global log if a global process environment has been installed, or else a logger that will output to
-     * {@link System#out}.
-     * 
-     * @return A logger that can safely be used by code that doesn't otherwise have access to one
-     */
-    static Logger getDefaultLog() {
-        final ProcessEnvironment processEnvironment = ProcessEnvironment.tryGet();
-        return processEnvironment != null ? processEnvironment.getLog() : new StreamLoggerImpl();
-    }
-
-    /**
-     * Get the global log if a global process environment has been installed, or else a
-     * {@link LoggerFactory#getLogger(Class)} for the given class.
-     * 
-     * @return A logger that can safely be used by code that doesn't otherwise have access to one
-     */
-    static Logger getDefaultLog(Class<?> clazz) {
-        final ProcessEnvironment processEnvironment = ProcessEnvironment.tryGet();
-        return processEnvironment != null ? processEnvironment.getLog() : LoggerFactory.getLogger(clazz);
-    }
-
     /**
      * Setter for the global instance. It is an error to invoke this without allowReplace if the global instance may
      * already have been set.

diff --git a/ModelFarm/src/main/java/io/deephaven/modelfarm/ConditionalModels.java b/ModelFarm/src/main/java/io/deephaven/modelfarm/ConditionalModels.java
@@ -6,7 +6,7 @@
 
 import io.deephaven.base.verify.Require;
 import io.deephaven.configuration.Configuration;
-import io.deephaven.util.process.ProcessEnvironment;
+import io.deephaven.internal.log.LoggerFactory;
 import io.deephaven.modelfarm.ModelMultiExec.Models;
 
 import java.util.ArrayList;
@@ -26,7 +26,7 @@
  */
 public class ConditionalModels<DATA_TYPE, STATE_TYPE, KEY_TYPE> implements Models<DATA_TYPE> {
 
-    private static final io.deephaven.io.logger.Logger log = ProcessEnvironment.getDefaultLog(ConditionalModels.class);
+    private static final io.deephaven.io.logger.Logger log = LoggerFactory.getLogger(ConditionalModels.class);
     private static final boolean LOG_PERF =
             Configuration.getInstance().getBooleanWithDefault("ModelFarm.logConditionalModelsPerformance", false);
 

diff --git a/ModelFarm/src/main/java/io/deephaven/modelfarm/ModelFarmOnDemand.java b/ModelFarm/src/main/java/io/deephaven/modelfarm/ModelFarmOnDemand.java
@@ -9,8 +9,8 @@
 import io.deephaven.engine.table.Table;
 import io.deephaven.engine.updategraph.UpdateGraphProcessor;
 import io.deephaven.engine.rowset.RowSet;
+import io.deephaven.internal.log.LoggerFactory;
 import io.deephaven.io.logger.Logger;
-import io.deephaven.util.process.ProcessEnvironment;
 import io.deephaven.engine.table.impl.NotificationStepSource;
 import io.deephaven.util.FunctionalInterfaces;
 
@@ -33,7 +33,7 @@ public class ModelFarmOnDemand<KEYTYPE, DATATYPE, ROWDATAMANAGERTYPE extends Row
 
     private static final boolean LOG_PERF =
             Configuration.getInstance().getBooleanWithDefault("ModelFarm.logModelFarmOnDemandPerformance", false);
-    private static final Logger log = ProcessEnvironment.getDefaultLog(ModelFarmOnDemand.class);
+    private static final Logger log = LoggerFactory.getLogger(ModelFarmOnDemand.class);
     private static final FunctionalInterfaces.ThrowingBiConsumer<QueryDataRetrievalOperation, NotificationStepSource, RuntimeException> DO_LOCKED_FUNCTION =
             getDoLockedConsumer(GetDataLockType.UGP_READ_LOCK);
 

diff --git a/Net/src/main/java/io/deephaven/net/impl/nio/NIODriver.java b/Net/src/main/java/io/deephaven/net/impl/nio/NIODriver.java
@@ -18,7 +18,6 @@
 import io.deephaven.io.sched.Scheduler;
 import io.deephaven.io.sched.TimedJob;
 import io.deephaven.io.sched.YASchedulerImpl;
-import io.deephaven.util.process.ProcessEnvironment;
 
 public class NIODriver implements Runnable {
     private static Logger log;
@@ -94,14 +93,7 @@ public static String summary() {
      */
     public static void init() {
         if (!initialized) {
-            final Logger log;
-            final ProcessEnvironment processEnvironment = ProcessEnvironment.tryGet();
-            if (processEnvironment == null) {
-                log = LoggerFactory.getLogger(NIODriver.class);
-            } else {
-                log = processEnvironment.getLog();
-            }
-            init(log);
+            init(LoggerFactory.getLogger(NIODriver.class));
         }
     }
 

diff --git a/build.gradle b/build.gradle
@@ -98,6 +98,7 @@ Set<Project> modsAreBasic = subprojects.findAll {it.name in [
     'deephaven2-wheel',
     'envoy',
     'grpc-proxy',
+    'web',
     'web-client-ui',
     'pyclient',
     'sphinx',
@@ -480,7 +481,7 @@ tasks.register('prepareCompose') {
     it.description 'A lifecycle task that prepares prerequisites for local docker-compose builds'
     it.dependsOn project(':docker-server-slim').tasks.findByName('buildDocker'),
             project(':docker-server').tasks.findByName('buildDocker-server'),
-            project(':web-client-ide').tasks.findByName('buildDocker'),
+            project(':docker-web').tasks.findByName('buildDocker'),
             project(':envoy').tasks.findByName('buildDocker'),
             project(':grpc-proxy').tasks.findByName('buildDocker')
 }
@@ -496,7 +497,7 @@ tasks.register('smoke') {
     it.dependsOn project(':grpc-api').tasks.findByName(LifecycleBasePlugin.CHECK_TASK_NAME)
     it.dependsOn project(':docker-server-slim').tasks.findByName('prepareDocker')
     it.dependsOn project(':docker-server').tasks.findByName('prepareDocker')
-    it.dependsOn project(':web-client-ide').tasks.findByName('prepareDocker')
+    it.dependsOn project(':web').tasks.findByName(LifecycleBasePlugin.ASSEMBLE_TASK_NAME)
     it.dependsOn project(':Generators').tasks.findByName(LifecycleBasePlugin.CHECK_TASK_NAME)
 }
 

diff --git a/buildSrc/src/main/groovy/Docker.groovy b/buildSrc/src/main/groovy/Docker.groovy
@@ -12,6 +12,7 @@ import com.bmuschko.gradle.docker.tasks.image.Dockerfile
 import com.github.dockerjava.api.command.InspectImageResponse
 import com.github.dockerjava.api.exception.DockerException
 import groovy.transform.CompileStatic
+import io.deephaven.tools.docker.Architecture
 import org.gradle.api.Action
 import org.gradle.api.GradleException
 import org.gradle.api.Project
@@ -158,6 +159,11 @@ class Docker {
          */
         Map<String, String> buildArgs;
 
+        /**
+         * Optional platform
+         */
+        String platform;
+
         /**
          * Logs are always printed from the build task when it runs, but entrypoint logs are only printed
          * when it fails. Set this flag to always show logs, even when entrypoint is successful.
@@ -270,6 +276,11 @@ class Docker {
                 if (cfg.buildArgs) {
                     buildArgs.putAll(cfg.buildArgs)
                 }
+
+                // the platform, if provided
+                if (cfg.platform) {
+                    platform.set(cfg.platform)
+                }
             }
         }
 
@@ -426,6 +437,12 @@ class Docker {
         // Produce a docker image from the copied inputs and provided dockerfile, and tag it
         TaskProvider<DockerBuildImage> makeImage = project.tasks.register(taskName, DockerBuildImage) { buildImage ->
             action.execute(buildImage)
+            if (!buildImage.platform.isPresent()) {
+                def targetArch = Architecture.targetArchitecture(project).toString()
+                buildImage.platform.set "linux/${targetArch}".toString()
+                // Use the same environment variable that buildkit uses
+                buildImage.buildArgs.put('TARGETARCH', targetArch)
+            }
             if (buildImage.images) {
                 buildImage.images.get().forEach { String imageName -> validateImageName(imageName) }
 

diff --git a/buildSrc/src/main/groovy/GwtTools.groovy b/buildSrc/src/main/groovy/GwtTools.groovy
@@ -6,10 +6,7 @@ import groovy.transform.CompileStatic
 import org.gradle.api.Project
 import org.gradle.api.artifacts.ProjectDependency
 import org.gradle.api.file.ConfigurableFileCollection
-import org.gradle.api.file.CopySpec
 import org.gradle.api.plugins.JavaPlugin
-import org.gradle.api.tasks.TaskOutputs
-import org.gradle.api.tasks.bundling.Jar
 import org.gradle.api.tasks.compile.JavaCompile
 
 /**
@@ -72,37 +69,6 @@ class GwtTools {
             gwtc.logger.quiet('Running in gwt dev mode; saving source to {}/dh/src', extras)
         }
 
-        // Add a jar task to create an artifact containing our compiled application
-        p.tasks.register "jsJar", Jar, {
-            Jar j ->
-                j.group = '~Deephaven'
-                j.description = description
-                // make a task dependency, AND make sure we rebuild this jar if the gwtc task output files change.
-                j.inputs.files gwtc.outputs.files
-                j.from("$gwt.compile.war/dhapi") {
-                    CopySpec c->
-                        c.exclude '**/extra' // don't put our source files in exported jar, thx
-                        c.into 'dhapi'
-                }
-                // let's also pull in the ide client's build output
-                TaskOutputs uiOutputs = p.project(':web-client-ui').tasks.getByName('ideClient').outputs
-                j.from(uiOutputs.files) {
-                    CopySpec c ->
-                        c.into 'dhide'
-                }
-
-                TaskOutputs internalOpenapiOutputs = p.project(':proto:raw-js-openapi').tasks.getByName('webpackSources').outputs
-                j.from(internalOpenapiOutputs.files) {
-                    CopySpec c ->
-                        c.into 'dhapi'
-                }
-
-                j.classifier = 'js'
-                j.doFirst {
-                    j.logger.info "Merging $gwt.compile.war and $uiOutputs into $j.archivePath"
-                }
-        }
-
         p.tasks.findByName('gwtCheck')?.enabled = false
     }
 

diff --git a/buildSrc/src/main/groovy/io/deephaven/tools/docker/Architecture.groovy b/buildSrc/src/main/groovy/io/deephaven/tools/docker/Architecture.groovy
@@ -0,0 +1,44 @@
+package io.deephaven.tools.docker
+
+import org.gradle.api.Project
+import org.gradle.nativeplatform.platform.internal.DefaultNativePlatform
+
+enum Architecture {
+    AMD64("amd64"),
+    ARM64("arm64");
+
+    static Architecture targetArchitecture(Project project) {
+        if (project.hasProperty('docker.targetArch')) {
+            return fromDockerName(project.property('docker.targetArch') as String)
+        } else {
+            return fromHost()
+        }
+    }
+
+    static Architecture fromHost() {
+        String archName = DefaultNativePlatform.host().getArchitecture().getName()
+        switch (archName) {
+            case "x86-64":
+                return AMD64
+            case "arm-v8":
+                return ARM64
+            default:
+                throw new IllegalStateException("Unable to determine proper docker architecture for " + archName)
+        }
+    }
+
+    static Architecture fromDockerName(String dockerName) {
+        values().find { a -> a.dockerName == dockerName }
+    }
+
+    private final String dockerName
+
+    Architecture(String dockerName) {
+        this.dockerName = dockerName
+    }
+
+    @Override
+    String toString() {
+        dockerName
+    }
+}
diff --git a/docker/server-slim/src/main/docker/image-bootstrap.properties b/docker/server-slim/src/main/docker/image-bootstrap.properties
@@ -1,9 +1,6 @@
 Configuration.rootFile=grpc-api.prop
-io.deephaven.configuration.PropertyInputStreamLoader.override=io.deephaven.configuration.PropertyInputStreamLoaderTraditional
-LoggerFactory.silenceOnProcessEnvironment=true
 workspace=.
 devroot=.
-stdout.toLogBuffer=true
-stderr.toLogBuffer=true
 
-deephaven.console.type=groovy
+deephaven.console.type=groovy
+deephaven.cache.dir=/cache