diff --git a/Util/src/main/java/io/deephaven/util/CompletionStageFuture.java b/Util/src/main/java/io/deephaven/util/CompletionStageFuture.java new file mode 100644 index 00000000000..c410765e187 --- /dev/null +++ b/Util/src/main/java/io/deephaven/util/CompletionStageFuture.java @@ -0,0 +1,201 @@ +/** + * Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending + */ +package io.deephaven.util; + +import org.jetbrains.annotations.NotNull; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.Executor; +import java.util.concurrent.Future; +import java.util.function.BiConsumer; +import java.util.function.BiFunction; +import java.util.function.Consumer; +import java.util.function.Function; + +/** + * A safe version of CompletableFuture that does not expose the completion API. + * + * @param The result type returned by this future's {@code join} + */ +public interface CompletionStageFuture extends Future, CompletionStage { + + /** + * Returns a new CompletionStageFuture that is already completed with the given value. + * + * @param value the value + * @param the type of the value + * @return the completed CompletionStageFuture + * @see CompletableFuture#completedFuture(Object) + */ + static CompletionStageFuture completedFuture(U value) { + final CompletionStageFutureImpl.Resolver resolver = CompletionStageFutureImpl.make(); + resolver.complete(value); + return resolver.getFuture(); + } + + /** + * Returns a new CompletableFuture that is already completed exceptionally with the given exception. + * + * @param ex the exception + * @param the type of the value + * @return the exceptionally completed CompletableFuture + * @since 9 + * @see CompletableFuture#failedFuture(Throwable) + */ + static CompletionStageFuture failedFuture(Throwable ex) { + final CompletionStageFutureImpl.Resolver resolver = CompletionStageFutureImpl.make(); + resolver.completeExceptionally(ex); + return resolver.getFuture(); + } + + interface Resolver { + + /** + * If not already completed, sets the value returned by {@link #get()} and related methods to the given value. + * + * @param value the result value + * @return {@code true} if this invocation caused this SafeCompletableFuture to transition to a completed state, + * else {@code false} + * @see java.util.concurrent.CompletableFuture#complete(Object) + */ + boolean complete(T value); + + /** + * If not already completed, causes invocations of {@link #get()} and related methods to throw the given + * exception. + * + * @param ex the exception + * @return {@code true} if this invocation caused this SafeCompletableFuture to transition to a completed state, + * else {@code false} + * @see java.util.concurrent.CompletableFuture#completeExceptionally(Throwable) + */ + boolean completeExceptionally(@NotNull Throwable ex); + + /** + * @return the underlying future to provide to the recipient + */ + CompletionStageFuture getFuture(); + } + + @Override + CompletionStageFuture thenApply(Function fn); + + @Override + CompletionStageFuture thenApplyAsync(Function fn); + + @Override + CompletionStageFuture thenApplyAsync(Function fn, Executor executor); + + @Override + CompletionStageFuture thenAccept(Consumer action); + + @Override + CompletionStageFuture thenAcceptAsync(Consumer action); + + @Override + CompletionStageFuture thenAcceptAsync(Consumer action, Executor executor); + + @Override + CompletionStageFuture thenRun(Runnable action); + + @Override + CompletionStageFuture thenRunAsync(Runnable action); + + @Override + CompletionStageFuture thenRunAsync(Runnable action, Executor executor); + + @Override + CompletionStageFuture thenCombine( + CompletionStage other, BiFunction fn); + + @Override + CompletionStageFuture thenCombineAsync( + CompletionStage other, BiFunction fn); + + @Override + CompletionStageFuture thenCombineAsync( + CompletionStage other, BiFunction fn, Executor executor); + + @Override + CompletionStageFuture thenAcceptBoth( + CompletionStage other, BiConsumer action); + + @Override + CompletionStageFuture thenAcceptBothAsync( + CompletionStage other, BiConsumer action); + + @Override + CompletionStageFuture thenAcceptBothAsync( + CompletionStage other, BiConsumer action, Executor executor); + + @Override + CompletionStageFuture runAfterBoth(CompletionStage other, Runnable action); + + @Override + CompletionStageFuture runAfterBothAsync(CompletionStage other, Runnable action); + + @Override + CompletionStageFuture runAfterBothAsync(CompletionStage other, Runnable action, Executor executor); + + @Override + CompletionStageFuture applyToEither(CompletionStage other, Function fn); + + @Override + CompletionStageFuture applyToEitherAsync(CompletionStage other, Function fn); + + @Override + CompletionStageFuture applyToEitherAsync( + CompletionStage other, Function fn, Executor executor); + + @Override + CompletionStageFuture acceptEither(CompletionStage other, Consumer action); + + @Override + CompletionStageFuture acceptEitherAsync(CompletionStage other, Consumer action); + + @Override + CompletionStageFuture acceptEitherAsync( + CompletionStage other, Consumer action, Executor executor); + + @Override + CompletionStageFuture runAfterEither(CompletionStage other, Runnable action); + + @Override + CompletionStageFuture runAfterEitherAsync(CompletionStage other, Runnable action); + + @Override + CompletionStageFuture runAfterEitherAsync(CompletionStage other, Runnable action, Executor executor); + + @Override + CompletionStageFuture thenCompose(Function> fn); + + @Override + CompletionStageFuture thenComposeAsync(Function> fn); + + @Override + CompletionStageFuture thenComposeAsync( + Function> fn, Executor executor); + + @Override + CompletionStageFuture handle(BiFunction fn); + + @Override + CompletionStageFuture handleAsync(BiFunction fn); + + @Override + CompletionStageFuture handleAsync(BiFunction fn, Executor executor); + + @Override + CompletionStageFuture whenComplete(BiConsumer action); + + @Override + CompletionStageFuture whenCompleteAsync(BiConsumer action); + + @Override + CompletionStageFuture whenCompleteAsync(BiConsumer action, Executor executor); + + @Override + CompletionStageFuture exceptionally(Function fn); +} diff --git a/Util/src/main/java/io/deephaven/util/CompletionStageFutureImpl.java b/Util/src/main/java/io/deephaven/util/CompletionStageFutureImpl.java new file mode 100644 index 00000000000..d0ebef551fe --- /dev/null +++ b/Util/src/main/java/io/deephaven/util/CompletionStageFutureImpl.java @@ -0,0 +1,356 @@ +/** + * Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending + */ +package io.deephaven.util; + +import org.jetbrains.annotations.NotNull; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; +import java.util.function.*; + +/** + * A {@link CompletableFuture} that prevents users from completing the future by hiding behind + * {@link CompletionStageFuture}. + * + * @param The result type returned by this future's {@code join} + */ +@SuppressWarnings("unchecked") +public class CompletionStageFutureImpl extends CompletableFuture implements CompletionStageFuture { + + /** + * Create a new incomplete future. + * + * @param The result type returned by this future's {@code join} + * @return a resolver for the future + */ + public static Resolver make() { + return new CompletionStageFutureImpl().new ResolverImpl(); + } + + /** + * A resolver for this future implementation. + */ + private class ResolverImpl implements CompletionStageFuture.Resolver { + public boolean complete(final T value) { + return safelyComplete(value); + } + + public boolean completeExceptionally(@NotNull final Throwable ex) { + return safelyCompleteExceptionally(ex); + } + + public CompletionStageFuture getFuture() { + return CompletionStageFutureImpl.this; + } + } + + private boolean safelyComplete(final T value) { + return super.complete(value); + } + + private boolean safelyCompleteExceptionally(@NotNull final Throwable ex) { + return super.completeExceptionally(ex); + } + + @Override + public CompletableFuture newIncompleteFuture() { + return new CompletionStageFutureImpl<>(); + } + + /////////////////////////// + // CompletableFuture API // + /////////////////////////// + + @Override + public boolean complete(final T value) { + throw erroneousCompletionException(); + } + + @Override + public boolean completeExceptionally(@NotNull final Throwable ex) { + throw erroneousCompletionException(); + } + + @Override + public void obtrudeValue(final T value) { + throw erroneousCompletionException(); + } + + @Override + public void obtrudeException(@NotNull final Throwable ex) { + throw erroneousCompletionException(); + } + + @Override + public CompletableFuture completeAsync( + @NotNull final Supplier supplier, + @NotNull final Executor executor) { + throw erroneousCompletionException(); + } + + @Override + public CompletableFuture completeAsync(@NotNull final Supplier supplier) { + throw erroneousCompletionException(); + } + + @Override + public CompletableFuture completeOnTimeout(final T value, long timeout, TimeUnit unit) { + throw erroneousCompletionException(); + } + + private static UnsupportedOperationException erroneousCompletionException() { + return new UnsupportedOperationException("Users should not complete futures."); + } + + ///////////////////////// + // CompletionStage API // + ///////////////////////// + + @Override + public CompletionStageFutureImpl thenApply(@NotNull final Function fn) { + return (CompletionStageFutureImpl) super.thenApply(fn); + } + + @Override + public CompletionStageFutureImpl thenApplyAsync(@NotNull final Function fn) { + return (CompletionStageFutureImpl) super.thenApplyAsync(fn); + } + + @Override + public CompletionStageFutureImpl thenApplyAsync( + @NotNull final Function fn, + @NotNull final Executor executor) { + return (CompletionStageFutureImpl) super.thenApplyAsync(fn, executor); + } + + @Override + public CompletionStageFutureImpl thenAccept(@NotNull final Consumer action) { + return (CompletionStageFutureImpl) super.thenAccept(action); + } + + @Override + public CompletionStageFutureImpl thenAcceptAsync(@NotNull final Consumer action) { + return (CompletionStageFutureImpl) super.thenAcceptAsync(action); + } + + @Override + public CompletionStageFutureImpl thenAcceptAsync( + @NotNull final Consumer action, + @NotNull final Executor executor) { + return (CompletionStageFutureImpl) super.thenAcceptAsync(action, executor); + } + + @Override + public CompletionStageFutureImpl thenRun(@NotNull final Runnable action) { + return (CompletionStageFutureImpl) super.thenRun(action); + } + + @Override + public CompletionStageFutureImpl thenRunAsync(@NotNull final Runnable action) { + return (CompletionStageFutureImpl) super.thenRunAsync(action); + } + + @Override + public CompletionStageFutureImpl thenRunAsync( + @NotNull final Runnable action, + @NotNull final Executor executor) { + return (CompletionStageFutureImpl) super.thenRunAsync(action, executor); + } + + @Override + public CompletionStageFutureImpl thenCombine( + @NotNull final CompletionStage other, + @NotNull final BiFunction fn) { + return (CompletionStageFutureImpl) super.thenCombine(other, fn); + } + + @Override + public CompletionStageFutureImpl thenCombineAsync( + @NotNull final CompletionStage other, + @NotNull final BiFunction fn) { + return (CompletionStageFutureImpl) super.thenCombineAsync(other, fn); + } + + @Override + public CompletionStageFutureImpl thenCombineAsync( + @NotNull final CompletionStage other, + @NotNull final BiFunction fn, + @NotNull final Executor executor) { + return (CompletionStageFutureImpl) super.thenCombineAsync(other, fn, executor); + } + + + @Override + public CompletionStageFutureImpl thenAcceptBoth( + @NotNull final CompletionStage other, + @NotNull final BiConsumer action) { + return (CompletionStageFutureImpl) super.thenAcceptBoth(other, action); + } + + + @Override + public CompletionStageFutureImpl thenAcceptBothAsync( + @NotNull final CompletionStage other, + @NotNull final BiConsumer action) { + return (CompletionStageFutureImpl) super.thenAcceptBothAsync(other, action); + } + + @Override + public CompletionStageFutureImpl thenAcceptBothAsync( + @NotNull final CompletionStage other, + @NotNull final BiConsumer action, + @NotNull final Executor executor) { + return (CompletionStageFutureImpl) super.thenAcceptBothAsync(other, action, executor); + } + + + @Override + public CompletionStageFutureImpl runAfterBoth( + @NotNull final CompletionStage other, + @NotNull final Runnable action) { + return (CompletionStageFutureImpl) super.runAfterBoth(other, action); + } + + @Override + public CompletionStageFutureImpl runAfterBothAsync( + @NotNull final CompletionStage other, + @NotNull final Runnable action) { + return (CompletionStageFutureImpl) super.runAfterBothAsync(other, action); + } + + @Override + public CompletionStageFutureImpl runAfterBothAsync( + @NotNull final CompletionStage other, + @NotNull final Runnable action, + @NotNull final Executor executor) { + return (CompletionStageFutureImpl) super.runAfterBothAsync(other, action, executor); + } + + @Override + public CompletionStageFutureImpl applyToEither( + @NotNull final CompletionStage other, + @NotNull final Function fn) { + return (CompletionStageFutureImpl) super.applyToEither(other, fn); + } + + @Override + public CompletionStageFutureImpl applyToEitherAsync( + @NotNull final CompletionStage other, + @NotNull final Function fn) { + return (CompletionStageFutureImpl) super.applyToEitherAsync(other, fn); + } + + @Override + public CompletionStageFutureImpl applyToEitherAsync( + @NotNull final CompletionStage other, + @NotNull final Function fn, Executor executor) { + return (CompletionStageFutureImpl) super.applyToEitherAsync(other, fn, executor); + } + + @Override + public CompletionStageFutureImpl acceptEither( + @NotNull final CompletionStage other, + @NotNull final Consumer action) { + return (CompletionStageFutureImpl) super.acceptEither(other, action); + } + + @Override + public CompletionStageFutureImpl acceptEitherAsync( + @NotNull final CompletionStage other, + @NotNull final Consumer action) { + return (CompletionStageFutureImpl) super.acceptEitherAsync(other, action); + } + + @Override + public CompletionStageFutureImpl acceptEitherAsync( + @NotNull final CompletionStage other, + @NotNull final Consumer action, + @NotNull final Executor executor) { + return (CompletionStageFutureImpl) super.acceptEitherAsync(other, action, executor); + } + + @Override + public CompletionStageFutureImpl runAfterEither( + @NotNull final CompletionStage other, + @NotNull final Runnable action) { + return (CompletionStageFutureImpl) super.runAfterEither(other, action); + } + + @Override + public CompletionStageFutureImpl runAfterEitherAsync( + @NotNull final CompletionStage other, + @NotNull final Runnable action) { + return (CompletionStageFutureImpl) super.runAfterEitherAsync(other, action); + } + + @Override + public CompletionStageFutureImpl runAfterEitherAsync( + @NotNull final CompletionStage other, + @NotNull final Runnable action, + @NotNull final Executor executor) { + return (CompletionStageFutureImpl) super.runAfterEitherAsync(other, action, executor); + } + + @Override + public CompletionStageFutureImpl thenCompose( + @NotNull final Function> fn) { + return (CompletionStageFutureImpl) super.thenCompose(fn); + } + + @Override + public CompletionStageFutureImpl thenComposeAsync( + @NotNull final Function> fn) { + return (CompletionStageFutureImpl) super.thenComposeAsync(fn); + } + + @Override + public CompletionStageFutureImpl thenComposeAsync( + @NotNull final Function> fn, + @NotNull final Executor executor) { + return (CompletionStageFutureImpl) super.thenComposeAsync(fn, executor); + } + + @Override + public CompletionStageFutureImpl handle(@NotNull final BiFunction fn) { + return (CompletionStageFutureImpl) super.handle(fn); + } + + @Override + public CompletionStageFutureImpl handleAsync( + @NotNull final BiFunction fn) { + return (CompletionStageFutureImpl) super.handleAsync(fn); + } + + @Override + public CompletionStageFutureImpl handleAsync( + @NotNull final BiFunction fn, + @NotNull final Executor executor) { + return (CompletionStageFutureImpl) super.handleAsync(fn, executor); + } + + @Override + public CompletionStageFutureImpl whenComplete(@NotNull final BiConsumer action) { + return (CompletionStageFutureImpl) super.whenComplete(action); + } + + @Override + public CompletionStageFutureImpl whenCompleteAsync( + @NotNull final BiConsumer action) { + return (CompletionStageFutureImpl) super.whenCompleteAsync(action); + } + + @Override + public CompletionStageFutureImpl whenCompleteAsync( + @NotNull final BiConsumer action, + @NotNull final Executor executor) { + return (CompletionStageFutureImpl) super.whenCompleteAsync(action, executor); + } + + @Override + public CompletionStageFutureImpl exceptionally(@NotNull final Function fn) { + return (CompletionStageFutureImpl) super.exceptionally(fn); + } +} diff --git a/Util/src/main/java/io/deephaven/util/datastructures/CachingSupplier.java b/Util/src/main/java/io/deephaven/util/datastructures/CachingSupplier.java new file mode 100644 index 00000000000..43ff488da7e --- /dev/null +++ b/Util/src/main/java/io/deephaven/util/datastructures/CachingSupplier.java @@ -0,0 +1,44 @@ +/** + * Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending + */ +package io.deephaven.util.datastructures; + +import org.jetbrains.annotations.NotNull; + +import java.util.function.Supplier; + +/** + * {@link Supplier} wrapper that caches the result. + * + * @param the type of results supplied by this supplier + */ +public final class CachingSupplier implements Supplier { + + private final Supplier internalSupplier; + + private volatile boolean hasCachedResult; + private volatile OUTPUT_TYPE cachedResult; + + /** + * Construct a {@link Supplier} wrapper. + * + * @param internalSupplier The {@link Supplier} to wrap. + */ + public CachingSupplier(@NotNull final Supplier internalSupplier) { + this.internalSupplier = internalSupplier; + } + + @Override + public OUTPUT_TYPE get() { + if (!hasCachedResult) { + synchronized (this) { + if (!hasCachedResult) { + cachedResult = internalSupplier.get(); + hasCachedResult = true; + } + } + } + + return cachedResult; + } +} diff --git a/engine/context/build.gradle b/engine/context/build.gradle index 0f683dcb5b9..da73cfac799 100644 --- a/engine/context/build.gradle +++ b/engine/context/build.gradle @@ -24,6 +24,7 @@ dependencies { implementation 'com.github.f4b6a3:uuid-creator:5.2.0' Classpaths.inheritCommonsText(project, 'implementation') + Classpaths.inheritImmutables(project) testImplementation TestTools.projectDependency(project, 'Base') testImplementation project(':engine-test-utils') diff --git a/engine/context/src/main/java/io/deephaven/engine/context/PoisonedQueryCompiler.java b/engine/context/src/main/java/io/deephaven/engine/context/PoisonedQueryCompiler.java index 98cba3f36ee..dfc14a1bb5a 100644 --- a/engine/context/src/main/java/io/deephaven/engine/context/PoisonedQueryCompiler.java +++ b/engine/context/src/main/java/io/deephaven/engine/context/PoisonedQueryCompiler.java @@ -1,14 +1,13 @@ -/* - * Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending +/** + * Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending */ package io.deephaven.engine.context; +import io.deephaven.util.CompletionStageFuture; import io.deephaven.util.ExecutionContextRegistrationException; import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; import java.io.File; -import java.util.Map; public class PoisonedQueryCompiler extends QueryCompiler { @@ -31,8 +30,9 @@ public void setParentClassLoader(ClassLoader parentClassLoader) { } @Override - public Class compile(@NotNull String className, @NotNull String classBody, @NotNull String packageNameRoot, - @Nullable StringBuilder codeLog, @NotNull Map> parameterClasses) { - return fail(); + public void compile( + @NotNull final QueryCompilerRequest[] requests, + @NotNull final CompletionStageFuture.Resolver>[] resolvers) { + fail(); } } diff --git a/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java b/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java index 6b54b9bb9fe..c1c07de9008 100644 --- a/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java +++ b/engine/context/src/main/java/io/deephaven/engine/context/QueryCompiler.java @@ -5,16 +5,19 @@ import io.deephaven.UncheckedDeephavenException; import io.deephaven.base.FileUtils; -import io.deephaven.base.Pair; import io.deephaven.configuration.Configuration; import io.deephaven.configuration.DataDir; import io.deephaven.datastructures.util.CollectionUtil; +import io.deephaven.engine.context.util.SynchronizedJavaFileManager; +import io.deephaven.engine.updategraph.OperationInitializer; import io.deephaven.internal.log.LoggerFactory; import io.deephaven.io.logger.Logger; import io.deephaven.util.ByteUtils; +import io.deephaven.util.CompletionStageFuture; +import io.deephaven.util.CompletionStageFutureImpl; +import org.apache.commons.lang3.mutable.MutableInt; import org.apache.commons.text.StringEscapeUtils; import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; import javax.tools.*; import java.io.*; @@ -31,9 +34,7 @@ import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.*; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.*; import java.util.function.Supplier; import java.util.jar.Attributes; import java.util.jar.Manifest; @@ -41,6 +42,7 @@ import java.util.stream.Stream; public class QueryCompiler { + private static final Logger log = LoggerFactory.getLogger(QueryCompiler.class); /** * We pick a number just shy of 65536, leaving a little elbow room for good luck. @@ -56,9 +58,9 @@ public class QueryCompiler { private static final long CODEGEN_TIMEOUT_MS_DEFAULT = TimeUnit.SECONDS.toMillis(10); // 10 seconds private static final String CODEGEN_LOOP_DELAY_PROP = "QueryCompiler.codegen.retry.delay"; private static final long CODEGEN_LOOP_DELAY_MS_DEFAULT = 100; - private static final long codegenTimeoutMs = + private static final long CODEGEN_TIMEOUT_MS = Configuration.getInstance().getLongWithDefault(CODEGEN_TIMEOUT_PROP, CODEGEN_TIMEOUT_MS_DEFAULT); - private static final long codegenLoopDelayMs = + private static final long CODEGEN_LOOP_DELAY_MS = Configuration.getInstance().getLongWithDefault(CODEGEN_LOOP_DELAY_PROP, CODEGEN_LOOP_DELAY_MS_DEFAULT); private static boolean logEnabled = Configuration.getInstance().getBoolean("QueryCompiler.logEnabledDefault"); @@ -76,7 +78,7 @@ static QueryCompiler createForUnitTests() { return new QueryCompiler(queryCompilerDir.toFile()); } - private final Map>> knownClasses = new HashMap<>(); + private final Map>> knownClasses = new HashMap<>(); private final String[] dynamicPatterns = new String[] {DYNAMIC_GROOVY_CLASS_PREFIX, FORMULA_PREFIX}; @@ -125,7 +127,7 @@ private QueryCompiler( /** * Enables or disables compilation logging. * - * @param logEnabled Whether or not logging should be enabled + * @param logEnabled Whether logging should be enabled * @return The value of {@code logEnabled} before calling this method. */ public static boolean setLogEnabled(boolean logEnabled) { @@ -204,66 +206,131 @@ public File getFakeClassDestination() { } public void setParentClassLoader(final ClassLoader parentClassLoader) { + // noinspection NonAtomicOperationOnVolatileField ucl = new WritableURLClassLoader(ucl.getURLs(), parentClassLoader); } - public final Class compile(String className, String classBody, String packageNameRoot) { - return compile(className, classBody, packageNameRoot, null, Collections.emptyMap()); - } - - public final Class compile(String className, String classBody, String packageNameRoot, - Map> parameterClasses) { - return compile(className, classBody, packageNameRoot, null, parameterClasses); + /** + * Compile a class. + * + * @param request The compilation request + */ + public Class compile(@NotNull final QueryCompilerRequest request) { + final CompletionStageFuture.Resolver> resolver = CompletionStageFutureImpl.make(); + compile(request, resolver); + try { + return resolver.getFuture().get(); + } catch (InterruptedException | ExecutionException e) { + throw new UncheckedDeephavenException("Could not compile class", e); + } } - public final Class compile(String className, String classBody, String packageNameRoot, StringBuilder codeLog) { - return compile(className, classBody, packageNameRoot, codeLog, Collections.emptyMap()); + /** + * Compile a class. + * + * @param request The compilation request + */ + public void compile( + @NotNull final QueryCompilerRequest request, + @NotNull final CompletionStageFuture.Resolver> resolver) { + // noinspection unchecked + compile(new QueryCompilerRequest[] {request}, new CompletionStageFuture.Resolver[] {resolver}); } /** - * Compile a class. + * Compiles all requests. * - * @param className Class name - * @param classBody Class body, before update with "$CLASS_NAME$" replacement and package name prefixing - * @param packageNameRoot Package name prefix - * @param codeLog Optional "log" for final class code - * @param parameterClasses Generic parameters, empty if none required - * @return The compiled class + * @param requests The compilation requests */ - public Class compile(@NotNull final String className, - @NotNull final String classBody, - @NotNull final String packageNameRoot, - @Nullable final StringBuilder codeLog, - @NotNull final Map> parameterClasses) { - CompletableFuture> future; - final boolean alreadyExists; + public void compile( + @NotNull final QueryCompilerRequest[] requests, + @NotNull final CompletionStageFuture.Resolver>[] resolvers) { + if (requests.length == 0) { + return; + } + if (requests.length != resolvers.length) { + throw new IllegalArgumentException("Requests and resolvers must be the same length"); + } + + // noinspection unchecked + final CompletionStageFuture>[] allFutures = new CompletionStageFuture[requests.length]; + + final List newRequests = new ArrayList<>(); + final List>> newResolvers = new ArrayList<>(); synchronized (this) { - future = knownClasses.get(classBody); - if (future != null) { - alreadyExists = true; - } else { - future = new CompletableFuture<>(); - knownClasses.put(classBody, future); - alreadyExists = false; + for (int ii = 0; ii < requests.length; ++ii) { + final QueryCompilerRequest request = requests[ii]; + final CompletionStageFuture.Resolver> resolver = resolvers[ii]; + + CompletionStageFuture> future = + knownClasses.putIfAbsent(request.classBody(), resolver.getFuture()); + if (future == null) { + newRequests.add(request); + newResolvers.add(resolver); + future = resolver.getFuture(); + } + allFutures[ii] = future; } } - // Someone else has already made the future. I'll just wait for the answer. - if (alreadyExists) { + /* + * @formatter:off + * 3. try to resolve CFs without compiling; retain next hash to try + * 4. compile all remaining with a single compilation task + * 5. goto step 3 + * 6. probably need Consumer> to fit DhFormulaColumn pattern? (other select columns don't need this) + * @formatter:on + */ + + if (!newResolvers.isEmpty()) { + // It's my job to fulfill the future of these futures. try { - return future.get(); - } catch (InterruptedException | ExecutionException error) { - throw new UncheckedDeephavenException(error); + compileHelper(newRequests, newResolvers); + } catch (RuntimeException e) { + // These failures are not applicable to a single request, so we can't just complete the future and + // leave the failure in the cache. + for (int ii = 0; ii < newRequests.size(); ++ii) { + if (newResolvers.get(ii).completeExceptionally(e)) { + knownClasses.remove(newRequests.get(ii).classBody()); + } + } + throw e; } } - // It's my job to fulfill the future. - try { - return compileHelper(className, classBody, packageNameRoot, codeLog, parameterClasses, future); - } catch (RuntimeException e) { - future.completeExceptionally(e); - throw e; + Error firstError = null; + RuntimeException firstException = null; + for (int ii = 0; ii < requests.length; ++ii) { + try { + resolvers[ii].complete(allFutures[ii].get()); + } catch (InterruptedException | ExecutionException e) { + final RuntimeException err; + if (e instanceof InterruptedException) { + err = new UncheckedDeephavenException("Interrupted while waiting for codegen", e); + } else { + Throwable cause = e.getCause(); + if (cause instanceof Error) { + firstError = (Error) cause; + resolvers[ii].completeExceptionally(cause); + continue; + } else if (cause instanceof RuntimeException) { + err = (RuntimeException) cause; + } else { + err = new UncheckedDeephavenException("Error during codegen", e); + } + } + if (firstException == null) { + firstException = err; + } + resolvers[ii].completeExceptionally(err); + } + } + if (firstError != null) { + throw firstError; + } + if (firstException != null) { + throw firstException; } } @@ -416,89 +483,166 @@ private String getClassPath() { return sb.toString(); } - private Class compileHelper(@NotNull final String className, - @NotNull final String classBody, - @NotNull final String packageNameRoot, - @Nullable final StringBuilder codeLog, - @NotNull final Map> parameterClasses, - @NotNull final CompletableFuture> resultFuture) { + private void compileHelper( + @NotNull final List requests, + @NotNull final List>> resolvers) { final MessageDigest digest; try { digest = MessageDigest.getInstance("SHA-256"); } catch (NoSuchAlgorithmException e) { throw new UncheckedDeephavenException("Unable to create SHA-256 hashing digest", e); } - final String basicHashText = - ByteUtils.byteArrToHex(digest.digest(classBody.getBytes(StandardCharsets.UTF_8))); - - for (int pi = 0; pi < MAX_CLASS_COLLISIONS; ++pi) { - final String packageNameSuffix = "c_" + basicHashText - + (pi == 0 ? "" : ("p" + pi)) - + "v" + JAVA_CLASS_VERSION; - final String packageName = (packageNameRoot.isEmpty() - ? packageNameSuffix - : packageNameRoot + (packageNameRoot.endsWith(".") ? "" : ".") + packageNameSuffix); - final String fqClassName = packageName + "." + className; - - // Ask the classloader to load an existing class with this name. This might: - // 1. Fail to find a class (returning null) - // 2. Find a class whose body has the formula we are looking for - // 3. Find a class whose body has a different formula (hash collision) - Class result = tryLoadClassByFqName(fqClassName, parameterClasses); - if (result == null) { - // Couldn't find one, so try to create it. This might: - // A. succeed - // B. Lose a race to another process on the same file system which is compiling the identical formula - // C. Lose a race to another process on the same file system compiling a different formula that - // happens to have the same hash (same packageName). - // However, regardless of A-C, there will be *some* class being found (i.e. tryLoadClassByFqName won't - // return null). - maybeCreateClass(className, classBody, packageName, fqClassName); - - // We could be running on a screwy filesystem that is slow (e.g. NFS). - // If we wrote a file and can't load it ... then give the filesystem some time. - result = tryLoadClassByFqName(fqClassName, parameterClasses); + + final String[] basicHashText = new String[requests.size()]; + for (int ii = 0; ii < requests.size(); ++ii) { + basicHashText[ii] = ByteUtils.byteArrToHex(digest.digest( + requests.get(ii).classBody().getBytes(StandardCharsets.UTF_8))); + } + + int numCompiled = 0; + final int[] next_pi = new int[requests.size()]; + final boolean[] compiled = new boolean[requests.size()]; + final String[] packageName = new String[requests.size()]; + final String[] fqClassName = new String[requests.size()]; + + while (numCompiled < requests.size()) { + for (int ii = 0; ii < requests.size(); ++ii) { + if (compiled[ii]) { + continue; + } + + while (true) { + final int pi = next_pi[ii]++; + final String packageNameSuffix = "c_" + basicHashText[ii] + + (pi == 0 ? "" : ("p" + pi)) + + "v" + JAVA_CLASS_VERSION; + + final QueryCompilerRequest request = requests.get(ii); + if (pi >= MAX_CLASS_COLLISIONS) { + Exception err = new IllegalStateException("Found too many collisions for package name root " + + request.packageNameRoot() + ", class name=" + request.className() + ", class body " + + "hash=" + basicHashText[ii] + " - contact Deephaven support!"); + resolvers.get(ii).completeExceptionally(err); + compiled[ii] = true; + ++numCompiled; + break; + } + + packageName[ii] = request.getPackageName(packageNameSuffix); + fqClassName[ii] = packageName[ii] + "." + request.className(); + + // Ask the classloader to load an existing class with this name. This might: + // 1. Fail to find a class (returning null) + // 2. Find a class whose body has the formula we are looking for + // 3. Find a class whose body has a different formula (hash collision) + Class result = tryLoadClassByFqName(fqClassName[ii], request.parameterClasses()); + if (result == null) { + break; // we'll try to compile it + } + + if (completeIfResultMatchesQueryCompilerRequest(packageName[ii], request, resolvers.get(ii), + result)) { + compiled[ii] = true; + ++numCompiled; + break; + } + } + } + + if (numCompiled == requests.size()) { + return; + } + + // Couldn't resolve at least one of the requests, so try a round of compilation. + final CompilationRequestAttempt[] compilationRequestAttempts = + new CompilationRequestAttempt[requests.size() - numCompiled]; + for (int ii = 0, jj = 0; ii < requests.size(); ++ii) { + if (!compiled[ii]) { + final QueryCompilerRequest request = requests.get(ii); + compilationRequestAttempts[jj++] = new CompilationRequestAttempt( + request, + packageName[ii], + fqClassName[ii], + resolvers.get(ii)); + } + } + + maybeCreateClass(compilationRequestAttempts); + + // We could be running on a screwy filesystem that is slow (e.g. NFS). If we wrote a file and can't load it + // ... then give the filesystem some time. All requests should use the same deadline. + final long deadline = System.currentTimeMillis() + CODEGEN_TIMEOUT_MS - CODEGEN_LOOP_DELAY_MS; + for (int ii = 0; ii < requests.size(); ++ii) { + if (compiled[ii]) { + continue; + } + + final QueryCompilerRequest request = requests.get(ii); + final CompletionStageFuture.Resolver> resolver = resolvers.get(ii); + if (resolver.getFuture().isDone()) { + compiled[ii] = true; + ++numCompiled; + continue; + } + + // This request may have: + // A. succeeded + // B. Lost a race to another process on the same file system which is compiling the identical formula + // C. Lost a race to another process on the same file system compiling a different formula that collides + + Class clazz = tryLoadClassByFqName(fqClassName[ii], request.parameterClasses()); try { - final long deadline = System.currentTimeMillis() + codegenTimeoutMs - codegenLoopDelayMs; - while (result == null && System.currentTimeMillis() < deadline) { - Thread.sleep(codegenLoopDelayMs); - result = tryLoadClassByFqName(fqClassName, parameterClasses); + while (clazz == null && System.currentTimeMillis() < deadline) { + // noinspection BusyWait + Thread.sleep(CODEGEN_LOOP_DELAY_MS); + clazz = tryLoadClassByFqName(fqClassName[ii], request.parameterClasses()); } - } catch (InterruptedException ignored) { - // we got interrupted, just quit looping and ignore it. + } catch (final InterruptedException ie) { + throw new UncheckedDeephavenException("Interrupted while waiting for codegen", ie); } - if (result == null) { + // However, regardless of A-C, there will be *some* class being found + if (clazz == null) { throw new IllegalStateException("Should have been able to load *some* class here"); } - } - final String identifyingFieldValue = loadIdentifyingField(result); - - // If the class we found was indeed the class we were looking for, then complete the future and return it. - if (classBody.equals(identifyingFieldValue)) { - if (codeLog != null) { - // If the caller wants a textual copy of the code we either made, or just found in the cache. - codeLog.append(makeFinalCode(className, classBody, packageName)); - } - resultFuture.complete(result); - synchronized (this) { - // Note we are doing something kind of subtle here. We are removing an entry whose key was matched - // by value equality and replacing it with a value-equal but reference-different string that is a - // static member of the class we just loaded. This should be easier on the garbage collector because - // we are replacing a calculated value with a classloaded value and so in effect we are - // "canonicalizing" the string. This is important because these long strings stay in knownClasses - // forever. - knownClasses.remove(identifyingFieldValue); - knownClasses.put(identifyingFieldValue, resultFuture); - } - return result; + if (completeIfResultMatchesQueryCompilerRequest(packageName[ii], request, resolver, clazz)) { + compiled[ii] = true; + ++numCompiled; + } } - // Try the next hash name } - throw new IllegalStateException("Found too many collisions for package name root " + packageNameRoot - + ", class name=" + className - + ", class body hash=" + basicHashText + " - contact Deephaven support!"); + } + + private boolean completeIfResultMatchesQueryCompilerRequest( + final String packageName, + final QueryCompilerRequest request, + final CompletionStageFuture.Resolver> resolver, + final Class result) { + final String identifyingFieldValue = loadIdentifyingField(result); + if (!request.classBody().equals(identifyingFieldValue)) { + return false; + } + + // If the caller wants a textual copy of the code we either made, or just found in the cache. + request.codeLog() + .ifPresent(sb -> sb.append(makeFinalCode(request.className(), request.classBody(), packageName))); + + // If the class we found was indeed the class we were looking for, then complete the future and return it. + resolver.complete(result); + + synchronized (this) { + // Note we are doing something kind of subtle here. We are removing an entry whose key was matched + // by value equality and replacing it with a value-equal but reference-different string that is a + // static member of the class we just loaded. This should be easier on the garbage collector because + // we are replacing a calculated value with a classloaded value and so in effect we are + // "canonicalizing" the string. This is important because these long strings stay in knownClasses + // forever. + knownClasses.remove(identifyingFieldValue); + knownClasses.put(identifyingFieldValue, resolver.getFuture()); + } + + return true; } private Class tryLoadClassByFqName(String fqClassName, Map> parameterClasses) { @@ -589,11 +733,19 @@ private static int calcBytesConsumed(final char ch) { } private static class JavaSourceFromString extends SimpleJavaFileObject { + final String description; final String code; + final CompletionStageFuture.Resolver> resolver; - JavaSourceFromString(String name, String code) { + JavaSourceFromString( + final String description, + final String name, + final String code, + final CompletionStageFuture.Resolver> resolver) { super(URI.create("string:///" + name.replace('.', '/') + Kind.SOURCE.extension), Kind.SOURCE); + this.description = description; this.code = code; + this.resolver = resolver; } public CharSequence getCharContent(boolean ignoreEncodingErrors) { @@ -601,56 +753,59 @@ public CharSequence getCharContent(boolean ignoreEncodingErrors) { } } - private static class JavaSourceFromFile extends SimpleJavaFileObject { - private static final int JAVA_LENGTH = Kind.SOURCE.extension.length(); - final String code; - - private JavaSourceFromFile(File basePath, File file) { - super(URI.create("string:///" + createName(basePath, file).replace('.', '/') + Kind.SOURCE.extension), - Kind.SOURCE); - try { - this.code = FileUtils.readTextFile(file); - } catch (IOException e) { - throw new UncheckedIOException(e); + private static class CompilationRequestAttempt { + final String description; + final String fqClassName; + final String finalCode; + final String packageName; + final String[] splitPackageName; + final QueryCompilerRequest request; + final CompletionStageFuture.Resolver> resolver; + + private CompilationRequestAttempt( + @NotNull final QueryCompilerRequest request, + @NotNull final String packageName, + @NotNull final String fqClassName, + @NotNull final CompletionStageFuture.Resolver> resolver) { + this.description = request.description(); + this.fqClassName = fqClassName; + this.resolver = resolver; + this.packageName = packageName; + this.request = request; + + finalCode = makeFinalCode(request.className(), request.classBody(), packageName); + + if (logEnabled) { + log.info().append("Generating code ").append(finalCode).endl(); } - } - private static String createName(File basePath, File file) { - final String base = basePath.getAbsolutePath(); - final String fileName = file.getAbsolutePath(); - if (!fileName.startsWith(base)) { - throw new IllegalArgumentException(file + " is not in " + basePath); - } - final String basename = fileName.substring(base.length()); - if (basename.endsWith(".java")) { - return basename.substring(0, basename.length() - JAVA_LENGTH); - } else { - return basename; + splitPackageName = packageName.split("\\."); + if (splitPackageName.length == 0) { + final Exception err = new UncheckedDeephavenException(String.format( + "packageName %s expected to have at least one .", packageName)); + resolver.completeExceptionally(err); } } - @Override - public CharSequence getCharContent(boolean ignoreEncodingErrors) { - return code; - } - } - - private void maybeCreateClass(String className, String code, String packageName, String fqClassName) { - final String finalCode = makeFinalCode(className, code, packageName); + public void ensureDirectories(@NotNull final String rootPath) { + if (splitPackageName.length == 0) { + // we've already failed + return; + } - if (logEnabled) { - log.info().append("Generating code ").append(finalCode).endl(); + final String[] truncatedSplitPackageName = Arrays.copyOf(splitPackageName, splitPackageName.length - 1); + final Path rootPathWithPackage = Paths.get(rootPath, truncatedSplitPackageName); + final File rpf = rootPathWithPackage.toFile(); + QueryCompiler.ensureDirectories(rpf, () -> "Couldn't create package directories: " + rootPathWithPackage); } - final File ctxClassDestination = getClassDestination(); - - final String[] splitPackageName = packageName.split("\\."); - if (splitPackageName.length == 0) { - throw new UncheckedDeephavenException(String.format( - "packageName %s expected to have at least one .", packageName)); + public JavaSourceFromString makeSource() { + return new JavaSourceFromString(description, fqClassName, finalCode, resolver); } - final String[] truncatedSplitPackageName = Arrays.copyOf(splitPackageName, splitPackageName.length - 1); + } + private void maybeCreateClass( + @NotNull final CompilationRequestAttempt[] requests) { // Get the destination root directory (e.g. /tmp/workspace/cache/classes) and populate it with the package // directories (e.g. io/deephaven/test) if they are not already there. This will be useful later. // Also create a temp directory e.g. /tmp/workspace/cache/classes/temporaryCompilationDirectory12345 @@ -663,125 +818,167 @@ private void maybeCreateClass(String className, String code, String packageName, final String rootPathAsString; final String tempDirAsString; try { - rootPathAsString = ctxClassDestination.getAbsolutePath(); - final Path rootPathWithPackage = Paths.get(rootPathAsString, truncatedSplitPackageName); - final File rpf = rootPathWithPackage.toFile(); - ensureDirectories(rpf, () -> "Couldn't create package directories: " + rootPathWithPackage); + rootPathAsString = getClassDestination().getAbsolutePath(); final Path tempPath = Files.createTempDirectory(Paths.get(rootPathAsString), "temporaryCompilationDirectory"); tempDirAsString = tempPath.toFile().getAbsolutePath(); - } catch (IOException ioe) { - throw new UncheckedIOException(ioe); - } - try { - maybeCreateClassHelper(fqClassName, finalCode, splitPackageName, rootPathAsString, tempDirAsString); - } finally { - try { - FileUtils.deleteRecursively(new File(tempDirAsString)); - } catch (Exception e) { - // ignore errors here + for (final CompilationRequestAttempt request : requests) { + request.ensureDirectories(rootPathAsString); } + } catch (IOException ioe) { + Exception err = new UncheckedIOException(ioe); + for (final CompilationRequestAttempt request : requests) { + request.resolver.completeExceptionally(err); + } + return; } - } - private void maybeCreateClassHelper(String fqClassName, String finalCode, String[] splitPackageName, - String rootPathAsString, String tempDirAsString) { - final StringWriter compilerOutput = new StringWriter(); final JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); if (compiler == null) { throw new UncheckedDeephavenException("No Java compiler provided - are you using a JRE instead of a JDK?"); } - final String classPathAsString = getClassPath() + File.pathSeparator + getJavaClassPath(); - final List compilerOptions = Arrays.asList("-d", tempDirAsString, "-cp", classPathAsString); - - final JavaFileManager fileManager = compiler.getStandardFileManager(null, null, null); + final JavaFileManager fileManager = new SynchronizedJavaFileManager( + compiler.getStandardFileManager(null, null, null)); - boolean result = false; - boolean exceptionThrown = false; + boolean exceptionCaught = false; try { - result = compiler.getTask(compilerOutput, - fileManager, - null, - compilerOptions, - null, - Collections.singletonList(new JavaSourceFromString(fqClassName, finalCode))) - .call(); - } catch (final Throwable err) { - exceptionThrown = true; - throw err; + final OperationInitializer operationInitializer = ExecutionContext.getContext().getOperationInitializer(); + int parallelismFactor = operationInitializer.parallelismFactor(); + + int requestsPerTask = Math.max(32, (requests.length + parallelismFactor - 1) / parallelismFactor); + log.info().append("Compiling with parallelismFactor = ").append(parallelismFactor) + .append(" requestsPerTask = ").append(requestsPerTask).endl(); + if (parallelismFactor == 1 || requestsPerTask >= requests.length) { + maybeCreateClassHelper(compiler, fileManager, requests, rootPathAsString, tempDirAsString, + 0, requests.length, false); + } else { + int numTasks = (requests.length + requestsPerTask - 1) / requestsPerTask; + final Future[] tasks = new Future[numTasks]; + for (int jobId = 0; jobId < numTasks; ++jobId) { + final int startInclusive = jobId * requestsPerTask; + final int endExclusive = Math.min(requests.length, (jobId + 1) * requestsPerTask); + tasks[jobId] = operationInitializer.submit(() -> { + maybeCreateClassHelper(compiler, fileManager, requests, rootPathAsString, tempDirAsString, + startInclusive, endExclusive, false); + }); + } + for (int jobId = 0; jobId < numTasks; ++jobId) { + try { + tasks[jobId].get(); + } catch (Exception err) { + throw new UncheckedDeephavenException("Exception waiting for compilation task", err); + } + } + } + } catch (final Throwable t) { + exceptionCaught = true; + throw t; } finally { + try { + FileUtils.deleteRecursively(new File(tempDirAsString)); + } catch (Exception e) { + // ignore errors here + } + try { fileManager.close(); - } catch (final IOException ioe) { - if (!exceptionThrown) { + } catch (IOException ioe) { + if (!exceptionCaught) { // noinspection ThrowFromFinallyBlock throw new UncheckedIOException("Could not close JavaFileManager", ioe); } } } - if (!result) { - throw new UncheckedDeephavenException("Error compiling class " + fqClassName + ":\n" + compilerOutput); + } + + private void maybeCreateClassHelper( + @NotNull final JavaCompiler compiler, + @NotNull final JavaFileManager fileManager, + @NotNull final CompilationRequestAttempt[] requests, + @NotNull final String rootPathAsString, + @NotNull final String tempDirAsString, + final int startInclusive, + final int endExclusive, + final boolean isRetry) { + final StringWriter compilerOutput = new StringWriter(); + + final String classPathAsString = getClassPath() + File.pathSeparator + getJavaClassPath(); + final List compilerOptions = Arrays.asList( + "-d", tempDirAsString, + "-cp", classPathAsString, + // this option allows the compiler to attempt to process all source files even if some of them fail + "--should-stop=ifError=GENERATE"); + + final MutableInt numFailures = new MutableInt(0); + compiler.getTask(compilerOutput, + fileManager, + diagnostic -> { + if (diagnostic.getKind() != Diagnostic.Kind.ERROR) { + return; + } + + final JavaSourceFromString source = (JavaSourceFromString) diagnostic.getSource(); + final UncheckedDeephavenException err = new UncheckedDeephavenException("Error Compiling " + + source.description + "\n" + diagnostic.getMessage(Locale.getDefault())); + if (source.resolver.completeExceptionally(err)) { + // only count the first failure for each source + numFailures.increment(); + } + }, + compilerOptions, + null, + Arrays.stream(requests, startInclusive, endExclusive) + .map(CompilationRequestAttempt::makeSource) + .collect(Collectors.toList())) + .call(); + + final List shouldRetry; + if (!isRetry && numFailures.intValue() > 0 && numFailures.intValue() != endExclusive - startInclusive) { + // if this is the first attempt, and we had some failures, but not all of them failed, then we should retry + shouldRetry = new ArrayList<>(); + } else { + shouldRetry = null; } + // The above has compiled into e.g. // /tmp/workspace/cache/classes/temporaryCompilationDirectory12345/io/deephaven/test/cm12862183232603186v52_0/{various // class files} // We want to atomically move it to e.g. // /tmp/workspace/cache/classes/io/deephaven/test/cm12862183232603186v52_0/{various class files} - Path srcDir = Paths.get(tempDirAsString, splitPackageName); - Path destDir = Paths.get(rootPathAsString, splitPackageName); - try { - Files.move(srcDir, destDir, StandardCopyOption.ATOMIC_MOVE); - } catch (IOException ioe) { - // The move might have failed for a variety of bad reasons. However, if the reason was because - // we lost the race to some other process, that's a harmless/desirable outcome, and we can ignore - // it. - if (!Files.exists(destDir)) { - throw new UncheckedIOException("Move failed for some reason other than destination already existing", - ioe); - } - } - } - - /** - * Try to compile the set of files, returning a pair of success and compiler output. - * - * @param basePath the base path for the java classes - * @param javaFiles the java source files - * @return a Pair of success, and the compiler output - */ - private Pair tryCompile(File basePath, Collection javaFiles) throws IOException { - final JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); - if (compiler == null) { - throw new UncheckedDeephavenException("No Java compiler provided - are you using a JRE instead of a JDK?"); - } - - final File outputDirectory = Files.createTempDirectory("temporaryCompilationDirectory").toFile(); - - try { - final StringWriter compilerOutput = new StringWriter(); - final String javaClasspath = getJavaClassPath(); - - final Collection javaFileObjects = javaFiles.stream() - .map(f -> new JavaSourceFromFile(basePath, f)).collect(Collectors.toList()); + Arrays.stream(requests, startInclusive, endExclusive).forEach(request -> { + final Path srcDir = Paths.get(tempDirAsString, request.splitPackageName); + final Path destDir = Paths.get(rootPathAsString, request.splitPackageName); + try { + Files.move(srcDir, destDir, StandardCopyOption.ATOMIC_MOVE); + } catch (IOException ioe) { + if (shouldRetry != null && !Files.exists(srcDir) && !request.resolver.getFuture().isDone()) { + // This source actually succeeded in compiling, but was not written because some other source failed + // to compile. Let's recursively call ourselves to try again. + shouldRetry.add(request); + return; + } - final boolean result = compiler.getTask(compilerOutput, null, null, - Arrays.asList("-d", outputDirectory.getAbsolutePath(), "-cp", - getClassPath() + File.pathSeparator + javaClasspath), - null, javaFileObjects).call(); + if (!Files.exists(destDir) && !request.resolver.getFuture().isDone()) { + // The move might have failed for a variety of bad reasons. However, if the reason was because + // we lost the race to some other process, that's a harmless/desirable outcome, and we can ignore + // it. + request.resolver.completeExceptionally(new UncheckedIOException( + "Move failed for some reason other than destination already existing", ioe)); + } + } + }); - return new Pair<>(result, compilerOutput.toString()); - } finally { - FileUtils.deleteRecursively(outputDirectory); + if (shouldRetry != null && !shouldRetry.isEmpty()) { + maybeCreateClassHelper(compiler, fileManager, shouldRetry.toArray(CompilationRequestAttempt[]::new), + rootPathAsString, tempDirAsString, 0, shouldRetry.size(), true); } } /** - * Retrieve the java class path from our existing Java class path, and IntelliJ/TeamCity environment variables. - * - * @return + * @return the java class path from our existing Java class path, and IntelliJ/TeamCity environment variables */ private static String getJavaClassPath() { String javaClasspath; @@ -792,14 +989,17 @@ private static String getJavaClassPath() { if (teamCityWorkDir != null) { // We are running in TeamCity, get the classpath differently final File[] classDirs = new File(teamCityWorkDir + "/_out_/classes").listFiles(); - - for (File f : classDirs) { - javaClasspathBuilder.append(File.pathSeparator).append(f.getAbsolutePath()); + if (classDirs != null) { + for (File f : classDirs) { + javaClasspathBuilder.append(File.pathSeparator).append(f.getAbsolutePath()); + } } - final File[] testDirs = new File(teamCityWorkDir + "/_out_/test-classes").listFiles(); - for (File f : testDirs) { - javaClasspathBuilder.append(File.pathSeparator).append(f.getAbsolutePath()); + final File[] testDirs = new File(teamCityWorkDir + "/_out_/test-classes").listFiles(); + if (testDirs != null) { + for (File f : testDirs) { + javaClasspathBuilder.append(File.pathSeparator).append(f.getAbsolutePath()); + } } final File[] jars = FileUtils.findAllFiles(new File(teamCityWorkDir + "/lib")); @@ -835,7 +1035,7 @@ private static String getJavaClassPath() { // use the default path separator final String filePaths = Stream.of(extendedClassPath.split("file:/")) .map(String::trim) - .filter(fileName -> fileName.length() > 0) + .filter(fileName -> !fileName.isEmpty()) .collect(Collectors.joining(File.pathSeparator)); // Remove the classpath jar in question, and expand it with the files from the manifest diff --git a/engine/context/src/main/java/io/deephaven/engine/context/QueryCompilerRequest.java b/engine/context/src/main/java/io/deephaven/engine/context/QueryCompilerRequest.java new file mode 100644 index 00000000000..38be0b97fe4 --- /dev/null +++ b/engine/context/src/main/java/io/deephaven/engine/context/QueryCompilerRequest.java @@ -0,0 +1,72 @@ +/** + * Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending + */ +package io.deephaven.engine.context; + +import io.deephaven.annotations.BuildableStyle; +import org.immutables.value.Value.Immutable; + +import java.util.Map; +import java.util.Optional; + +/** + * A request to compile a java class. + */ +@Immutable +@BuildableStyle +public abstract class QueryCompilerRequest { + public static Builder builder() { + return ImmutableQueryCompilerRequest.builder(); + } + + /** + * @return the description to add to the query performance recorder nugget for this request + */ + public abstract String description(); + + /** + * @return the class name to use for the generated class + */ + public abstract String className(); + + /** + * @return the class body, before update with "$CLASS_NAME$" replacement and package name prefixing + */ + public abstract String classBody(); + + /** + * @return the package name prefix + */ + public abstract String packageNameRoot(); + + /** Optional "log" for final class code. */ + public abstract Optional codeLog(); + + /** + * @return the generic parameters, empty if none required + */ + public abstract Map> parameterClasses(); + + String getPackageName(final String packageNameSuffix) { + final String root = packageNameRoot(); + return root.isEmpty() + ? packageNameSuffix + : root + (root.endsWith(".") ? "" : ".") + packageNameSuffix; + } + + public interface Builder { + Builder description(String description); + + Builder className(String className); + + Builder classBody(String classBody); + + Builder packageNameRoot(String packageNameRoot); + + Builder codeLog(StringBuilder codeLog); + + Builder putAllParameterClasses(Map> parameterClasses); + + QueryCompilerRequest build(); + } +} diff --git a/engine/context/src/main/java/io/deephaven/engine/context/util/SynchronizedJavaFileManager.java b/engine/context/src/main/java/io/deephaven/engine/context/util/SynchronizedJavaFileManager.java new file mode 100644 index 00000000000..80a4d0c1667 --- /dev/null +++ b/engine/context/src/main/java/io/deephaven/engine/context/util/SynchronizedJavaFileManager.java @@ -0,0 +1,132 @@ +package io.deephaven.engine.context.util; + +import javax.tools.FileObject; +import javax.tools.JavaFileManager; +import javax.tools.JavaFileObject; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Iterator; +import java.util.ServiceLoader; +import java.util.Set; + +public class SynchronizedJavaFileManager implements JavaFileManager { + + private final JavaFileManager delegate; + + public SynchronizedJavaFileManager(JavaFileManager delegate) { + this.delegate = delegate; + } + + @Override + public synchronized ClassLoader getClassLoader(Location location) { + return delegate.getClassLoader(location); + } + + @Override + public synchronized Iterable list( + Location location, + String packageName, + Set kinds, + boolean recurse) throws IOException { + return delegate.list(location, packageName, kinds, recurse); + } + + @Override + public synchronized String inferBinaryName(Location location, JavaFileObject file) { + return delegate.inferBinaryName(location, file); + } + + @Override + public synchronized boolean isSameFile(FileObject a, FileObject b) { + return delegate.isSameFile(a, b); + } + + @Override + public synchronized boolean handleOption(String current, Iterator remaining) { + return delegate.handleOption(current, remaining); + } + + @Override + public synchronized boolean hasLocation(Location location) { + return delegate.hasLocation(location); + } + + @Override + public synchronized JavaFileObject getJavaFileForInput( + Location location, + String className, + JavaFileObject.Kind kind) throws IOException { + return delegate.getJavaFileForInput(location, className, kind); + } + + @Override + public synchronized JavaFileObject getJavaFileForOutput( + Location location, + String className, + JavaFileObject.Kind kind, + FileObject sibling) throws IOException { + return delegate.getJavaFileForOutput(location, className, kind, sibling); + } + + @Override + public synchronized FileObject getFileForInput( + Location location, + String packageName, + String relativeName) throws IOException { + return delegate.getFileForInput(location, packageName, relativeName); + } + + @Override + public synchronized FileObject getFileForOutput( + Location location, + String packageName, + String relativeName, + FileObject sibling) throws IOException { + return delegate.getFileForOutput(location, packageName, relativeName, sibling); + } + + @Override + public synchronized void flush() throws IOException { + delegate.flush(); + } + + @Override + public synchronized void close() throws IOException { + delegate.close(); + } + + @Override + public synchronized int isSupportedOption(String option) { + return delegate.isSupportedOption(option); + } + + @Override + public synchronized Location getLocationForModule(Location location, String moduleName) throws IOException { + return delegate.getLocationForModule(location, moduleName); + } + + @Override + public synchronized Location getLocationForModule(Location location, JavaFileObject fo) throws IOException { + return delegate.getLocationForModule(location, fo); + } + + @Override + public synchronized ServiceLoader getServiceLoader(Location location, Class service) throws IOException { + return delegate.getServiceLoader(location, service); + } + + @Override + public synchronized String inferModuleName(Location location) throws IOException { + return delegate.inferModuleName(location); + } + + @Override + public synchronized Iterable> listLocationsForModules(Location location) throws IOException { + return delegate.listLocationsForModules(location); + } + + @Override + public synchronized boolean contains(Location location, FileObject fo) throws IOException { + return delegate.contains(location, fo); + } +} diff --git a/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java b/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java index 02374a67bf0..65d08b6f7f1 100644 --- a/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java +++ b/engine/context/src/test/java/io/deephaven/engine/context/TestQueryCompiler.java @@ -3,9 +3,13 @@ */ package io.deephaven.engine.context; +import io.deephaven.UncheckedDeephavenException; +import io.deephaven.base.verify.Assert; import io.deephaven.configuration.Configuration; import io.deephaven.engine.testutil.junit4.EngineCleanup; import io.deephaven.time.DateTimeUtils; +import io.deephaven.util.CompletionStageFuture; +import io.deephaven.util.CompletionStageFutureImpl; import io.deephaven.util.SafeCloseable; import org.junit.After; import org.junit.Before; @@ -18,8 +22,8 @@ import java.time.Instant; import java.time.LocalDateTime; import java.util.ArrayList; -import java.util.Collections; import java.util.List; +import java.util.concurrent.ExecutionException; public class TestQueryCompiler { private final static int NUM_THREADS = 500; @@ -165,7 +169,7 @@ private void sleepIgnoringInterruptions(final long waitMillis) { } } - private void compile(boolean printDetails, final String className) throws Exception { + private void compile(boolean printDetails, final String className) { final long startMillis; if (printDetails) { startMillis = System.currentTimeMillis(); @@ -173,8 +177,13 @@ private void compile(boolean printDetails, final String className) throws Except } else { startMillis = 0; } - ExecutionContext.getContext().getQueryCompiler() - .compile(className, CLASS_CODE, "io.deephaven.temp"); + ExecutionContext.getContext().getQueryCompiler().compile( + QueryCompilerRequest.builder() + .description("Test Compile") + .className(className) + .classBody(CLASS_CODE) + .packageNameRoot("io.deephaven.temp") + .build()); if (printDetails) { final long endMillis = System.currentTimeMillis(); System.out.println(printMillis(endMillis) + ": Thread 0 ending compile: (" + (endMillis - startMillis) @@ -201,8 +210,14 @@ public void testSimpleCompile() throws Exception { "}"); StringBuilder codeLog = new StringBuilder(); - final Class clazz1 = ExecutionContext.getContext().getQueryCompiler() - .compile("Test", program1Text, "com.deephaven.test", codeLog, Collections.emptyMap()); + final Class clazz1 = ExecutionContext.getContext().getQueryCompiler().compile( + QueryCompilerRequest.builder() + .description("Test Compile") + .className("Test") + .classBody(program1Text) + .packageNameRoot("com.deephaven.test") + .codeLog(codeLog) + .build()); final Method m1 = clazz1.getMethod("main", String[].class); Object[] args1 = new Object[] {new String[] {"hello", "there"}}; m1.invoke(null, args1); @@ -224,21 +239,84 @@ public void testCollidingCompile() throws Exception { Thread t = new Thread(() -> { StringBuilder codeLog = new StringBuilder(); try { - final Class clazz1 = ExecutionContext.getContext().getQueryCompiler() - .compile("Test", program1Text, "com.deephaven.test", codeLog, - Collections.emptyMap()); + final Class clazz1 = ExecutionContext.getContext().getQueryCompiler().compile( + QueryCompilerRequest.builder() + .description("Test Compile") + .className("Test") + .classBody(program1Text) + .packageNameRoot("com.deephaven.test") + .codeLog(codeLog) + .build()); final Method m1 = clazz1.getMethod("main", String[].class); Object[] args1 = new Object[] {new String[] {"hello", "there"}}; m1.invoke(null, args1); } catch (Exception e) { - throw new RuntimeException(e); + throw new UncheckedDeephavenException(e); } }); t.start(); threads.add(t); } - for (int i = 0; i < threads.size(); ++i) { - threads.get(i).join(); + for (final Thread thread : threads) { + thread.join(); + } + } + + @Test + public void testMultiCompileWithFailure() throws ExecutionException, InterruptedException { + final String goodProgram = String.join( + "\n", + "public class GoodTest {", + " public static void main (String [] args) {", + " }", + "}"); + final String badProgram = String.join( + "\n", + "public class BadTest {", + " public static void main (String [] args) {", + " }", + "}}"); + + QueryCompilerRequest[] requests = new QueryCompilerRequest[] { + QueryCompilerRequest.builder() + .description("Test Bad Compile") + .className("BadTest") + .classBody(badProgram) + .packageNameRoot("com.deephaven.test") + .build(), + QueryCompilerRequest.builder() + .description("Test Good Compile") + .className("GoodTest") + .classBody(goodProgram) + .packageNameRoot("com.deephaven.test") + .build(), + }; + + // noinspection unchecked + CompletionStageFuture.Resolver>[] resolvers = + (CompletionStageFuture.Resolver>[]) new CompletionStageFuture.Resolver[] { + CompletionStageFutureImpl.make(), + CompletionStageFutureImpl.make(), + }; + + Exception firstErr; + try { + ExecutionContext.getContext().getQueryCompiler().compile(requests, resolvers); + // noinspection DataFlowIssue + throw Assert.statementNeverExecuted(); + } catch (Exception err) { + firstErr = err; + } + + Assert.eqTrue(resolvers[0].getFuture().isDone(), "resolvers[0].getFuture().isDone()"); + Assert.eqTrue(resolvers[1].getFuture().isDone(), "resolvers[0].getFuture().isDone()"); + Assert.neqNull(resolvers[1].getFuture().get(), "resolvers[1].getFuture().get()"); + try { + resolvers[0].getFuture().get(); + // noinspection DataFlowIssue + throw Assert.statementNeverExecuted(); + } catch (ExecutionException err) { + Assert.eq(firstErr, "firstErr", err.getCause(), "err"); } } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/DeferredViewTable.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/DeferredViewTable.java index dc526030bfd..7b8887d51a2 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/DeferredViewTable.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/DeferredViewTable.java @@ -20,6 +20,7 @@ import java.util.*; import java.util.function.Function; +import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -46,16 +47,20 @@ public DeferredViewTable(@NotNull final TableDefinition definition, this.deferredViewColumns = deferredViewColumns == null ? SelectColumn.ZERO_LENGTH_SELECT_COLUMN_ARRAY : deferredViewColumns; final TableDefinition parentDefinition = tableReference.getDefinition(); + final Supplier> variableSupplier = SelectAndViewAnalyzer.newQueryScopeVariableSupplier(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); SelectAndViewAnalyzer.initializeSelectColumns( - parentDefinition.getColumnNameMap(), this.deferredViewColumns); + parentDefinition.getColumnNameMap(), variableSupplier, this.deferredViewColumns, compilationProcessor); this.deferredFilters = deferredFilters == null ? WhereFilter.ZERO_LENGTH_SELECT_FILTER_ARRAY : deferredFilters; for (final WhereFilter sf : this.deferredFilters) { - sf.init(parentDefinition); + sf.init(parentDefinition, variableSupplier, compilationProcessor); if (sf instanceof LivenessReferent && sf.isRefreshing()) { manage((LivenessReferent) sf); setRefreshing(true); } } + compilationProcessor.compile(); // we really only expect one of these things to be set! final boolean haveDrop = this.deferredDropColumns.length > 0; @@ -78,9 +83,13 @@ public DeferredViewTable(@NotNull final TableDefinition definition, @Override public Table where(Filter filter) { final WhereFilter[] whereFilters = WhereFilter.fromInternal(filter); + final Supplier> variableSupplier = SelectAndViewAnalyzer.newQueryScopeVariableSupplier(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); for (WhereFilter f : whereFilters) { - f.init(definition); + f.init(definition, variableSupplier, compilationProcessor); } + compilationProcessor.compile(); return getResultTableWithWhere(whereFilters); } @@ -189,8 +198,11 @@ private PreAndPostFilters applyFilterRenamings(WhereFilter[] filters) { } } + final Supplier> variableSupplier = SelectAndViewAnalyzer.newQueryScopeVariableSupplier(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); for (final WhereFilter filter : filters) { - filter.init(definition); + filter.init(definition, variableSupplier, compilationProcessor); final boolean isPostView = Stream.of(filter.getColumns(), filter.getColumnArrays()) .flatMap(Collection::stream) @@ -220,6 +232,7 @@ private PreAndPostFilters applyFilterRenamings(WhereFilter[] filters) { postViewFilters.add(filter); } } + compilationProcessor.compile(); return new PreAndPostFilters(preViewFilters.toArray(WhereFilter.ZERO_LENGTH_SELECT_FILTER_ARRAY), postViewFilters.toArray(WhereFilter.ZERO_LENGTH_SELECT_FILTER_ARRAY)); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/PartitionAwareSourceTable.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/PartitionAwareSourceTable.java index 166e088657d..8dddb33187b 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/PartitionAwareSourceTable.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/PartitionAwareSourceTable.java @@ -8,6 +8,7 @@ import io.deephaven.api.filter.Filter; import io.deephaven.base.verify.Assert; import io.deephaven.engine.table.*; +import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; import io.deephaven.engine.updategraph.UpdateSourceRegistrar; import io.deephaven.engine.table.impl.perf.QueryPerformanceRecorder; import io.deephaven.engine.util.TableTools; @@ -24,6 +25,7 @@ import java.util.*; import java.util.function.Function; +import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.stream.StreamSupport; @@ -151,16 +153,20 @@ protected TableAndRemainingFilters getWithWhere(WhereFilter... whereFilters) { @Override public Table selectDistinctInternal(Collection columns) { final List selectColumns = Arrays.asList(SelectColumn.from(columns)); + try { + SelectAndViewAnalyzer.initializeSelectColumns(table.getDefinition().getColumnNameMap(), + selectColumns.toArray(SelectColumn[]::new)); + } catch (Exception e) { + return null; + } + + final Set newColumns = new HashSet<>(); for (final SelectColumn selectColumn : selectColumns) { - try { - selectColumn.initDef(getDefinition().getColumnNameMap()); - } catch (Exception e) { - return null; - } - if (!((PartitionAwareSourceTable) table).isValidAgainstColumnPartitionTable(selectColumn.getColumns(), - selectColumn.getColumnArrays())) { + if (!((PartitionAwareSourceTable) table).isValidAgainstColumnPartitionTable( + selectColumn.getColumns(), selectColumn.getColumnArrays(), newColumns)) { return null; } + newColumns.add(selectColumn.getName()); } return table.selectDistinct(selectColumns); } @@ -280,8 +286,11 @@ private Table whereImpl(final WhereFilter[] whereFilters) { Set groupingColumnNames = groupingColumns.stream().map(ColumnDefinition::getName).collect(Collectors.toSet()); + final Supplier> variableSupplier = SelectAndViewAnalyzer.newQueryScopeVariableSupplier(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); for (WhereFilter whereFilter : whereFilters) { - whereFilter.init(definition); + whereFilter.init(definition, variableSupplier, compilationProcessor); List columns = whereFilter.getColumns(); if (whereFilter instanceof ReindexingFilter) { otherFilters.add(whereFilter); @@ -294,6 +303,7 @@ private Table whereImpl(final WhereFilter[] whereFilters) { otherFilters.add(whereFilter); } } + compilationProcessor.compile(); // if there was nothing that actually required the partition, defer the result. if (partitionFilters.isEmpty()) { @@ -323,13 +333,18 @@ private Table whereImpl(final WhereFilter[] whereFilters) { @Override public final Table selectDistinct(Collection columns) { final List selectColumns = Arrays.asList(SelectColumn.from(columns)); + SelectAndViewAnalyzer.initializeSelectColumns( + definition.getColumnNameMap(), selectColumns.toArray(SelectColumn[]::new)); + + final Set newColumns = new HashSet<>(); for (SelectColumn selectColumn : selectColumns) { - selectColumn.initDef(definition.getColumnNameMap()); - if (!isValidAgainstColumnPartitionTable(selectColumn.getColumns(), selectColumn.getColumnArrays())) { + if (!isValidAgainstColumnPartitionTable( + selectColumn.getColumns(), selectColumn.getColumnArrays(), newColumns)) { // Be sure to invoke the super-class version of this method, rather than the array-based one that // delegates to this method. return super.selectDistinct(selectColumns); } + newColumns.add(selectColumn.getName()); } initializeAvailableLocations(); final List existingLocationKeys = @@ -352,15 +367,20 @@ public final Table selectDistinct(Collection columns) { // Needs lazy region allocation. } - private boolean isValidAgainstColumnPartitionTable(@NotNull final Collection columnNames, + private boolean isValidAgainstColumnPartitionTable( + @NotNull final Collection columnNames, @NotNull final Collection columnArrayNames) { - if (columnArrayNames.size() > 0) { - return false; - } - return columnNames.stream().allMatch(partitioningColumnDefinitions::containsKey); + return isValidAgainstColumnPartitionTable(columnNames, columnArrayNames, Collections.emptySet()); } - private boolean isValidAgainstColumnPartitionTable(Collection columns) { - return columns.stream().map(ColumnName::name).allMatch(partitioningColumnDefinitions::containsKey); + private boolean isValidAgainstColumnPartitionTable( + @NotNull final Collection columnNames, + @NotNull final Collection columnArrayNames, + @NotNull final Collection newColumns) { + if (!columnArrayNames.isEmpty()) { + return false; + } + return columnNames.stream().allMatch( + columnName -> partitioningColumnDefinitions.containsKey(columnName) || newColumns.contains(columnName)); } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryCompilerRequestProcessor.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryCompilerRequestProcessor.java new file mode 100644 index 00000000000..20518ba64fe --- /dev/null +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryCompilerRequestProcessor.java @@ -0,0 +1,93 @@ +/** + * Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending + */ +package io.deephaven.engine.table.impl; + +import io.deephaven.engine.context.ExecutionContext; +import io.deephaven.engine.context.QueryCompiler; +import io.deephaven.engine.context.QueryCompilerRequest; +import io.deephaven.engine.table.impl.perf.QueryPerformanceRecorder; +import io.deephaven.util.SafeCloseable; +import io.deephaven.util.CompletionStageFuture; +import io.deephaven.util.CompletionStageFutureImpl; +import org.jetbrains.annotations.NotNull; + +import java.util.ArrayList; +import java.util.List; + +public interface QueryCompilerRequestProcessor { + /** + * Submit a request for compilation. The QueryCompilerRequestProcessor is not required to immediately compile this + * request. + * + * @param request the request to compile + */ + CompletionStageFuture> submit(@NotNull QueryCompilerRequest request); + + /** + * A QueryCompilerRequestProcessor that immediately compiles requests. + */ + class ImmediateProcessor implements QueryCompilerRequestProcessor { + public static final ImmediateProcessor INSTANCE = new ImmediateProcessor(); + + @Override + public CompletionStageFuture> submit(@NotNull final QueryCompilerRequest request) { + final String desc = "Compile " + request.description(); + final CompletionStageFuture.Resolver> resolver = CompletionStageFutureImpl.make(); + try (final SafeCloseable ignored = QueryPerformanceRecorder.getInstance().getCompilationNugget(desc)) { + ExecutionContext.getContext().getQueryCompiler().compile(request, resolver); + } + return resolver.getFuture(); + } + } + + /** + * A QueryCompilerRequestProcessor that batches requests and compiles them all at once. + *

+ * The compile method must be called to actually compile the requests. + */ + class BatchProcessor implements QueryCompilerRequestProcessor { + private final List requests = new ArrayList<>(); + private final List>> resolvers = new ArrayList<>(); + + @Override + public CompletionStageFuture> submit(@NotNull final QueryCompilerRequest request) { + final CompletionStageFuture.Resolver> resolver = CompletionStageFutureImpl.make(); + requests.add(request); + resolvers.add(resolver); + return resolver.getFuture(); + } + + /** + * Compile all the requests that have been submitted. + */ + public void compile() { + if (requests.isEmpty()) { + return; + } + + final String desc; + if (requests.size() == 1) { + desc = "Compile: " + requests.get(0).description(); + } else { + final StringBuilder descriptionBuilder = new StringBuilder(); + descriptionBuilder.append("Batch Compile of ").append(requests.size()).append(" requests:\n"); + for (final QueryCompilerRequest request : requests) { + descriptionBuilder.append('\t').append(request.description()).append('\n'); + } + desc = descriptionBuilder.toString(); + } + + try (final SafeCloseable ignored = QueryPerformanceRecorder.getInstance().getCompilationNugget(desc)) { + final QueryCompiler compiler = ExecutionContext.getContext().getQueryCompiler(); + if (requests.size() == 1) { + compiler.compile(requests.get(0), resolvers.get(0)); + } else { + compiler.compile( + requests.toArray(QueryCompilerRequest[]::new), + resolvers.toArray(CompletionStageFuture.Resolver[]::new)); + } + } + } + } +} diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryTable.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryTable.java index 72c4d20aff0..78dafeff815 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryTable.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryTable.java @@ -3,7 +3,6 @@ */ package io.deephaven.engine.table.impl; -import io.deephaven.UncheckedDeephavenException; import io.deephaven.api.AsOfJoinMatch; import io.deephaven.api.AsOfJoinRule; import io.deephaven.api.ColumnName; @@ -1218,8 +1217,12 @@ private QueryTable whereInternal(final WhereFilter... filters) { List selectFilters = new LinkedList<>(); List>>> shiftColPairs = new LinkedList<>(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); + final Supplier> variableSupplier = + SelectAndViewAnalyzer.newQueryScopeVariableSupplier(); for (final WhereFilter filter : filters) { - filter.init(getDefinition()); + filter.init(getDefinition(), variableSupplier, compilationProcessor); if (filter instanceof AbstractConditionFilter && ((AbstractConditionFilter) filter).hasConstantArrayAccess()) { shiftColPairs.add(((AbstractConditionFilter) filter).getFormulaShiftColPair()); @@ -1227,6 +1230,7 @@ private QueryTable whereInternal(final WhereFilter... filters) { selectFilters.add(filter); } } + compilationProcessor.compile(); if (!shiftColPairs.isEmpty()) { return (QueryTable) ShiftedColumnsFactory.where(this, shiftColPairs, selectFilters); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/RedefinableTable.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/RedefinableTable.java index ed787d1baab..2f554a3b413 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/RedefinableTable.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/RedefinableTable.java @@ -9,6 +9,7 @@ import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.select.*; +import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; import org.jetbrains.annotations.NotNull; import java.util.*; @@ -49,8 +50,12 @@ private Table viewInternal(Collection selectables, boolean final Map> resultColumnsExternal = new LinkedHashMap<>(); final Map> allColumns = new HashMap<>(definition.getColumnNameMap()); boolean simpleRetain = true; + + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); for (final SelectColumn selectColumn : columns) { - List usedColumnNames = selectColumn.initDef(allColumns); + List usedColumnNames = selectColumn.initDef( + allColumns, SelectAndViewAnalyzer.newQueryScopeVariableSupplier(), compilationProcessor); usedColumnNames.addAll(selectColumn.getColumnArrays()); resultColumnsInternal.addAll(usedColumnNames.stream() .filter(usedColumnName -> !resultColumnsExternal.containsKey(usedColumnName)) @@ -66,6 +71,9 @@ private Table viewInternal(Collection selectables, boolean allColumns.put(selectColumn.getName(), columnDef); } + // compile all formulas at once + compilationProcessor.compile(); + TableDefinition newDefExternal = TableDefinition.of( resultColumnsExternal.values().toArray(ColumnDefinition.ZERO_LENGTH_COLUMN_DEFINITION_ARRAY)); if (simpleRetain) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/WouldMatchOperation.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/WouldMatchOperation.java index cb2b169d7d6..09bba0861e2 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/WouldMatchOperation.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/WouldMatchOperation.java @@ -10,6 +10,7 @@ import io.deephaven.engine.rowset.*; import io.deephaven.engine.rowset.RowSetFactory; import io.deephaven.engine.table.*; +import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; import io.deephaven.engine.updategraph.NotificationQueue; import io.deephaven.engine.liveness.LivenessArtifact; import io.deephaven.engine.table.impl.select.WhereFilter; @@ -21,6 +22,7 @@ import org.jetbrains.annotations.Nullable; import java.util.*; +import java.util.function.Supplier; import java.util.stream.Collectors; /** @@ -91,9 +93,18 @@ public Result initialize(boolean usePrev, long beforeClock) { final List dependencies = new ArrayList<>(); final Map> newColumns = new LinkedHashMap<>(parent.getColumnSourceMap()); - matchColumns.forEach(holder -> { - final WhereFilter filter = holder.getFilter(); - filter.init(parent.getDefinition()); + final Supplier> variableSupplier = + SelectAndViewAnalyzer.newQueryScopeVariableSupplier(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); + final WhereFilter[] filters = matchColumns.stream().map(ColumnHolder::getFilter) + .peek(holder -> holder.init(parent.getDefinition(), variableSupplier, compilationProcessor)) + .toArray(WhereFilter[]::new); + compilationProcessor.compile(); + + for (int ii = 0; ii < filters.length; ++ii) { + final ColumnHolder holder = matchColumns.get(ii); + final WhereFilter filter = filters[ii]; final WritableRowSet result = filter.filter(fullRowSet, fullRowSet, parent, usePrev); holder.column = new IndexWrapperColumnSource( holder.getColumnName(), parent, result.toTracking(), filter); @@ -114,7 +125,7 @@ public Result initialize(boolean usePrev, long beforeClock) { if (filter.isRefreshing()) { anyRefreshing.setTrue(); } - }); + } this.resultTable = new QueryTable(parent.getRowSet(), newColumns); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/AggregationProcessor.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/AggregationProcessor.java index 3aef55a0c50..3dd6867fd8d 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/AggregationProcessor.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/AggregationProcessor.java @@ -38,11 +38,8 @@ import io.deephaven.engine.table.ChunkSource; import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.ColumnSource; -import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.*; import io.deephaven.engine.table.Table; -import io.deephaven.engine.table.impl.BaseTable; -import io.deephaven.engine.table.impl.QueryTable; -import io.deephaven.engine.table.impl.TupleSourceFactory; import io.deephaven.engine.table.impl.by.rollup.NullColumns; import io.deephaven.engine.table.impl.by.rollup.RollupAggregation; import io.deephaven.engine.table.impl.by.rollup.RollupAggregationOutputs; @@ -96,6 +93,7 @@ import io.deephaven.engine.table.impl.by.ssmcountdistinct.unique.ShortRollupUniqueOperator; import io.deephaven.engine.table.impl.by.ssmminmax.SsmChunkedMinMaxOperator; import io.deephaven.engine.table.impl.by.ssmpercentile.SsmChunkedPercentileOperator; +import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; import io.deephaven.engine.table.impl.sources.ReinterpretUtils; import io.deephaven.engine.table.impl.ssms.SegmentedSortedMultiSet; import io.deephaven.engine.table.impl.util.freezeby.FreezeByCountOperator; @@ -282,7 +280,13 @@ public AggregationContext makeAggregationContext( @NotNull final String... groupByColumnNames) { switch (type) { case NORMAL: - return new NormalConverter(table, requireStateChangeRecorder, groupByColumnNames).build(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); + final AggregationContext aggContext = new NormalConverter( + table, requireStateChangeRecorder, SelectAndViewAnalyzer.newQueryScopeVariableSupplier(), + compilationProcessor, groupByColumnNames).build(); + compilationProcessor.compile(); + return aggContext; case ROLLUP_BASE: return new RollupBaseConverter(table, requireStateChangeRecorder, groupByColumnNames).build(); case ROLLUP_REAGGREGATED: @@ -664,12 +668,18 @@ final void addWeightedAvgOrSumOperator( * {@link AggregationContext} for standard aggregations. Accumulates state by visiting each aggregation. */ private final class NormalConverter extends Converter { + private final Supplier> queryScopeVariables; + private final QueryCompilerRequestProcessor compilationProcessor; private NormalConverter( @NotNull final Table table, final boolean requireStateChangeRecorder, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor, @NotNull final String... groupByColumnNames) { super(table, requireStateChangeRecorder, groupByColumnNames); + this.queryScopeVariables = queryScopeVariables; + this.compilationProcessor = compilationProcessor; } // ------------------------------------------------------------------------------------------------------------- @@ -744,7 +754,8 @@ public void visit(@NotNull final AggSpecFormula formula) { final GroupByChunkedOperator groupByChunkedOperator = new GroupByChunkedOperator(table, false, null, resultPairs.stream().map(pair -> MatchPair.of((Pair) pair.input())).toArray(MatchPair[]::new)); final FormulaChunkedOperator formulaChunkedOperator = new FormulaChunkedOperator(groupByChunkedOperator, - true, formula.formula(), formula.paramToken(), MatchPair.fromPairs(resultPairs)); + true, formula.formula(), formula.paramToken(), queryScopeVariables, compilationProcessor, + MatchPair.fromPairs(resultPairs)); addNoInputOperator(formulaChunkedOperator); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/FormulaChunkedOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/FormulaChunkedOperator.java index 7f489a060b6..173cb2f693c 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/FormulaChunkedOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/FormulaChunkedOperator.java @@ -13,6 +13,7 @@ import io.deephaven.engine.table.*; import io.deephaven.engine.table.ChunkSource.GetContext; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.FormulaUtil; import io.deephaven.engine.liveness.LivenessReferent; import io.deephaven.engine.table.ModifiedColumnSet; @@ -29,6 +30,7 @@ import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; +import java.util.function.Supplier; import java.util.function.UnaryOperator; import static io.deephaven.engine.table.impl.sources.ArrayBackedColumnSource.BLOCK_SIZE; @@ -69,10 +71,13 @@ class FormulaChunkedOperator implements IterativeChunkedAggregationOperator { * @param columnParamName The token to substitute column names for * @param resultColumnPairs The names for formula input and result columns */ - FormulaChunkedOperator(@NotNull final GroupByChunkedOperator groupBy, + FormulaChunkedOperator( + @NotNull final GroupByChunkedOperator groupBy, final boolean delegateToBy, @NotNull final String formula, @NotNull final String columnParamName, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor, @NotNull final MatchPair... resultColumnPairs) { this.groupBy = groupBy; this.delegateToBy = delegateToBy; @@ -95,9 +100,10 @@ class FormulaChunkedOperator implements IterativeChunkedAggregationOperator { final ColumnDefinition inputColumnDefinition = ColumnDefinition .fromGenericType(inputColumnName, inputColumnSource.getType(), inputColumnSource.getComponentType()); - formulaColumn.initDef(Collections.singletonMap(inputColumnName, inputColumnDefinition)); - // noinspection unchecked - resultColumns[ci] = ArrayBackedColumnSource.getMemoryColumnSource(0, formulaColumn.getReturnedType()); + formulaColumn.initDef(Collections.singletonMap(inputColumnName, inputColumnDefinition), + queryScopeVariables, compilationProcessor); + resultColumns[ci] = ArrayBackedColumnSource.getMemoryColumnSource( + 0, formulaColumn.getReturnedType(), formulaColumn.getReturnedComponentType()); } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/typed/TypedHasherFactory.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/typed/TypedHasherFactory.java index 601ae2fb3bc..178aca03f04 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/by/typed/TypedHasherFactory.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/by/typed/TypedHasherFactory.java @@ -11,6 +11,7 @@ import io.deephaven.chunk.util.hashing.CharChunkHasher; import io.deephaven.configuration.Configuration; import io.deephaven.engine.context.ExecutionContext; +import io.deephaven.engine.context.QueryCompilerRequest; import io.deephaven.engine.rowset.RowSequence; import io.deephaven.engine.rowset.RowSet; import io.deephaven.engine.rowset.RowSetBuilderRandom; @@ -19,6 +20,7 @@ import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.impl.MultiJoinModifiedSlotTracker; import io.deephaven.engine.table.impl.NaturalJoinModifiedSlotTracker; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.asofjoin.RightIncrementalAsOfJoinStateManagerTypedBase; import io.deephaven.engine.table.impl.asofjoin.StaticAsOfJoinStateManagerTypedBase; import io.deephaven.engine.table.impl.asofjoin.TypedAsOfJoinFactory; @@ -566,12 +568,18 @@ public static T make(HasherConfig hasherConfig, ColumnSource[] tableKe final String javaString = Arrays.stream(javaStrings).filter(s -> !s.startsWith("package ")).collect(Collectors.joining("\n")); - final Class clazz = ExecutionContext.getContext().getQueryCompiler().compile(className, javaString, - "io.deephaven.engine.table.impl.by.typed." + hasherConfig.packageMiddle + ".gen"); + final Class clazz = ExecutionContext.getContext().getQueryCompiler().compile(QueryCompilerRequest.builder() + .description("TypedHasherFactory: " + className) + .className(className) + .classBody(javaString) + .packageNameRoot("io.deephaven.engine.table.impl.by.typed." + hasherConfig.packageMiddle + ".gen") + .build()); + if (!hasherConfig.baseClass.isAssignableFrom(clazz)) { throw new IllegalStateException("Generated class is not a " + hasherConfig.baseClass.getCanonicalName()); } + // noinspection unchecked final Class castedClass = (Class) clazz; T retVal; diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/BaseNodeOperationsRecorder.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/BaseNodeOperationsRecorder.java index 7e3da401317..650c47090f5 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/BaseNodeOperationsRecorder.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/BaseNodeOperationsRecorder.java @@ -9,10 +9,7 @@ import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.hierarchical.RollupTable; import io.deephaven.engine.table.hierarchical.TreeTable; -import io.deephaven.engine.table.impl.AbsoluteSortColumnConventions; -import io.deephaven.engine.table.impl.NoSuchColumnException; -import io.deephaven.engine.table.impl.QueryTable; -import io.deephaven.engine.table.impl.TableAdapter; +import io.deephaven.engine.table.impl.*; import io.deephaven.engine.table.impl.select.SelectColumn; import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; import io.deephaven.engine.table.impl.sources.NullValueColumnSource; @@ -281,16 +278,24 @@ private Stream absoluteSelectColumns() { // custom columns in the future. For now, we've plumbed absolute column value sorting via naming // conventions. Note that we simply avoid telling the client about these columns when sending schemas, so we // have no need to drop them post-sort. - return sortColumns.stream() + + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); + + final SelectColumn[] columns = sortColumns.stream() .map(sc -> sc.column().name()) .filter(AbsoluteSortColumnConventions::isAbsoluteColumnName) .map(cn -> { final String baseColumnName = AbsoluteSortColumnConventions.absoluteColumnNameToBaseName(cn); final Selectable selectable = AbsoluteSortColumnConventions.makeSelectable(cn, baseColumnName); final SelectColumn selectColumn = SelectColumn.of(selectable); - selectColumn.initDef(Map.of(baseColumnName, getDefinition().getColumn(baseColumnName))); + selectColumn.initDef(Map.of(baseColumnName, getDefinition().getColumn(baseColumnName)), + SelectAndViewAnalyzer.newQueryScopeVariableSupplier(), compilationProcessor); return selectColumn; - }); + }).toArray(SelectColumn[]::new); + + compilationProcessor.compile(); + return Stream.of(columns); } } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/RollupTableImpl.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/RollupTableImpl.java index 0dc6f9b7be0..aaf0603eb4d 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/RollupTableImpl.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/RollupTableImpl.java @@ -16,12 +16,14 @@ import io.deephaven.engine.table.hierarchical.RollupTable; import io.deephaven.engine.table.impl.BaseTable.CopyAttributeOperation; import io.deephaven.engine.table.impl.NotificationStepSource; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.QueryTable; import io.deephaven.engine.table.impl.SortOperation; import io.deephaven.engine.table.impl.by.AggregationProcessor; import io.deephaven.engine.table.impl.by.AggregationRowLookup; import io.deephaven.engine.table.impl.select.SelectColumn; import io.deephaven.engine.table.impl.select.WhereFilter; +import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; import io.deephaven.engine.table.impl.sources.NullValueColumnSource; import io.deephaven.engine.table.impl.util.RowRedirection; import io.deephaven.util.type.TypeUtils; @@ -32,6 +34,7 @@ import java.util.*; import java.util.function.Function; import java.util.function.LongUnaryOperator; +import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -272,8 +275,11 @@ public static WhereFilter[] initializeAndValidateFilters( @NotNull final Collection filters, @NotNull final Function exceptionFactory) { final WhereFilter[] whereFilters = WhereFilter.from(filters); + final Supplier> variableSupplier = SelectAndViewAnalyzer.newQueryScopeVariableSupplier(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); for (final WhereFilter whereFilter : whereFilters) { - whereFilter.init(source.getDefinition()); + whereFilter.init(source.getDefinition(), variableSupplier, compilationProcessor); final List invalidColumnsUsed = whereFilter.getColumns().stream().map(ColumnName::of) .filter(cn -> !groupByColumns.contains(cn)).map(ColumnName::name).collect(Collectors.toList()); if (!invalidColumnsUsed.isEmpty()) { @@ -287,6 +293,8 @@ public static WhereFilter[] initializeAndValidateFilters( + " may not use column arrays, but uses column arrays from " + whereFilter.getColumnArrays()); } } + compilationProcessor.compile(); + return whereFilters; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeNodeOperationsRecorder.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeNodeOperationsRecorder.java index 875a7322f68..536b142cf5f 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeNodeOperationsRecorder.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeNodeOperationsRecorder.java @@ -6,14 +6,17 @@ import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.hierarchical.TreeTable; import io.deephaven.engine.table.hierarchical.TreeTable.NodeOperationsRecorder; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.SelectColumn; import io.deephaven.engine.table.impl.select.WhereFilter; +import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import java.util.Arrays; import java.util.Collection; import java.util.List; +import java.util.Map; +import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -134,7 +137,16 @@ public Table where(Filter filter) { } private Stream whereFilters() { - return Stream.of(WhereFilter.fromInternal(filter)).peek(wf -> wf.init(getDefinition())); + final Supplier> variableSupplier = + SelectAndViewAnalyzer.newQueryScopeVariableSupplier(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); + final WhereFilter[] filters = WhereFilter.fromInternal(filter); + for (final WhereFilter filter : filters) { + filter.init(getDefinition(), variableSupplier, compilationProcessor); + } + compilationProcessor.compile(); + return Stream.of(filters); } } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeTableFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeTableFilter.java index 9cee5c0eed1..ff887257308 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeTableFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeTableFilter.java @@ -19,6 +19,7 @@ import io.deephaven.engine.table.impl.*; import io.deephaven.engine.table.impl.chunkboxer.ChunkBoxer; import io.deephaven.engine.table.impl.select.WhereFilter; +import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; import io.deephaven.engine.table.impl.sources.ArrayBackedColumnSource; import io.deephaven.engine.table.impl.remote.ConstructSnapshot; import io.deephaven.util.SafeCloseable; @@ -31,6 +32,7 @@ import java.util.*; import java.util.function.Function; +import java.util.function.Supplier; import java.util.stream.Stream; /** @@ -151,7 +153,12 @@ private TreeTableFilter(@NotNull final TreeTableImpl tree, @NotNull final WhereF parentIdColumnName = tree.getParentIdentifierColumn(); sourceRowLookup = tree.getSourceRowLookup(); this.filters = filters; - Arrays.stream(filters).forEach((final WhereFilter filter) -> filter.init(source.getDefinition())); + final Supplier> variableSupplier = SelectAndViewAnalyzer.newQueryScopeVariableSupplier(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); + Arrays.stream(filters).forEach((final WhereFilter filter) -> filter.init(source.getDefinition(), + variableSupplier, compilationProcessor)); + compilationProcessor.compile(); idSource = source.getColumnSource(tree.getIdentifierColumn().name()); parentIdSource = source.getColumnSource(tree.getParentIdentifierColumn().name()); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeTableImpl.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeTableImpl.java index bf56bf038fc..7a151997bfd 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeTableImpl.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeTableImpl.java @@ -13,6 +13,7 @@ import io.deephaven.engine.table.impl.by.AggregationProcessor; import io.deephaven.engine.table.impl.by.AggregationRowLookup; import io.deephaven.engine.table.impl.select.WhereFilter; +import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; import io.deephaven.engine.table.impl.sources.NullValueColumnSource; import org.apache.commons.lang3.mutable.MutableObject; import org.jetbrains.annotations.NotNull; @@ -20,6 +21,7 @@ import java.util.*; import java.util.function.LongUnaryOperator; +import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -151,13 +153,18 @@ public TreeTable withFilter(@NotNull Filter filter) { if (whereFilters.length == 0) { return noopResult(); } + final Supplier> variableSupplier = SelectAndViewAnalyzer.newQueryScopeVariableSupplier(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); final Map> nodeSuitabilityToFilters = Stream.of(whereFilters) - .peek(wf -> wf.init(source.getDefinition())) + .peek(wf -> wf.init(source.getDefinition(), variableSupplier, compilationProcessor)) .collect(Collectors.partitioningBy(wf -> { // Node-level filters have only node-filter columns and use no column arrays return wf.getColumns().stream().map(ColumnName::of).allMatch(nodeFilterColumns::contains) && wf.getColumnArrays().isEmpty(); })); + compilationProcessor.compile(); + final List nodeFilters = nodeSuitabilityToFilters.get(true); final List sourceFilters = nodeSuitabilityToFilters.get(false); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/BaseTableTransformationColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/BaseTableTransformationColumn.java index be54278bcd9..e7f30e70141 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/BaseTableTransformationColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/BaseTableTransformationColumn.java @@ -28,6 +28,12 @@ public final Class getReturnedType() { return Table.class; } + @Override + public Class getReturnedComponentType() { + // Table does not have a component type + return null; + } + @Override public final List getColumnArrays() { return Collections.emptyList(); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/BiTableTransformationColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/BiTableTransformationColumn.java index 796cef1b07b..2b24940749c 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/BiTableTransformationColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/BiTableTransformationColumn.java @@ -11,6 +11,7 @@ import io.deephaven.engine.rowset.RowSequence; import io.deephaven.engine.rowset.TrackingRowSet; import io.deephaven.engine.table.*; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.Formula; import io.deephaven.engine.table.impl.select.SelectColumn; import io.deephaven.engine.table.impl.sources.ViewColumnSource; @@ -21,6 +22,7 @@ import java.util.List; import java.util.Map; import java.util.function.BinaryOperator; +import java.util.function.Supplier; /** * {@link SelectColumn} implementation to wrap transformer functions for {@link PartitionedTable#partitionedTransform @@ -57,7 +59,10 @@ public List initInputs( } @Override - public List initDef(@NotNull final Map> columnDefinitionMap) { + public List initDef( + @NotNull final Map> columnDefinitionMap, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { validateInputColumnDefinition(inputOutputColumnName, columnDefinitionMap); validateInputColumnDefinition(secondInputColumnName, columnDefinitionMap); return getColumns(); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/LongConstantColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/LongConstantColumn.java index 14d476211bf..9e08d4f4d25 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/LongConstantColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/LongConstantColumn.java @@ -10,6 +10,7 @@ import io.deephaven.engine.rowset.TrackingRowSet; import io.deephaven.engine.table.*; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.Formula; import io.deephaven.engine.table.impl.select.SelectColumn; import io.deephaven.engine.table.impl.sources.LongSingleValueSource; @@ -20,6 +21,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.Supplier; /** * {@link SelectColumn} implementation to assign a constant {@code long} value. @@ -44,7 +46,10 @@ public List initInputs( } @Override - public List initDef(@NotNull final Map> columnDefinitionMap) { + public List initDef( + @NotNull final Map> columnDefinitionMap, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { return getColumns(); } @@ -74,6 +79,12 @@ public final Class getReturnedType() { return long.class; } + @Override + public Class getReturnedComponentType() { + // long does not have a component type + return null; + } + @Override public final List getColumnArrays() { return Collections.emptyList(); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/PartitionedTableImpl.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/PartitionedTableImpl.java index f6b39838ad4..16641e9a5fa 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/PartitionedTableImpl.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/PartitionedTableImpl.java @@ -19,13 +19,11 @@ import io.deephaven.engine.rowset.RowSet; import io.deephaven.engine.rowset.RowSetFactory; import io.deephaven.engine.table.*; -import io.deephaven.engine.table.impl.BaseTable; -import io.deephaven.engine.table.impl.MatchPair; -import io.deephaven.engine.table.impl.MemoizedOperationKey; -import io.deephaven.engine.table.impl.QueryTable; +import io.deephaven.engine.table.impl.*; import io.deephaven.engine.table.impl.remote.ConstructSnapshot; import io.deephaven.engine.table.impl.select.MatchFilter; import io.deephaven.engine.table.impl.select.WhereFilter; +import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; import io.deephaven.engine.table.impl.sources.NullValueColumnSource; import io.deephaven.engine.table.impl.sources.UnionSourceManager; import io.deephaven.engine.table.iterators.ChunkedObjectColumnIterator; @@ -42,6 +40,7 @@ import java.lang.ref.WeakReference; import java.util.*; import java.util.function.BinaryOperator; +import java.util.function.Supplier; import java.util.function.UnaryOperator; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -235,10 +234,14 @@ private Map computeSharedAttributes(@NotNull final Iterator filters) { final WhereFilter[] whereFilters = WhereFilter.from(filters); + final Supplier> variableSupplier = SelectAndViewAnalyzer.newQueryScopeVariableSupplier(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); final boolean invalidFilter = Arrays.stream(whereFilters).flatMap((final WhereFilter filter) -> { - filter.init(table.getDefinition()); + filter.init(table.getDefinition(), variableSupplier, compilationProcessor); return Stream.concat(filter.getColumns().stream(), filter.getColumnArrays().stream()); }).anyMatch((final String columnName) -> columnName.equals(constituentColumnName)); + compilationProcessor.compile(); if (invalidFilter) { throw new IllegalArgumentException("Unsupported filter against constituent column " + constituentColumnName + " found in filters: " + filters); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/PartitionedTableProxyImpl.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/PartitionedTableProxyImpl.java index 5bfa7de3160..531ce5d89d0 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/PartitionedTableProxyImpl.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/PartitionedTableProxyImpl.java @@ -32,6 +32,7 @@ import java.util.*; import java.util.concurrent.atomic.AtomicLong; import java.util.function.BinaryOperator; +import java.util.function.Supplier; import java.util.function.UnaryOperator; import java.util.stream.Collectors; @@ -464,9 +465,13 @@ public PartitionedTable.Proxy sort(Collection columnsToSortBy) { public PartitionedTable.Proxy where(Filter filter) { final WhereFilter[] whereFilters = WhereFilter.fromInternal(filter); final TableDefinition definition = target.constituentDefinition(); + final Supplier> variableSupplier = SelectAndViewAnalyzer.newQueryScopeVariableSupplier(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); for (WhereFilter whereFilter : whereFilters) { - whereFilter.init(definition); + whereFilter.init(definition, variableSupplier, compilationProcessor); } + compilationProcessor.compile(); return basicTransform(ct -> ct.where(Filter.and(WhereFilter.copyFrom(whereFilters)))); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/TableTransformationColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/TableTransformationColumn.java index 6a7cb5db1c1..56eaefdcb83 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/TableTransformationColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/partitioned/TableTransformationColumn.java @@ -12,6 +12,7 @@ import io.deephaven.engine.rowset.RowSequence; import io.deephaven.engine.rowset.TrackingRowSet; import io.deephaven.engine.table.*; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.Formula; import io.deephaven.engine.table.impl.select.SelectColumn; import io.deephaven.engine.table.impl.sources.ViewColumnSource; @@ -21,6 +22,7 @@ import java.util.List; import java.util.Map; import java.util.function.Function; +import java.util.function.Supplier; /** * {@link SelectColumn} implementation to wrap transformer functions for {@link PartitionedTable#transform @@ -52,7 +54,10 @@ public List initInputs( } @Override - public List initDef(@NotNull final Map> columnDefinitionMap) { + public List initDef( + @NotNull final Map> columnDefinitionMap, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { validateInputColumnDefinition(inputOutputColumnName, columnDefinitionMap); return getColumns(); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/rangejoin/ValidFloatingPointFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/rangejoin/ValidFloatingPointFilter.java index cfadcba1fd2..6681132edc4 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/rangejoin/ValidFloatingPointFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/rangejoin/ValidFloatingPointFilter.java @@ -14,13 +14,16 @@ import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; import io.deephaven.engine.table.impl.select.WhereFilter; import io.deephaven.engine.table.impl.select.WhereFilterImpl; import org.jetbrains.annotations.NotNull; import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.function.Supplier; import static io.deephaven.util.QueryConstants.NULL_DOUBLE; import static io.deephaven.util.QueryConstants.NULL_FLOAT; @@ -47,7 +50,10 @@ public List getColumnArrays() { } @Override - public void init(@NotNull final TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeSupplier, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { final ColumnDefinition columnDefinition = tableDefinition.getColumn(columnName.name()); if (columnDefinition == null) { throw new IllegalArgumentException(String.format("Missing expected input column %s", diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractConditionFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractConditionFilter.java index 66a8513fa4c..1d142fd58bd 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractConditionFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractConditionFilter.java @@ -3,35 +3,30 @@ */ package io.deephaven.engine.table.impl.select; -import io.deephaven.api.util.NameValidator; import io.deephaven.base.Pair; -import io.deephaven.engine.context.ExecutionContext; -import io.deephaven.engine.context.QueryScope; import io.deephaven.engine.context.QueryScopeParam; import io.deephaven.engine.rowset.RowSet; import io.deephaven.engine.rowset.WritableRowSet; -import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.impl.MatchPair; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.BaseTable; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.lang.QueryLanguageParser; +import io.deephaven.engine.table.impl.select.codegen.FormulaAnalyzer; import io.deephaven.engine.table.impl.select.python.ArgumentsChunked; import io.deephaven.engine.table.impl.select.python.DeephavenCompatibleFunction; import io.deephaven.engine.util.PyCallableWrapperJpyImpl; import io.deephaven.internal.log.LoggerFactory; import io.deephaven.io.logger.Logger; import io.deephaven.time.TimeLiteralReplacedExpression; -import io.deephaven.vector.ObjectVector; import org.jetbrains.annotations.NotNull; import org.jpy.PyObject; import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.ParameterizedType; -import java.lang.reflect.Type; import java.net.MalformedURLException; import java.util.*; -import java.util.function.BiConsumer; +import java.util.function.Supplier; import java.util.stream.Collectors; import static io.deephaven.engine.table.impl.select.DhFormulaColumn.COLUMN_SUFFIX; @@ -80,89 +75,22 @@ public List getColumnArrays() { } @Override - public synchronized void init(TableDefinition tableDefinition) { + public synchronized void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (initialized) { return; } - final Map> possibleVariables = new HashMap<>(); - possibleVariables.put("i", int.class); - possibleVariables.put("ii", long.class); - possibleVariables.put("k", long.class); - - final Map[]> possibleVariableParameterizedTypes = new HashMap<>(); - try { - final QueryScope queryScope = ExecutionContext.getContext().getQueryScope(); - final Map queryScopeVariables = queryScope.toMap( - (name, value) -> NameValidator.isValidQueryParameterName(name)); - for (Map.Entry param : queryScopeVariables.entrySet()) { - possibleVariables.put(param.getKey(), QueryScopeParamTypeUtil.getDeclaredClass(param.getValue())); - Type declaredType = QueryScopeParamTypeUtil.getDeclaredType(param.getValue()); - if (declaredType instanceof ParameterizedType) { - ParameterizedType pt = (ParameterizedType) declaredType; - Class[] paramTypes = Arrays.stream(pt.getActualTypeArguments()) - .map(QueryScopeParamTypeUtil::classFromType) - .toArray(Class[]::new); - possibleVariableParameterizedTypes.put(param.getKey(), paramTypes); - } - } - - final Set columnVariables = new HashSet<>(); - columnVariables.add("i"); - columnVariables.add("ii"); - columnVariables.add("k"); - - final BiConsumer> createColumnMappings = (columnName, column) -> { - final Class vectorType = DhFormulaColumn.getVectorType(column.getDataType()); - - columnVariables.add(columnName); - if (possibleVariables.put(columnName, column.getDataType()) != null) { - possibleVariableParameterizedTypes.remove(columnName); - } - columnVariables.add(columnName + COLUMN_SUFFIX); - if (possibleVariables.put(columnName + COLUMN_SUFFIX, vectorType) != null) { - possibleVariableParameterizedTypes.remove(columnName + COLUMN_SUFFIX); - } - - final Class compType = column.getComponentType(); - if (compType != null && !compType.isPrimitive()) { - possibleVariableParameterizedTypes.put(columnName, new Class[] {compType}); - } - if (vectorType == ObjectVector.class) { - possibleVariableParameterizedTypes.put(columnName + COLUMN_SUFFIX, - new Class[] {column.getDataType()}); - } - }; - - // By default all columns are available to the formula - for (final ColumnDefinition column : tableDefinition.getColumns()) { - createColumnMappings.accept(column.getName(), column); - } - // Overwrite any existing column mapping using the provided renames. - for (final Map.Entry entry : outerToInnerNames.entrySet()) { - final String columnName = entry.getKey(); - final ColumnDefinition column = tableDefinition.getColumn(entry.getValue()); - createColumnMappings.accept(columnName, column); - } - - log.debug("Expression (before) : " + formula); - final TimeLiteralReplacedExpression timeConversionResult = TimeLiteralReplacedExpression.convertExpression(formula); - log.debug("Expression (after time conversion) : " + timeConversionResult.getConvertedFormula()); - - possibleVariables.putAll(timeConversionResult.getNewVariables()); + final QueryLanguageParser.Result result = FormulaAnalyzer.parseFormula( + timeConversionResult, tableDefinition.getColumnNameMap(), outerToInnerNames, + queryScopeVariables.get(), unboxArguments); - final QueryLanguageParser.Result result = new QueryLanguageParser( - timeConversionResult.getConvertedFormula(), - ExecutionContext.getContext().getQueryLibrary().getPackageImports(), - ExecutionContext.getContext().getQueryLibrary().getClassImports(), - ExecutionContext.getContext().getQueryLibrary().getStaticImports(), - possibleVariables, possibleVariableParameterizedTypes, queryScopeVariables, columnVariables, - unboxArguments) - .getResult(); formulaShiftColPair = result.getFormulaShiftColPair(); if (formulaShiftColPair != null) { log.debug("Formula (after shift conversion) : " + formulaShiftColPair.getFirst()); @@ -225,7 +153,7 @@ public synchronized void init(TableDefinition tableDefinition) { final Class resultType = result.getType(); checkReturnType(result, resultType); - generateFilterCode(tableDefinition, timeConversionResult, result); + generateFilterCode(tableDefinition, timeConversionResult, result, compilationProcessor); initialized = true; } } catch (Exception e) { @@ -270,15 +198,19 @@ private void checkAndInitializeVectorization(QueryLanguageParser.Result result, checkReturnType(result, pyCallableWrapper.getReturnType()); for (String variable : result.getVariablesUsed()) { - if (variable.equals("i")) { - usesI = true; - usedColumns.add("i"); - } else if (variable.equals("ii")) { - usesII = true; - usedColumns.add("ii"); - } else if (variable.equals("k")) { - usesK = true; - usedColumns.add("k"); + switch (variable) { + case "i": + usesI = true; + usedColumns.add("i"); + break; + case "ii": + usesII = true; + usedColumns.add("ii"); + break; + case "k": + usesK = true; + usedColumns.add("k"); + break; } } ArgumentsChunked argumentsChunked = pyCallableWrapper.buildArgumentsChunked(usedColumns); @@ -302,9 +234,12 @@ private void checkReturnType(QueryLanguageParser.Result result, Class resultT } } - protected abstract void generateFilterCode(TableDefinition tableDefinition, - TimeLiteralReplacedExpression timeConversionResult, - QueryLanguageParser.Result result) throws MalformedURLException, ClassNotFoundException; + protected abstract void generateFilterCode( + @NotNull TableDefinition tableDefinition, + @NotNull TimeLiteralReplacedExpression timeConversionResult, + @NotNull QueryLanguageParser.Result result, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) + throws MalformedURLException, ClassNotFoundException; @NotNull @Override diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractFormulaColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractFormulaColumn.java index 791be4beaf9..50a182e1e4e 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractFormulaColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractFormulaColumn.java @@ -3,12 +3,14 @@ */ package io.deephaven.engine.table.impl.select; +import io.deephaven.UncheckedDeephavenException; import io.deephaven.base.verify.Require; import io.deephaven.configuration.Configuration; import io.deephaven.engine.table.*; import io.deephaven.engine.context.QueryScopeParam; import io.deephaven.engine.table.impl.BaseTable; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.util.CompletionStageFuture; import io.deephaven.vector.Vector; import io.deephaven.engine.table.impl.vector.*; import io.deephaven.engine.table.impl.select.formula.*; @@ -18,9 +20,14 @@ import io.deephaven.internal.log.LoggerFactory; import io.deephaven.io.logger.Logger; import io.deephaven.api.util.NameValidator; +import org.apache.commons.text.StringEscapeUtils; import org.jetbrains.annotations.NotNull; import java.util.*; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.function.Consumer; /** @@ -34,11 +41,12 @@ public abstract class AbstractFormulaColumn implements FormulaColumn { protected String formulaString; + protected final String originalFormulaString; protected List usedColumns; @NotNull protected final String columnName; - protected FormulaFactory formulaFactory; + protected Future formulaFactory; private Formula formula; protected QueryScopeParam[] params; protected Map> columnSources; @@ -61,6 +69,7 @@ public abstract class AbstractFormulaColumn implements FormulaColumn { */ protected AbstractFormulaColumn(String columnName, String formulaString) { this.formulaString = Require.neqNull(formulaString, "formulaString"); + this.originalFormulaString = formulaString; this.columnName = NameValidator.validateColumnName(columnName); } @@ -69,6 +78,11 @@ public Class getReturnedType() { return returnedType; } + @Override + public Class getReturnedComponentType() { + return returnedType.getComponentType(); + } + @Override public List initInputs( @NotNull final TrackingRowSet rowSet, @@ -79,6 +93,9 @@ public List initInputs( if (usedColumns != null) { return usedColumns; } + + // we'll have to assume that initDef has already been invoked if we could have grouped compilation requests + // otherwise this call will compile immediately if necessary return initDef(extractDefinitions(columnsOfInterest)); } @@ -220,7 +237,15 @@ private ColumnSource getViewColumnSource(boolean lazy) { private Formula getFormula(boolean initLazyMap, Map> columnsToData, QueryScopeParam... params) { - formula = formulaFactory.createFormula(rowSet, initLazyMap, columnsToData, params); + try { + // the future must already be completed or else it is an error + formula = formulaFactory.get(0, TimeUnit.SECONDS).createFormula( + StringEscapeUtils.escapeJava(columnName), rowSet, initLazyMap, columnsToData, params); + } catch (TimeoutException e) { + throw new IllegalStateException("Formula factory not already compiled!"); + } catch (InterruptedException | ExecutionException e) { + throw new UncheckedDeephavenException("Error creating formula for " + columnName, e); + } return formula; } @@ -254,26 +279,29 @@ private static Vector makeAppropriateVectorWrapper(ColumnSource cs, RowSet return new ObjectVectorColumnWrapper<>((ColumnSource) cs, rowSet); } - protected FormulaFactory createKernelFormulaFactory(final FormulaKernelFactory formulaKernelFactory) { + protected Future createKernelFormulaFactory( + @NotNull final CompletionStageFuture formulaKernelFactoryFuture) { final FormulaSourceDescriptor sd = getSourceDescriptor(); - return (rowSet, lazy, columnsToData, params) -> { - // Maybe warn that we ignore "lazy". By the way, "lazy" is the wrong term anyway. "lazy" doesn't mean - // "cached", which is how we are using it. - final Map> netColumnSources = new HashMap<>(); - for (final String columnName : sd.sources) { - final ColumnSource columnSourceToUse = columnsToData.get(columnName); - netColumnSources.put(columnName, columnSourceToUse); - } - - final Vector[] vectors = new Vector[sd.arrays.length]; - for (int ii = 0; ii < sd.arrays.length; ++ii) { - final ColumnSource cs = columnsToData.get(sd.arrays[ii]); - vectors[ii] = makeAppropriateVectorWrapper(cs, rowSet); - } - final FormulaKernel fk = formulaKernelFactory.createInstance(vectors, params); - return new FormulaKernelAdapter(rowSet, sd, netColumnSources, fk); - }; + return formulaKernelFactoryFuture + .thenApply(formulaKernelFactory -> (columnName, rowSet, lazy, columnsToData, params) -> { + // Maybe warn that we ignore "lazy". By the way, "lazy" is the wrong term anyway. "lazy" doesn't + // mean + // "cached", which is how we are using it. + final Map> netColumnSources = new HashMap<>(); + for (final String sourceColumnName : sd.sources) { + final ColumnSource columnSourceToUse = columnsToData.get(sourceColumnName); + netColumnSources.put(sourceColumnName, columnSourceToUse); + } + + final Vector[] vectors = new Vector[sd.arrays.length]; + for (int ii = 0; ii < sd.arrays.length; ++ii) { + final ColumnSource cs = columnsToData.get(sd.arrays[ii]); + vectors[ii] = makeAppropriateVectorWrapper(cs, rowSet); + } + final FormulaKernel fk = formulaKernelFactory.createInstance(vectors, params); + return new FormulaKernelAdapter(rowSet, sd, netColumnSources, fk); + }); } protected abstract FormulaSourceDescriptor getSourceDescriptor(); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractRangeFilter.java index 4260d8fa2ae..14aa3647c61 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractRangeFilter.java @@ -32,13 +32,13 @@ public abstract class AbstractRangeFilter extends WhereFilterImpl { /** * The chunkFilter can be applied to the columns native type. - * + *

* In practice, this is for non-reinterpretable DateTimes. */ ChunkFilter chunkFilter; /** - * If the column can be be reinterpreted to a long, then we should prefer to use the longFilter instead. - * + * If the column can be reinterpreted to a long, then we should prefer to use the longFilter instead. + *

* In practice, this is used for reinterpretable DateTimes. */ ChunkFilter longFilter; diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/BaseIncrementalReleaseFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/BaseIncrementalReleaseFilter.java index 065558b0f8a..678ccfcdec0 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/BaseIncrementalReleaseFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/BaseIncrementalReleaseFilter.java @@ -8,6 +8,7 @@ import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.rowset.RowSet; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.updategraph.NotificationQueue; import io.deephaven.engine.updategraph.UpdateGraph; import io.deephaven.util.QueryConstants; @@ -16,10 +17,12 @@ import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.function.Supplier; /** * Base class for filters that will release more rows of a table on each UGP cycle. - * + *

* The use case is for benchmarks that want to replay a table in order to better understand incremental processing * capacity. */ @@ -67,7 +70,10 @@ public List getColumnArrays() { } @Override - public void init(TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { initialized = true; if (!started) { return; diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ByteRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ByteRangeFilter.java index 0527375f786..5f47376959c 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ByteRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ByteRangeFilter.java @@ -13,6 +13,7 @@ import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.chunkfilter.ByteRangeComparator; import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; import io.deephaven.gui.table.filters.Condition; @@ -21,6 +22,9 @@ import io.deephaven.util.type.TypeUtils; import org.jetbrains.annotations.NotNull; +import java.util.Map; +import java.util.function.Supplier; + public class ByteRangeFilter extends AbstractRangeFilter { public static ByteRangeFilter lt(String columnName, byte x) { return new ByteRangeFilter(columnName, QueryConstants.NULL_BYTE, x, true, false); @@ -68,7 +72,10 @@ static WhereFilter makeByteRangeFilter(String columnName, Condition condition, S } @Override - public void init(TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/CharRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/CharRangeFilter.java index 03a6dab0630..9625628329b 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/CharRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/CharRangeFilter.java @@ -8,6 +8,7 @@ import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.chunkfilter.CharRangeComparator; import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; import io.deephaven.gui.table.filters.Condition; @@ -16,6 +17,9 @@ import io.deephaven.util.type.TypeUtils; import org.jetbrains.annotations.NotNull; +import java.util.Map; +import java.util.function.Supplier; + public class CharRangeFilter extends AbstractRangeFilter { public static CharRangeFilter lt(String columnName, char x) { return new CharRangeFilter(columnName, QueryConstants.NULL_CHAR, x, true, false); @@ -63,7 +67,10 @@ static WhereFilter makeCharRangeFilter(String columnName, Condition condition, S } @Override - public void init(TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ClockFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ClockFilter.java index 400541e7069..751cb991958 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ClockFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ClockFilter.java @@ -11,6 +11,7 @@ import io.deephaven.engine.rowset.RowSetFactory; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.lang.QueryLanguageFunctionUtils; import io.deephaven.engine.table.impl.sources.ReinterpretUtils; import io.deephaven.engine.updategraph.DynamicNode; @@ -24,6 +25,8 @@ import java.time.Instant; import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.function.Supplier; /** * Boilerplate super-class for various clock-oriented filters. @@ -46,7 +49,10 @@ public ClockFilter(@NotNull final String columnName, @NotNull final Clock clock, } @Override - public final void init(@NotNull final TableDefinition tableDefinition) {} + public final void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) {} @Override public final List getColumns() { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ComparableRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ComparableRangeFilter.java index 227d53094fb..de972d0d773 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ComparableRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ComparableRangeFilter.java @@ -6,6 +6,7 @@ import io.deephaven.base.verify.Assert; import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; import io.deephaven.util.compare.ObjectComparisons; import io.deephaven.engine.table.ColumnSource; @@ -17,6 +18,9 @@ import io.deephaven.util.annotations.TestUseOnly; import org.jetbrains.annotations.NotNull; +import java.util.Map; +import java.util.function.Supplier; + public class ComparableRangeFilter extends AbstractRangeFilter { private final Comparable upper; private final Comparable lower; @@ -41,7 +45,10 @@ public static ComparableRangeFilter makeForTest(String columnName, Comparable } @Override - public void init(TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ComposedFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ComposedFilter.java index 4248b6d9208..8446972304e 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ComposedFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ComposedFilter.java @@ -4,12 +4,15 @@ package io.deephaven.engine.table.impl.select; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.updategraph.NotificationQueue; import io.deephaven.engine.liveness.LivenessArtifact; import io.deephaven.engine.table.impl.DependencyStreamProvider; import io.deephaven.util.annotations.TestUseOnly; +import org.jetbrains.annotations.NotNull; import java.util.*; +import java.util.function.Supplier; import java.util.stream.Stream; public abstract class ComposedFilter extends WhereFilterLivenessArtifactImpl implements DependencyStreamProvider { @@ -56,9 +59,12 @@ public List getColumnArrays() { } @Override - public void init(TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { for (WhereFilter filter : componentFilters) { - filter.init(tableDefinition); + filter.init(tableDefinition, queryScopeVariables, compilationProcessor); } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ConditionFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ConditionFilter.java index 3b4f3040ca3..39147a185ba 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ConditionFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ConditionFilter.java @@ -7,6 +7,7 @@ import io.deephaven.chunk.attributes.Any; import io.deephaven.engine.context.QueryCompiler; import io.deephaven.engine.context.ExecutionContext; +import io.deephaven.engine.context.QueryCompilerRequest; import io.deephaven.engine.rowset.RowSetFactory; import io.deephaven.engine.table.Context; import io.deephaven.engine.table.SharedContext; @@ -14,11 +15,11 @@ import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.lang.QueryLanguageParser; import io.deephaven.engine.table.impl.util.codegen.CodeGenerator; import io.deephaven.engine.context.QueryScopeParam; import io.deephaven.time.TimeLiteralReplacedExpression; -import io.deephaven.engine.table.impl.perf.QueryPerformanceRecorder; import io.deephaven.engine.table.ColumnSource; import io.deephaven.chunk.*; import io.deephaven.engine.rowset.chunkattributes.OrderedRowKeys; @@ -34,6 +35,10 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.function.Consumer; import static io.deephaven.engine.table.impl.select.DhFormulaColumn.COLUMN_SUFFIX; @@ -44,7 +49,7 @@ public class ConditionFilter extends AbstractConditionFilter { public static final int CHUNK_SIZE = 4096; - private Class filterKernelClass = null; + private Future> filterKernelClass = null; private List>> usedInputs; // that is columns and special variables private String classBody; private Filter filter = null; @@ -378,46 +383,53 @@ private static String toTitleCase(String input) { @Override protected void generateFilterCode( - TableDefinition tableDefinition, - TimeLiteralReplacedExpression timeConversionResult, - QueryLanguageParser.Result result) { + @NotNull final TableDefinition tableDefinition, + @NotNull final TimeLiteralReplacedExpression timeConversionResult, + @NotNull final QueryLanguageParser.Result result, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { final StringBuilder classBody = getClassBody(tableDefinition, timeConversionResult, result); - if (classBody == null) + if (classBody == null) { return; - try (final SafeCloseable ignored = QueryPerformanceRecorder.getInstance().getCompilationNugget(formula)) { - final List> paramClasses = new ArrayList<>(); - final Consumer> addParamClass = (cls) -> { - if (cls != null) { - paramClasses.add(cls); - } - }; - for (String usedColumn : usedColumns) { - usedColumn = outerToInnerNames.getOrDefault(usedColumn, usedColumn); - final ColumnDefinition column = tableDefinition.getColumn(usedColumn); - addParamClass.accept(column.getDataType()); - addParamClass.accept(column.getComponentType()); - } - for (String usedColumn : usedColumnArrays) { - usedColumn = outerToInnerNames.getOrDefault(usedColumn, usedColumn); - final ColumnDefinition column = tableDefinition.getColumn(usedColumn); - addParamClass.accept(column.getDataType()); - addParamClass.accept(column.getComponentType()); - } - for (final QueryScopeParam param : params) { - addParamClass.accept(QueryScopeParamTypeUtil.getDeclaredClass(param.getValue())); - } - - filterKernelClass = ExecutionContext.getContext().getQueryCompiler() - .compile("GeneratedFilterKernel", this.classBody = classBody.toString(), - QueryCompiler.FORMULA_PREFIX, QueryScopeParamTypeUtil.expandParameterClasses(paramClasses)); } + + final List> paramClasses = new ArrayList<>(); + final Consumer> addParamClass = (cls) -> { + if (cls != null) { + paramClasses.add(cls); + } + }; + for (String usedColumn : usedColumns) { + usedColumn = outerToInnerNames.getOrDefault(usedColumn, usedColumn); + final ColumnDefinition column = tableDefinition.getColumn(usedColumn); + addParamClass.accept(column.getDataType()); + addParamClass.accept(column.getComponentType()); + } + for (String usedColumn : usedColumnArrays) { + usedColumn = outerToInnerNames.getOrDefault(usedColumn, usedColumn); + final ColumnDefinition column = tableDefinition.getColumn(usedColumn); + addParamClass.accept(column.getDataType()); + addParamClass.accept(column.getComponentType()); + } + for (final QueryScopeParam param : params) { + addParamClass.accept(QueryScopeParamTypeUtil.getDeclaredClass(param.getValue())); + } + + this.classBody = classBody.toString(); + + filterKernelClass = compilationProcessor.submit(QueryCompilerRequest.builder() + .description("Filter Expression: " + formula) + .className("GeneratedFilterKernel") + .classBody(this.classBody) + .packageNameRoot(QueryCompiler.FORMULA_PREFIX) + .putAllParameterClasses(QueryScopeParamTypeUtil.expandParameterClasses(paramClasses)) + .build()); } @Nullable private StringBuilder getClassBody( - TableDefinition tableDefinition, - TimeLiteralReplacedExpression timeConversionResult, - QueryLanguageParser.Result result) { + @NotNull final TableDefinition tableDefinition, + @NotNull final TimeLiteralReplacedExpression timeConversionResult, + @NotNull final QueryLanguageParser.Result result) { if (filterKernelClass != null) { return null; } @@ -577,15 +589,23 @@ private StringBuilder getClassBody( protected Filter getFilter(Table table, RowSet fullSet) throws InstantiationException, IllegalAccessException, NoSuchMethodException, InvocationTargetException { if (filter == null) { - final FilterKernel filterKernel = (FilterKernel) filterKernelClass - .getConstructor(Table.class, RowSet.class, QueryScopeParam[].class) - .newInstance(table, fullSet, (Object) params); - final String[] columnNames = usedInputs.stream() - .map(p -> outerToInnerNames.getOrDefault(p.first, p.first)) - .toArray(String[]::new); - filter = new ChunkFilter(filterKernel, columnNames, CHUNK_SIZE); - // note this filter is not valid for use in other contexts, as it captures references from the source table - filterValidForCopy = false; + try { + final FilterKernel filterKernel = (FilterKernel) filterKernelClass + .get(0, TimeUnit.SECONDS) + .getConstructor(Table.class, RowSet.class, QueryScopeParam[].class) + .newInstance(table, fullSet, (Object) params); + final String[] columnNames = usedInputs.stream() + .map(p -> outerToInnerNames.getOrDefault(p.first, p.first)) + .toArray(String[]::new); + filter = new ChunkFilter(filterKernel, columnNames, CHUNK_SIZE); + // note this filter is not valid for use in other contexts, as it captures references from the source + // table + filterValidForCopy = false; + } catch (TimeoutException e) { + throw new IllegalStateException("Formula factory not already compiled!"); + } catch (InterruptedException | ExecutionException e) { + throw new FormulaCompilationException("Formula compilation error for: " + formula, e); + } } return filter; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java index 4a807eb2c79..d9f0bd9bf49 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DhFormulaColumn.java @@ -3,18 +3,20 @@ */ package io.deephaven.engine.table.impl.select; +import io.deephaven.UncheckedDeephavenException; import io.deephaven.base.Pair; import io.deephaven.chunk.ChunkType; import io.deephaven.configuration.Configuration; import io.deephaven.engine.context.ExecutionContext; import io.deephaven.engine.context.QueryCompiler; +import io.deephaven.engine.context.QueryCompilerRequest; import io.deephaven.engine.context.QueryScopeParam; import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.impl.MatchPair; import io.deephaven.engine.table.Table; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.lang.QueryLanguageParser; -import io.deephaven.engine.table.impl.perf.QueryPerformanceRecorder; import io.deephaven.engine.table.impl.select.codegen.FormulaAnalyzer; import io.deephaven.engine.table.impl.select.codegen.JavaKernelBuilder; import io.deephaven.engine.table.impl.select.codegen.RichType; @@ -31,11 +33,10 @@ import io.deephaven.internal.log.LoggerFactory; import io.deephaven.io.logger.Logger; import io.deephaven.time.TimeLiteralReplacedExpression; -import io.deephaven.util.SafeCloseable; +import io.deephaven.util.CompletionStageFuture; import io.deephaven.util.type.TypeUtils; import io.deephaven.vector.ObjectVector; import io.deephaven.vector.Vector; -import org.apache.commons.text.StringEscapeUtils; import org.jetbrains.annotations.NotNull; import org.jpy.PyObject; @@ -43,13 +44,11 @@ import java.math.BigInteger; import java.time.Instant; import java.time.ZonedDateTime; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; +import java.util.concurrent.ExecutionException; import java.util.function.Consumer; import java.util.function.Function; +import java.util.function.Supplier; import static io.deephaven.engine.util.IterableUtils.makeCommaSeparatedList; @@ -182,7 +181,10 @@ public static Class getVectorType(Class declaredType) { } @Override - public List initDef(Map> columnDefinitionMap) { + public List initDef( + @NotNull final Map> columnDefinitionMap, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { if (formulaFactory != null) { validateColumnDefinition(columnDefinitionMap); return formulaColumnPython != null ? formulaColumnPython.usedColumns : usedColumns; @@ -191,8 +193,8 @@ public List initDef(Map> columnDefinitionMap try { final TimeLiteralReplacedExpression timeConversionResult = TimeLiteralReplacedExpression.convertExpression(formulaString); - final QueryLanguageParser.Result result = FormulaAnalyzer.getCompiledFormula(columnDefinitionMap, - timeConversionResult); + final QueryLanguageParser.Result result = FormulaAnalyzer.parseFormula( + timeConversionResult, columnDefinitionMap, Collections.emptyMap(), queryScopeVariables.get(), true); analyzedFormula = FormulaAnalyzer.analyze(formulaString, columnDefinitionMap, timeConversionResult, result); hasConstantValue = result.isConstantValueExpression(); @@ -209,18 +211,23 @@ public List initDef(Map> columnDefinitionMap formulaString = result.getConvertedExpression(); // check if this is a column to be created with a Python vectorizable function - checkAndInitializeVectorization(columnDefinitionMap); + checkAndInitializeVectorization(columnDefinitionMap, queryScopeVariables, compilationRequestProcessor); } catch (Exception e) { - throw new FormulaCompilationException("Formula compilation error for: " + formulaString, e); + throw new FormulaCompilationException("Formula compilation error for: " + originalFormulaString, e); } - formulaFactory = useKernelFormulasProperty - ? createKernelFormulaFactory(getFormulaKernelFactory()) - : createFormulaFactory(); + if (useKernelFormulasProperty) { + formulaFactory = createKernelFormulaFactory(getFormulaKernelFactory(compilationRequestProcessor)); + } else { + compileFormula(compilationRequestProcessor); + } return formulaColumnPython != null ? formulaColumnPython.usedColumns : usedColumns; } - private void checkAndInitializeVectorization(Map> columnDefinitionMap) { + private void checkAndInitializeVectorization( + @NotNull final Map> columnDefinitionMap, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { // noinspection SuspiciousToArrayCall final PyCallableWrapperJpyImpl[] cws = Arrays.stream(params) .filter(p -> p.getValue() instanceof PyCallableWrapperJpyImpl) @@ -241,7 +248,7 @@ private void checkAndInitializeVectorization(Map> co pyCallableWrapper.getReturnType(), this.analyzedFormula.sourceDescriptor.sources, argumentsChunked, true)); - formulaColumnPython.initDef(columnDefinitionMap); + formulaColumnPython.initDef(columnDefinitionMap, queryScopeVariables, compilationRequestProcessor); } } @@ -253,6 +260,7 @@ String generateClassBody() { CodeGenerator.create(ExecutionContext.getContext().getQueryLibrary().getImportStrings().toArray()), "", "public class $CLASSNAME$ extends [[FORMULA_CLASS_NAME]]", CodeGenerator.block( generateFormulaFactoryLambda(), "", + "private final String __columnName;", CodeGenerator.repeated("instanceVar", "private final [[TYPE]] [[NAME]];"), "private final Map [[LAZY_RESULT_CACHE_NAME]];", analyzedFormula.timeInstanceVariables, "", @@ -304,12 +312,14 @@ private CodeGenerator generateFormulaFactoryLambda() { private CodeGenerator generateConstructor() { final CodeGenerator g = CodeGenerator.create( - "public $CLASSNAME$(final TrackingRowSet __rowSet,", CodeGenerator.indent( + "public $CLASSNAME$(final String __columnName,", CodeGenerator.indent( + "final TrackingRowSet __rowSet,", "final boolean __lazy,", "final java.util.Map __columnsToData,", "final [[PARAM_CLASSNAME]]... __params)"), CodeGenerator.block( "super(__rowSet);", + "this.__columnName = __columnName;", CodeGenerator.repeated("initColumn", "[[COLUMN_NAME]] = __columnsToData.get(\"[[COLUMN_NAME]]\");"), CodeGenerator.repeated("initNormalColumnArray", @@ -355,7 +365,7 @@ private CodeGenerator generateApplyFormulaPerItem(final TypeAnalyzer ta) { "try", CodeGenerator.block( "return [[FORMULA_STRING]];"), CodeGenerator.samelineBlock("catch (java.lang.Exception __e)", - "throw new [[EXCEPTION_TYPE]](\"In formula: [[COLUMN_NAME]] = \" + [[JOINED_FORMULA_STRING]], __e);"))); + "throw new [[EXCEPTION_TYPE]](\"In formula: \" + __columnName + \" = \" + [[JOINED_FORMULA_STRING]], __e);"))); g.replace("RETURN_TYPE", ta.typeString); final List args = visitFormulaParameters(n -> n.typeString + " " + n.name, n -> n.typeString + " " + n.name, @@ -363,8 +373,7 @@ private CodeGenerator generateApplyFormulaPerItem(final TypeAnalyzer ta) { null); g.replace("ARGS", makeCommaSeparatedList(args)); g.replace("FORMULA_STRING", ta.wrapWithCastIfNecessary(formulaString)); - g.replace("COLUMN_NAME", StringEscapeUtils.escapeJava(columnName)); - final String joinedFormulaString = QueryCompiler.createEscapedJoinedString(formulaString); + final String joinedFormulaString = QueryCompiler.createEscapedJoinedString(originalFormulaString); g.replace("JOINED_FORMULA_STRING", joinedFormulaString); g.replace("EXCEPTION_TYPE", EVALUATION_EXCEPTION_CLASSNAME); return g.freeze(); @@ -705,11 +714,13 @@ protected FormulaSourceDescriptor getSourceDescriptor() { return analyzedFormula.sourceDescriptor; } - protected FormulaKernelFactory getFormulaKernelFactory() { - return invokeKernelBuilder().formulaKernelFactory; + protected CompletionStageFuture getFormulaKernelFactory( + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { + return invokeKernelBuilder(compilationRequestProcessor).thenApply(result -> result.formulaKernelFactory); } - private JavaKernelBuilder.Result invokeKernelBuilder() { + private CompletionStageFuture invokeKernelBuilder( + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { final FormulaAnalyzer.Result af = analyzedFormula; final FormulaSourceDescriptor sd = af.sourceDescriptor; final Map columnDict = makeNameToRichTypeDict(sd.sources, columnDefinitions); @@ -722,8 +733,15 @@ private JavaKernelBuilder.Result invokeKernelBuilder() { for (final String p : sd.params) { paramDict.put(p, allParamDict.get(p)); } - return JavaKernelBuilder.create(af.cookedFormulaString, sd.returnType, af.timeInstanceVariables, columnDict, - arrayDict, paramDict); + return JavaKernelBuilder.create( + originalFormulaString, + af.cookedFormulaString, + sd.returnType, + af.timeInstanceVariables, + columnDict, + arrayDict, + paramDict, + compilationRequestProcessor); } /** @@ -731,7 +749,11 @@ private JavaKernelBuilder.Result invokeKernelBuilder() { */ @NotNull String generateKernelClassBody() { - return invokeKernelBuilder().classBody; + try { + return invokeKernelBuilder(QueryCompilerRequestProcessor.ImmediateProcessor.INSTANCE).get().classBody; + } catch (InterruptedException | ExecutionException e) { + throw new UncheckedDeephavenException("Failed to compile formula: ", e); + } } @Override @@ -758,48 +780,46 @@ public Pair>> getFormulaShiftColPair() { return formulaShiftColPair; } - private FormulaFactory createFormulaFactory() { - final String classBody = generateClassBody(); + private void compileFormula(@NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { final String what = "Compile regular formula: " + formulaString; - final Class clazz = compileFormula(what, classBody, "Formula"); - try { - return (FormulaFactory) clazz.getField(FORMULA_FACTORY_NAME).get(null); - } catch (ReflectiveOperationException e) { - throw new FormulaCompilationException("Formula compilation error for: " + what, e); - } - } + final String className = "Formula"; + final String classBody = generateClassBody(); - @SuppressWarnings("SameParameterValue") - private Class compileFormula(final String what, final String classBody, final String className) { - // System.out.printf("compileFormula: what is %s. Code is...%n%s%n", what, classBody); - try (final SafeCloseable ignored = QueryPerformanceRecorder.getInstance().getCompilationNugget(what)) { - // Compilation needs to take place with elevated privileges, but the created object should not have them. + final List> paramClasses = new ArrayList<>(); + final Consumer> addParamClass = (cls) -> { + if (cls != null) { + paramClasses.add(cls); + } + }; + visitFormulaParameters(null, + csp -> { + addParamClass.accept(csp.type); + addParamClass.accept(csp.columnDefinition.getComponentType()); + return null; + }, + cap -> { + addParamClass.accept(cap.dataType); + addParamClass.accept(cap.columnDefinition.getComponentType()); + return null; + }, + p -> { + addParamClass.accept(p.type); + return null; + }); - final List> paramClasses = new ArrayList<>(); - final Consumer> addParamClass = (cls) -> { - if (cls != null) { - paramClasses.add(cls); - } - }; - visitFormulaParameters(null, - csp -> { - addParamClass.accept(csp.type); - addParamClass.accept(csp.columnDefinition.getComponentType()); - return null; - }, - cap -> { - addParamClass.accept(cap.dataType); - addParamClass.accept(cap.columnDefinition.getComponentType()); - return null; - }, - p -> { - addParamClass.accept(p.type); - return null; - }); - final QueryCompiler compiler = ExecutionContext.getContext().getQueryCompiler(); - return compiler.compile(className, classBody, QueryCompiler.FORMULA_PREFIX, - QueryScopeParamTypeUtil.expandParameterClasses(paramClasses)); - } + formulaFactory = compilationRequestProcessor.submit(QueryCompilerRequest.builder() + .description("Formula Expression: " + formulaString) + .className(className) + .classBody(classBody) + .packageNameRoot(QueryCompiler.FORMULA_PREFIX) + .putAllParameterClasses(QueryScopeParamTypeUtil.expandParameterClasses(paramClasses)) + .build()).thenApply(clazz -> { + try { + return (FormulaFactory) clazz.getField(FORMULA_FACTORY_NAME).get(null); + } catch (ReflectiveOperationException e) { + throw new FormulaCompilationException("Formula compilation error for: " + what, e); + } + }); } private static class IndexParameter { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DoubleRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DoubleRangeFilter.java index 378336e4d60..bb9e1710f2a 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DoubleRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DoubleRangeFilter.java @@ -11,6 +11,7 @@ import io.deephaven.engine.rowset.WritableRowSet; import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.util.compare.DoubleComparisons; import io.deephaven.engine.table.impl.chunkfilter.DoubleRangeComparator; import io.deephaven.engine.table.ColumnSource; @@ -19,6 +20,9 @@ import io.deephaven.util.QueryConstants; import org.jetbrains.annotations.NotNull; +import java.util.Map; +import java.util.function.Supplier; + public class DoubleRangeFilter extends AbstractRangeFilter { public static DoubleRangeFilter lt(String columnName, double x) { @@ -79,7 +83,10 @@ static WhereFilter makeDoubleRangeFilter(String columnName, Condition condition, } @Override - public void init(TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DownsampledWhereFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DownsampledWhereFilter.java index 9f1c9288844..f5fdb019455 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DownsampledWhereFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DownsampledWhereFilter.java @@ -9,6 +9,7 @@ import io.deephaven.engine.rowset.RowSetFactory; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.time.DateTimeUtils; import io.deephaven.engine.updategraph.DynamicNode; import io.deephaven.engine.table.ColumnSource; @@ -17,6 +18,8 @@ import java.time.Instant; import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.function.Supplier; /** * Utilities for downsampling non-ticking time series data within a query. The input table must be sorted by the @@ -85,7 +88,10 @@ public List getColumnArrays() { } @Override - public void init(TableDefinition tableDefinition) {} + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) {} @NotNull @Override diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DynamicWhereFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DynamicWhereFilter.java index 29152e038d5..534fc4954b2 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DynamicWhereFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/DynamicWhereFilter.java @@ -26,6 +26,7 @@ import org.jetbrains.annotations.NotNull; import java.util.*; +import java.util.function.Supplier; /** * A where filter that extracts a set of inclusion or exclusion keys from a set table. @@ -221,7 +222,10 @@ public List getColumnArrays() { } @Override - public void init(TableDefinition tableDefinition) {} + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) {} @NotNull @Override diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FloatRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FloatRangeFilter.java index 397d807c61b..12ef7a82366 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FloatRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FloatRangeFilter.java @@ -6,6 +6,7 @@ import io.deephaven.engine.rowset.WritableRowSet; import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.util.compare.FloatComparisons; import io.deephaven.engine.table.impl.chunkfilter.FloatRangeComparator; import io.deephaven.engine.table.ColumnSource; @@ -14,6 +15,9 @@ import io.deephaven.util.QueryConstants; import org.jetbrains.annotations.NotNull; +import java.util.Map; +import java.util.function.Supplier; + public class FloatRangeFilter extends AbstractRangeFilter { public static FloatRangeFilter lt(String columnName, float x) { @@ -74,7 +78,10 @@ static WhereFilter makeFloatRangeFilter(String columnName, Condition condition, } @Override - public void init(TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FunctionalColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FunctionalColumn.java index 2dda0e38dda..936d6fb2f8c 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FunctionalColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/FunctionalColumn.java @@ -8,6 +8,7 @@ import io.deephaven.api.util.NameValidator; import io.deephaven.engine.table.impl.MatchPair; import io.deephaven.engine.table.impl.NoSuchColumnException; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.sources.InMemoryColumnSource; import io.deephaven.engine.table.impl.sources.SparseArrayColumnSource; import io.deephaven.engine.table.impl.sources.ViewColumnSource; @@ -25,6 +26,7 @@ import java.util.Map; import java.util.function.BiFunction; import java.util.function.Function; +import java.util.function.Supplier; public class FunctionalColumn implements SelectColumn { @@ -106,7 +108,10 @@ public List initInputs(TrackingRowSet rowSet, Map initDef(Map> columnDefinitionMap) { + public List initDef( + @NotNull final Map> columnDefinitionMap, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { // noinspection unchecked final ColumnDefinition sourceColumnDefinition = (ColumnDefinition) columnDefinitionMap.get(sourceName); if (sourceColumnDefinition == null) { @@ -125,6 +130,11 @@ public Class getReturnedType() { return destDataType; } + @Override + public Class getReturnedComponentType() { + return componentType; + } + @Override public List getColumns() { return Collections.singletonList(sourceName); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/InstantRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/InstantRangeFilter.java index 3fae7cfe932..d8f5fd795f6 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/InstantRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/InstantRangeFilter.java @@ -6,6 +6,7 @@ import io.deephaven.base.verify.Assert; import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; import io.deephaven.engine.rowset.chunkattributes.OrderedRowKeys; import io.deephaven.engine.table.ColumnSource; @@ -18,6 +19,8 @@ import org.jetbrains.annotations.NotNull; import java.time.Instant; +import java.util.Map; +import java.util.function.Supplier; public class InstantRangeFilter extends LongRangeFilter { @@ -37,7 +40,10 @@ public InstantRangeFilter( } @Override - public void init(TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/IntRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/IntRangeFilter.java index 8badf3ed87c..60856b99f28 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/IntRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/IntRangeFilter.java @@ -13,6 +13,7 @@ import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.chunkfilter.IntRangeComparator; import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; import io.deephaven.gui.table.filters.Condition; @@ -21,6 +22,9 @@ import io.deephaven.util.type.TypeUtils; import org.jetbrains.annotations.NotNull; +import java.util.Map; +import java.util.function.Supplier; + public class IntRangeFilter extends AbstractRangeFilter { public static IntRangeFilter lt(String columnName, int x) { return new IntRangeFilter(columnName, QueryConstants.NULL_INT, x, true, false); @@ -68,7 +72,10 @@ static WhereFilter makeIntRangeFilter(String columnName, Condition condition, St } @Override - public void init(TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/LongRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/LongRangeFilter.java index f5fc6a7cdaf..38507c46642 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/LongRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/LongRangeFilter.java @@ -13,6 +13,7 @@ import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.chunkfilter.LongRangeComparator; import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; import io.deephaven.gui.table.filters.Condition; @@ -21,6 +22,9 @@ import io.deephaven.util.type.TypeUtils; import org.jetbrains.annotations.NotNull; +import java.util.Map; +import java.util.function.Supplier; + public class LongRangeFilter extends AbstractRangeFilter { public static LongRangeFilter lt(String columnName, long x) { return new LongRangeFilter(columnName, QueryConstants.NULL_LONG, x, true, false); @@ -68,7 +72,10 @@ static WhereFilter makeLongRangeFilter(String columnName, Condition condition, S } @Override - public void init(TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java index 27b7e671973..5eb6d418a6b 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java @@ -10,6 +10,7 @@ import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.preview.DisplayWrapper; import io.deephaven.engine.context.QueryScope; import io.deephaven.time.DateTimeUtils; @@ -23,6 +24,7 @@ import java.math.BigInteger; import java.time.Instant; import java.util.*; +import java.util.function.Supplier; public class MatchFilter extends WhereFilterImpl { @@ -116,7 +118,10 @@ public List getColumnArrays() { } @Override - public synchronized void init(TableDefinition tableDefinition) { + public synchronized void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariablesSupplier, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (initialized) { return; } @@ -130,12 +135,12 @@ public synchronized void init(TableDefinition tableDefinition) { return; } final List valueList = new ArrayList<>(); - final QueryScope queryScope = ExecutionContext.getContext().getQueryScope(); + final Map queryScopeVariables = queryScopeVariablesSupplier.get(); final ColumnTypeConvertor convertor = ColumnTypeConvertorFactory.getConvertor(column.getDataType(), column.getName()); for (String strValue : strValues) { - if (queryScope != null && queryScope.hasParamName(strValue)) { - Object paramValue = queryScope.readParamValue(strValue); + if (queryScopeVariables.containsKey(strValue)) { + Object paramValue = queryScopeVariables.get(strValue); if (paramValue != null && paramValue.getClass().isArray()) { ArrayTypeUtils.ArrayAccessor accessor = ArrayTypeUtils.getArrayAccessor(paramValue); for (int ai = 0; ai < accessor.length(); ++ai) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MultiSourceFunctionalColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MultiSourceFunctionalColumn.java index 29d77004a69..116968b8ca2 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MultiSourceFunctionalColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MultiSourceFunctionalColumn.java @@ -9,6 +9,7 @@ import io.deephaven.engine.table.impl.MatchPair; import io.deephaven.engine.table.impl.NoSuchColumnException; import io.deephaven.engine.table.impl.PrevColumnSource; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.sources.InMemoryColumnSource; import io.deephaven.engine.table.impl.sources.SparseArrayColumnSource; import io.deephaven.engine.table.impl.sources.ViewColumnSource; @@ -19,11 +20,11 @@ import io.deephaven.engine.table.impl.chunkfillers.ChunkFiller; import io.deephaven.engine.rowset.RowSequence; import io.deephaven.engine.rowset.TrackingRowSet; -import org.apache.commons.lang3.mutable.MutableObject; import org.jetbrains.annotations.NotNull; import java.util.*; import java.util.function.BiFunction; +import java.util.function.Supplier; import java.util.stream.Collectors; // TODO: Comment the heck out of this... @@ -94,7 +95,10 @@ public List initInputs(TrackingRowSet rowSet, Map initDef(Map> columnDefinitionMap) { + public List initDef( + @NotNull final Map> columnDefinitionMap, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { NoSuchColumnException.throwIf(columnDefinitionMap.keySet(), sourceNames); return getColumns(); } @@ -104,6 +108,11 @@ public Class getReturnedType() { return destDataType; } + @Override + public Class getReturnedComponentType() { + return componentType; + } + @Override public List getColumns() { return Collections.unmodifiableList(sourceNames); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/NullSelectColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/NullSelectColumn.java index 8a544d80048..a16d74189b0 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/NullSelectColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/NullSelectColumn.java @@ -5,6 +5,7 @@ import io.deephaven.engine.table.*; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.sources.*; import io.deephaven.engine.rowset.TrackingRowSet; import org.jetbrains.annotations.NotNull; @@ -12,6 +13,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.Supplier; /** * A SelectColumn implementation that can be used to replace columns with {@link NullValueColumnSource}s @@ -32,7 +34,10 @@ public List initInputs(final TrackingRowSet rowSet, } @Override - public List initDef(final Map> columnDefinitionMap) { + public List initDef( + @NotNull final Map> columnDefinitionMap, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { return Collections.emptyList(); } @@ -41,6 +46,11 @@ public Class getReturnedType() { return nvcs.getType(); } + @Override + public Class getReturnedComponentType() { + return nvcs.getComponentType(); + } + @Override public List getColumns() { return Collections.emptyList(); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeConditionFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeConditionFilter.java index caed4eab668..c859bf113de 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeConditionFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RangeConditionFilter.java @@ -7,6 +7,7 @@ import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.time.DateTimeUtils; import io.deephaven.engine.rowset.WritableRowSet; import io.deephaven.engine.rowset.RowSet; @@ -18,6 +19,8 @@ import java.math.BigInteger; import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.function.Supplier; /** * A filter for comparable types (including Instant) for {@link Condition} values:
@@ -129,7 +132,10 @@ public List getColumnArrays() { } @Override - public void init(TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (filter != null) { return; } @@ -179,7 +185,7 @@ public void init(TableDefinition tableDefinition) { } } - filter.init(tableDefinition); + filter.init(tableDefinition, queryScopeVariables, compilationProcessor); } public static char parseCharFilter(String value) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ReinterpretedColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ReinterpretedColumn.java index b76c64456d9..7acfb5d5c8e 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ReinterpretedColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ReinterpretedColumn.java @@ -13,6 +13,7 @@ import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.WritableColumnSource; import io.deephaven.engine.rowset.TrackingRowSet; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.sources.ConvertibleTimeSource; import io.deephaven.engine.table.impl.sources.LocalDateWrapperSource; import io.deephaven.engine.table.impl.sources.LocalTimeWrapperSource; @@ -34,17 +35,18 @@ import java.util.List; import java.util.Map; import java.util.function.Function; +import java.util.function.Supplier; /** * Allows {@link ColumnSource} reinterpretation via view-type ({@link Table#view} and {@link Table#updateView}) * {@link Table} operations. - * + *

* TODO: If we come up with other valid, useful reinterpretations, it would be trivial to create a general purpose * syntax for use in view()/updateView() column expressions. - * + *

* The syntax I have in mind is: "<ColumnNameB>=<ColumnNameA>.as(<ClassName>)" * "<ColumnName>.as(<ClassName>)" - * + *

* Making this work would consist of any one of: 1. Adding a V1 version and updating SelectColumnFactory and * SelectColumnAdaptor 2. Adding the appropriate if-regex-matches to realColumn selection in V2 SwitchColumn 3. Creating * a V2-native SelectColumnFactory @@ -151,7 +153,10 @@ public List initInputs(TrackingRowSet rowSet, Map initDef(Map> columnDefinitionMap) { + public List initDef( + @NotNull final Map> columnDefinitionMap, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { // noinspection unchecked final ColumnDefinition sourceColumnDefinition = (ColumnDefinition) columnDefinitionMap.get(sourceName); if (sourceColumnDefinition == null) { @@ -169,6 +174,12 @@ public Class getReturnedType() { return destDataType; } + @Override + public Class getReturnedComponentType() { + // we don't support reinterpretting column types with components + return null; + } + @Override public List getColumns() { return Collections.singletonList(sourceName); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RollingReleaseFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RollingReleaseFilter.java index dac007e0554..452811a0572 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RollingReleaseFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/RollingReleaseFilter.java @@ -9,12 +9,15 @@ import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.rowset.RowSet; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.updategraph.NotificationQueue; import io.deephaven.engine.updategraph.UpdateGraph; import org.jetbrains.annotations.NotNull; import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.function.Supplier; /** * This will filter a table starting off with the first N rows, and then adding new rows to the table on each run. @@ -45,7 +48,10 @@ public List getColumnArrays() { } @Override - public void init(TableDefinition tableDefinition) {} + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) {} @NotNull @Override diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SelectColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SelectColumn.java index 31295a0e0ea..8948667c57d 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SelectColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SelectColumn.java @@ -12,6 +12,7 @@ import io.deephaven.api.expression.Method; import io.deephaven.api.filter.Filter; import io.deephaven.api.literal.Literal; +import io.deephaven.engine.context.ExecutionContext; import io.deephaven.engine.context.QueryCompiler; import io.deephaven.engine.rowset.TrackingRowSet; import io.deephaven.engine.table.ColumnDefinition; @@ -19,6 +20,9 @@ import io.deephaven.engine.table.impl.MatchPair; import io.deephaven.engine.table.WritableColumnSource; import io.deephaven.engine.table.impl.BaseTable; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; +import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; +import io.deephaven.util.annotations.FinalDefault; import org.jetbrains.annotations.NotNull; import java.util.Arrays; @@ -26,6 +30,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.Supplier; import java.util.stream.Collectors; /** @@ -71,7 +76,8 @@ static Collection copyFrom(Collection selectColumns) List initInputs(TrackingRowSet rowSet, Map> columnsOfInterest); /** - * Initialize any internal column definitions from the provided initial. + * Initialize any internal column definitions from the provided initial. Any formulae will be compiled immediately + * using the {@link QueryCompiler} in the current {@link ExecutionContext}. * * @param columnDefinitionMap the starting set of column definitions; valid for this call only * @@ -80,7 +86,29 @@ static Collection copyFrom(Collection selectColumns) * {@link QueryCompiler} usage needs to be resolved within initDef. Implementations must be idempotent. * Implementations that want to hold on to the {@code columnDefinitionMap} must make a defensive copy. */ - List initDef(Map> columnDefinitionMap); + @FinalDefault + default List initDef(@NotNull Map> columnDefinitionMap) { + return initDef(columnDefinitionMap, SelectAndViewAnalyzer.newQueryScopeVariableSupplier(), + QueryCompilerRequestProcessor.ImmediateProcessor.INSTANCE); + } + + /** + * Initialize any internal column definitions from the provided initial. A compilation request consumer is provided + * to allow for deferred compilation of expressions that belong to the same query. + * + * @param columnDefinitionMap the starting set of column definitions; valid for this call only + * @param queryScopeVariables a caching supplier of the set of query scope variables; valid for this call only + * @param compilationRequestProcessor a consumer to submit compilation requests; valid for this call only + * + * @return a list of columns on which the result of this is dependent + * @apiNote Any {@link io.deephaven.engine.context.QueryLibrary}, {@link io.deephaven.engine.context.QueryScope}, or + * {@link QueryCompiler} usage needs to be resolved within initDef. Implementations must be idempotent. + * Implementations that want to hold on to the {@code columnDefinitionMap} must make a defensive copy. + */ + List initDef( + @NotNull Map> columnDefinitionMap, + @NotNull Supplier> queryScopeVariables, + @NotNull QueryCompilerRequestProcessor compilationRequestProcessor); /** * Get the data type stored in the resultant column. @@ -89,6 +117,13 @@ static Collection copyFrom(Collection selectColumns) */ Class getReturnedType(); + /** + * Get the data component type stored in the resultant column. + * + * @return the component type + */ + Class getReturnedComponentType(); + /** * Get a list of the names of columns used in this SelectColumn. Behavior is undefined if none of the init* methods * have been called yet. diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ShortRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ShortRangeFilter.java index 234123185cf..1d3caaaf048 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ShortRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/ShortRangeFilter.java @@ -13,6 +13,7 @@ import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.chunkfilter.ShortRangeComparator; import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; import io.deephaven.gui.table.filters.Condition; @@ -21,6 +22,9 @@ import io.deephaven.util.type.TypeUtils; import org.jetbrains.annotations.NotNull; +import java.util.Map; +import java.util.function.Supplier; + public class ShortRangeFilter extends AbstractRangeFilter { public static ShortRangeFilter lt(String columnName, short x) { return new ShortRangeFilter(columnName, QueryConstants.NULL_SHORT, x, true, false); @@ -68,7 +72,10 @@ static WhereFilter makeShortRangeFilter(String columnName, Condition condition, } @Override - public void init(TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SingleSidedComparableRangeFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SingleSidedComparableRangeFilter.java index 2749780468e..59e2e608a05 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SingleSidedComparableRangeFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SingleSidedComparableRangeFilter.java @@ -6,6 +6,7 @@ import io.deephaven.base.verify.Assert; import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; import io.deephaven.util.compare.ObjectComparisons; import io.deephaven.engine.table.ColumnSource; @@ -17,6 +18,9 @@ import io.deephaven.util.annotations.TestUseOnly; import org.jetbrains.annotations.NotNull; +import java.util.Map; +import java.util.function.Supplier; + public class SingleSidedComparableRangeFilter extends AbstractRangeFilter { private final Comparable pivot; private final boolean isGreaterThan; @@ -34,7 +38,10 @@ public static SingleSidedComparableRangeFilter makeForTest(String columnName, Co } @Override - public void init(TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { if (chunkFilter != null) { return; } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SourceColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SourceColumn.java index bebb014aabd..fe411c639db 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SourceColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SourceColumn.java @@ -7,6 +7,7 @@ import io.deephaven.base.verify.Assert; import io.deephaven.engine.table.*; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.sources.InMemoryColumnSource; import io.deephaven.api.util.NameValidator; import io.deephaven.engine.table.impl.NoSuchColumnException; @@ -19,6 +20,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.Supplier; public class SourceColumn implements SelectColumn { @@ -46,7 +48,7 @@ public SourceColumn(String sourceName, String destName) { this(NameValidator.validateColumnName(sourceName), NameValidator.validateColumnName(destName), true); } - private SourceColumn(String sourceName, String destName, boolean unused) { + private SourceColumn(@NotNull final String sourceName, @NotNull final String destName, boolean unused) { this.sourceName = sourceName; this.destName = destName; } @@ -61,7 +63,10 @@ public List initInputs(TrackingRowSet rowSet, Map initDef(Map> columnDefinitionMap) { + public List initDef( + @NotNull final Map> columnDefinitionMap, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { sourceDefinition = columnDefinitionMap.get(sourceName); if (sourceDefinition == null) { throw new NoSuchColumnException(columnDefinitionMap.keySet(), sourceName); @@ -78,6 +83,15 @@ public Class getReturnedType() { return sourceColumn.getType(); } + @Override + public Class getReturnedComponentType() { + // Try to be a little flexible, depending on whether initInputs or initDef was called. + if (sourceDefinition != null) { + return sourceDefinition.getComponentType(); + } + return sourceColumn.getComponentType(); + } + @Override public List getColumns() { return Collections.singletonList(sourceName); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SwitchColumn.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SwitchColumn.java index d2483b6433c..87588fc04c5 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SwitchColumn.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/SwitchColumn.java @@ -8,6 +8,7 @@ import io.deephaven.api.util.NameValidator; import io.deephaven.engine.table.impl.BaseTable; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.python.FormulaColumnPython; import io.deephaven.engine.table.WritableColumnSource; import io.deephaven.engine.rowset.TrackingRowSet; @@ -15,6 +16,7 @@ import java.util.List; import java.util.Map; +import java.util.function.Supplier; public class SwitchColumn implements SelectColumn { @@ -45,7 +47,10 @@ public List initInputs(TrackingRowSet rowSet, Map initDef(Map> columnDefinitionMap) { + public List initDef( + @NotNull final Map> columnDefinitionMap, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { if (realColumn == null) { if (columnDefinitionMap.get(expression) != null) { realColumn = new SourceColumn(expression, columnName); @@ -53,7 +58,8 @@ public List initDef(Map> columnDefinitionMap realColumn = FormulaColumn.createFormulaColumn(columnName, expression, parser); } } - List usedColumns = realColumn.initDef(columnDefinitionMap); + final List usedColumns = realColumn.initDef( + columnDefinitionMap, queryScopeVariables, compilationRequestProcessor); if (realColumn instanceof DhFormulaColumn) { FormulaColumnPython formulaColumnPython = ((DhFormulaColumn) realColumn).getFormulaColumnPython(); realColumn = formulaColumnPython != null ? formulaColumnPython : realColumn; @@ -66,6 +72,11 @@ public Class getReturnedType() { return getRealColumn().getReturnedType(); } + @Override + public Class getReturnedComponentType() { + return getRealColumn().getReturnedComponentType(); + } + @Override public List getColumns() { return getRealColumn().getColumns(); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/TimeSeriesFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/TimeSeriesFilter.java index 3a736ff098f..5fba3c8d098 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/TimeSeriesFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/TimeSeriesFilter.java @@ -13,6 +13,7 @@ import io.deephaven.engine.rowset.RowSetFactory; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.updategraph.NotificationQueue; import io.deephaven.engine.updategraph.UpdateGraph; import io.deephaven.time.DateTimeUtils; @@ -22,6 +23,8 @@ import java.time.Instant; import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.function.Supplier; /** * This will filter a table for the most recent N nanoseconds (must be on a date time column). @@ -55,7 +58,10 @@ public List getColumnArrays() { } @Override - public void init(TableDefinition tableDefinition) {} + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) {} @NotNull @Override diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilter.java index e62154dc016..54a483ce24e 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilter.java @@ -11,8 +11,10 @@ import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.BaseTable; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.QueryTable; import io.deephaven.engine.table.impl.remote.ConstructSnapshot; +import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; import io.deephaven.util.annotations.FinalDefault; import io.deephaven.util.annotations.InternalUseOnly; import org.jetbrains.annotations.NotNull; @@ -20,6 +22,8 @@ import java.util.Arrays; import java.util.Collection; import java.util.List; +import java.util.Map; +import java.util.function.Supplier; /** * Interface for individual filters within a where clause. @@ -103,7 +107,25 @@ interface RecomputeListener { * @apiNote Any {@link io.deephaven.engine.context.QueryLibrary}, {@link io.deephaven.engine.context.QueryScope}, or * {@link QueryCompiler} usage needs to be resolved within init. Implementations must be idempotent. */ - void init(TableDefinition tableDefinition); + @FinalDefault + default void init(@NotNull TableDefinition tableDefinition) { + init(tableDefinition, SelectAndViewAnalyzer.newQueryScopeVariableSupplier(), + QueryCompilerRequestProcessor.ImmediateProcessor.INSTANCE); + } + + /** + * Initialize this select filter given the table definition + * + * @param tableDefinition the definition of the table that will be filtered + * @param queryScopeVariables a caching supplier of the set of query scope variables; valid for this call only + * @param compilationProcessor the processor to use for compilation + * @apiNote Any {@link io.deephaven.engine.context.QueryLibrary}, {@link io.deephaven.engine.context.QueryScope}, or + * {@link QueryCompiler} usage needs to be resolved within init. Implementations must be idempotent. + */ + void init( + @NotNull TableDefinition tableDefinition, + @NotNull Supplier> queryScopeVariables, + @NotNull QueryCompilerRequestProcessor compilationProcessor); /** * Validate that this {@code WhereFilter} is safe to use in the context of the provided sourceTable. diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterInvertedImpl.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterInvertedImpl.java index 3dbe26c1048..c973e2fc1d7 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterInvertedImpl.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterInvertedImpl.java @@ -8,12 +8,15 @@ import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.BaseTable; import io.deephaven.engine.table.impl.DependencyStreamProvider; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.updategraph.NotificationQueue; import io.deephaven.util.annotations.VisibleForTesting; import org.jetbrains.annotations.NotNull; import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.function.Supplier; import java.util.stream.Stream; class WhereFilterInvertedImpl @@ -54,8 +57,11 @@ public List getColumnArrays() { } @Override - public void init(TableDefinition tableDefinition) { - filter.init(tableDefinition); + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { + filter.init(tableDefinition, queryScopeVariables, compilationProcessor); } @Override diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterPatternImpl.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterPatternImpl.java index cadbe1bb488..ce205409497 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterPatternImpl.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereFilterPatternImpl.java @@ -16,12 +16,15 @@ import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter; import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter.ObjectChunkFilter; import org.jetbrains.annotations.NotNull; import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.function.Supplier; final class WhereFilterPatternImpl extends WhereFilterImpl { @@ -43,7 +46,10 @@ private WhereFilterPatternImpl(FilterPattern filterPattern) { } @Override - public void init(TableDefinition tableDefinition) { + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { final String columnName = columnName(); final ColumnDefinition column = tableDefinition.getColumn(columnName); if (column == null) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereNoneFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereNoneFilter.java index 13afd3d00a2..df7d4248596 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereNoneFilter.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/WhereNoneFilter.java @@ -8,10 +8,13 @@ import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.rowset.RowSet; import io.deephaven.engine.rowset.RowSetFactory; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import org.jetbrains.annotations.NotNull; import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.function.Supplier; /** * A Select filter that always returns an empty RowSet. @@ -33,7 +36,10 @@ public List getColumnArrays() { } @Override - public void init(TableDefinition tableDefinition) {} + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) {} @NotNull @Override diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/BaseLayer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/BaseLayer.java index f751407b8dc..20ea139c2b1 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/BaseLayer.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/BaseLayer.java @@ -59,16 +59,6 @@ final Map> getColumnSourcesRecurse(GetMode mode) { return result; } - @Override - public void updateColumnDefinitionsFromTopLayer(Map> columnDefinitions) { - for (Map.Entry> entry : sources.entrySet()) { - final String name = entry.getKey(); - final ColumnSource cs = entry.getValue(); - final ColumnDefinition cd = ColumnDefinition.fromGenericType(name, cs.getType(), cs.getComponentType()); - columnDefinitions.put(name, cd); - } - } - @Override public void applyUpdate(TableUpdate upstream, RowSet toClear, UpdateHelper helper, JobScheduler jobScheduler, @Nullable LivenessNode liveResultOwner, SelectLayerCompletionHandler onCompletion) { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/DependencyLayerBase.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/DependencyLayerBase.java index cc37e773af2..9a126b15698 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/DependencyLayerBase.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/DependencyLayerBase.java @@ -36,14 +36,6 @@ public abstract class DependencyLayerBase extends SelectAndViewAnalyzer { this.myModifiedColumnSet = mcsBuilder; } - - @Override - public void updateColumnDefinitionsFromTopLayer(Map> columnDefinitions) { - final ColumnDefinition cd = - ColumnDefinition.fromGenericType(name, columnSource.getType(), columnSource.getComponentType()); - columnDefinitions.put(name, cd); - } - @Override void populateModifiedColumnSetRecurse(ModifiedColumnSet mcsBuilder, Set remainingDepsToSatisfy) { // Later-defined columns override earlier-defined columns. So we satisfy column dependencies "on the way diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/RedirectionLayer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/RedirectionLayer.java index 27ca97af1b6..fea85388dbb 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/RedirectionLayer.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/RedirectionLayer.java @@ -164,11 +164,6 @@ public SelectAndViewAnalyzer getInner() { return inner; } - @Override - public void updateColumnDefinitionsFromTopLayer(Map> columnDefinitions) { - inner.updateColumnDefinitionsFromTopLayer(columnDefinitions); - } - @Override public void startTrackingPrev() { rowRedirection.startTrackingPrevValues(); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectAndViewAnalyzer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectAndViewAnalyzer.java index d2972981e17..86e8cf417c4 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectAndViewAnalyzer.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/SelectAndViewAnalyzer.java @@ -3,15 +3,19 @@ */ package io.deephaven.engine.table.impl.select.analyzers; +import io.deephaven.api.util.NameValidator; import io.deephaven.base.Pair; import io.deephaven.base.log.LogOutputAppendable; import io.deephaven.datastructures.util.CollectionUtil; +import io.deephaven.engine.context.ExecutionContext; +import io.deephaven.engine.context.QueryScope; import io.deephaven.engine.liveness.LivenessNode; import io.deephaven.engine.rowset.RowSet; import io.deephaven.engine.rowset.RowSetFactory; import io.deephaven.engine.rowset.TrackingRowSet; import io.deephaven.engine.table.*; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.QueryTable; import io.deephaven.engine.table.impl.select.FormulaColumn; import io.deephaven.engine.table.impl.select.SelectColumn; @@ -28,12 +32,14 @@ import io.deephaven.io.log.impl.LogOutputStringImpl; import io.deephaven.util.SafeCloseable; import io.deephaven.util.SafeCloseablePair; +import io.deephaven.util.datastructures.CachingSupplier; import io.deephaven.vector.Vector; import org.jetbrains.annotations.Nullable; import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.function.Consumer; +import java.util.function.Supplier; import java.util.stream.Stream; public abstract class SelectAndViewAnalyzer implements LogOutputAppendable { @@ -44,14 +50,31 @@ public enum Mode { VIEW_LAZY, VIEW_EAGER, SELECT_STATIC, SELECT_REFRESHING, SELECT_REDIRECTED_REFRESHING, SELECT_REDIRECTED_STATIC } + public static Supplier> newQueryScopeVariableSupplier() { + final QueryScope queryScope = ExecutionContext.getContext().getQueryScope(); + return new CachingSupplier<>( + () -> queryScope.toMap((name, value) -> NameValidator.isValidQueryParameterName(name))); + } + public static void initializeSelectColumns( final Map> parentColumnMap, final SelectColumn[] selectColumns) { + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); + initializeSelectColumns(parentColumnMap, newQueryScopeVariableSupplier(), selectColumns, compilationProcessor); + compilationProcessor.compile(); + } + + public static void initializeSelectColumns( + final Map> parentColumnMap, + final Supplier> queryScopeVariables, + final SelectColumn[] selectColumns, + final QueryCompilerRequestProcessor compilationProcessor) { final Map> targetColumnMap = new HashMap<>(parentColumnMap); for (SelectColumn column : selectColumns) { - column.initDef(targetColumnMap); - final ColumnDefinition columnDefinition = - ColumnDefinition.fromGenericType(column.getName(), column.getReturnedType()); + column.initDef(targetColumnMap, queryScopeVariables, compilationProcessor); + final ColumnDefinition columnDefinition = ColumnDefinition.fromGenericType( + column.getName(), column.getReturnedType(), column.getReturnedComponentType()); targetColumnMap.put(column.getName(), columnDefinition); } } @@ -89,30 +112,63 @@ public static SelectAndViewAnalyzerWrapper create( rowRedirection = null; } - final TrackingRowSet originalRowSet = rowSet; - boolean flatResult = rowSet.isFlat(); - // if we preserve a column, we set this to false - boolean flattenedResult = !flatResult - && allowInternalFlatten - && (columnSources.isEmpty() || !publishTheseSources) - && mode == Mode.SELECT_STATIC; - int numberOfInternallyFlattenedColumns = 0; - List processedCols = new LinkedList<>(); List remainingCols = null; FormulaColumn shiftColumn = null; boolean shiftColumnHasPositiveOffset = false; final HashSet resultColumns = new HashSet<>(); - final HashMap> resultAlias = new HashMap<>(); + + // First pass to initialize all columns and to compile formulas in one batch. + final Supplier> variableSupplier = SelectAndViewAnalyzer.newQueryScopeVariableSupplier(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); + for (Map.Entry> entry : columnSources.entrySet()) { + final String name = entry.getKey(); + final ColumnSource cs = entry.getValue(); + final ColumnDefinition cd = ColumnDefinition.fromGenericType(name, cs.getType(), cs.getComponentType()); + columnDefinitions.put(name, cd); + } + for (final SelectColumn sc : selectColumns) { if (remainingCols != null) { remainingCols.add(sc); continue; } - analyzer.updateColumnDefinitionsFromTopLayer(columnDefinitions); - sc.initDef(columnDefinitions); + sc.initDef(columnDefinitions, variableSupplier, compilationProcessor); + final ColumnDefinition cd = ColumnDefinition.fromGenericType( + sc.getName(), sc.getReturnedType(), sc.getReturnedComponentType()); + columnDefinitions.put(sc.getName(), cd); + + if (useShiftedColumns && hasConstantArrayAccess(sc)) { + remainingCols = new LinkedList<>(); + shiftColumn = sc instanceof FormulaColumn + ? (FormulaColumn) sc + : (FormulaColumn) ((SwitchColumn) sc).getRealColumn(); + shiftColumnHasPositiveOffset = hasPositiveOffsetConstantArrayAccess(sc); + continue; + } + + processedCols.add(sc); + } + + // compile all formulas at once + compilationProcessor.compile(); + + // Second pass builds the analyzer and destination columns + final TrackingRowSet originalRowSet = rowSet; + boolean flatResult = rowSet.isFlat(); + // if we preserve a column, we set this to false + boolean flattenedResult = !flatResult + && allowInternalFlatten + && (columnSources.isEmpty() || !publishTheseSources) + && mode == Mode.SELECT_STATIC; + int numberOfInternallyFlattenedColumns = 0; + + final HashMap> resultAlias = new HashMap<>(); + for (final SelectColumn sc : processedCols) { + sc.initInputs(rowSet, analyzer.getAllColumnSources()); // When flattening the result, intermediate columns generate results in position space. When we discover @@ -138,12 +194,8 @@ public static SelectAndViewAnalyzerWrapper create( final ModifiedColumnSet mcsBuilder = new ModifiedColumnSet(parentMcs); if (useShiftedColumns && hasConstantArrayAccess(sc)) { - remainingCols = new LinkedList<>(); - shiftColumn = sc instanceof FormulaColumn - ? (FormulaColumn) sc - : (FormulaColumn) ((SwitchColumn) sc).getRealColumn(); - shiftColumnHasPositiveOffset = hasPositiveOffsetConstantArrayAccess(sc); - continue; + // we use the first shifted column to split between processed columns and remaining columns + throw new IllegalStateException("Found ShiftedColumn in processed column list"); } // shifted columns appear to not be safe for refresh, so we do not validate them until they are rewritten @@ -152,8 +204,6 @@ public static SelectAndViewAnalyzerWrapper create( sc.validateSafeForRefresh(sourceTable); } - processedCols.add(sc); - if (hasConstantValue(sc)) { final WritableColumnSource constViewSource = SingleValueColumnSource.getSingleValueColumnSource(sc.getReturnedType()); @@ -177,8 +227,7 @@ public static SelectAndViewAnalyzerWrapper create( if (!sourceIsNew) { if (numberOfInternallyFlattenedColumns > 0) { // we must preserve this column, but have already created an analyzer for the internally - // flattened - // column, therefore must start over without permitting internal flattening + // flattened column, therefore must start over without permitting internal flattening return create(sourceTable, mode, columnSources, originalRowSet, parentMcs, publishTheseSources, useShiftedColumns, false, selectColumns); } else { @@ -265,6 +314,7 @@ public static SelectAndViewAnalyzerWrapper create( throw new UnsupportedOperationException("Unsupported case " + mode); } } + return new SelectAndViewAnalyzerWrapper(analyzer, shiftColumn, shiftColumnHasPositiveOffset, remainingCols, processedCols); } @@ -522,8 +572,6 @@ public final Map calcEffects(boolean forcePublishAllResources) public abstract SelectAndViewAnalyzer getInner(); - public abstract void updateColumnDefinitionsFromTopLayer(Map> columnDefinitions); - public abstract void startTrackingPrev(); /** @@ -562,8 +610,8 @@ public boolean alreadyFlattenedSources() { abstract public boolean allowCrossColumnParallelization(); /** - * A class that handles the completion of one select column. The handlers are chained together so that when a column - * completes all of the downstream dependencies may execute. + * A class that handles the completion of one select column. The handlers are chained together; all downstream + * dependencies may execute when a column completes. */ public static abstract class SelectLayerCompletionHandler { /** @@ -591,7 +639,7 @@ public static abstract class SelectLayerCompletionHandler { * Create the final completion handler, which has no next handler. * * @param requiredColumns the columns required for this handler to fire - * @param completedColumns the set of completed columns, shared with all of the other handlers + * @param completedColumns the set of completed columns, shared with all the other handlers */ public SelectLayerCompletionHandler(BitSet requiredColumns, BitSet completedColumns) { this.requiredColumns = requiredColumns; @@ -601,9 +649,9 @@ public SelectLayerCompletionHandler(BitSet requiredColumns, BitSet completedColu /** * Called when a single column is completed. - * + *

* If we are ready, then we call {@link #onAllRequiredColumnsCompleted()}. - * + *

* We may not be ready, but other columns downstream of us may be ready, so they are also notified (the * nextHandler). * @@ -639,7 +687,7 @@ protected void onError(Exception error) { } /** - * Called when all of the required columns are completed. + * Called when all required columns are completed. */ protected abstract void onAllRequiredColumnsCompleted(); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/StaticFlattenLayer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/StaticFlattenLayer.java index 1ed084934df..b94fea7abf8 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/StaticFlattenLayer.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/analyzers/StaticFlattenLayer.java @@ -117,11 +117,6 @@ int getLayerIndexFor(String column) { return inner.getLayerIndexFor(column); } - @Override - public void updateColumnDefinitionsFromTopLayer(Map> columnDefinitions) { - inner.updateColumnDefinitionsFromTopLayer(columnDefinitions); - } - @Override public void startTrackingPrev() { throw new UnsupportedOperationException("StaticFlattenLayer is used in only non-refreshing scenarios"); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/FormulaAnalyzer.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/FormulaAnalyzer.java index acc064075d3..4de1b66bbb7 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/FormulaAnalyzer.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/FormulaAnalyzer.java @@ -3,9 +3,9 @@ */ package io.deephaven.engine.table.impl.select.codegen; -import io.deephaven.api.util.NameValidator; import io.deephaven.datastructures.util.CollectionUtil; import io.deephaven.engine.context.ExecutionContext; +import io.deephaven.engine.context.QueryLibrary; import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.impl.lang.QueryLanguageParser; import io.deephaven.engine.table.impl.select.QueryScopeParamTypeUtil; @@ -17,10 +17,14 @@ import io.deephaven.engine.rowset.TrackingWritableRowSet; import io.deephaven.internal.log.LoggerFactory; import io.deephaven.io.logger.Logger; +import org.jetbrains.annotations.NotNull; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; import java.util.*; +import java.util.function.BiConsumer; + +import static io.deephaven.engine.table.impl.select.AbstractFormulaColumn.COLUMN_SUFFIX; public class FormulaAnalyzer { private static final Logger log = LoggerFactory.getLogger(FormulaAnalyzer.class); @@ -67,8 +71,41 @@ public static Result analyze(final String rawFormulaString, queryLanguageResult.isConstantValueExpression()); } - public static QueryLanguageParser.Result getCompiledFormula(Map> availableColumns, - TimeLiteralReplacedExpression timeConversionResult) throws Exception { + /** + * Get the compiled formula for a given formula string. + * + * @param timeConversionResult The formula preprocessed to extract time literals + * @param availableColumns The columns available for use in the formula + * @param columnRenames Outer to inner column name mapping + * @param queryScopeVariables The query scope variables + * @return The parsed formula {@link QueryLanguageParser.Result result} + * @throws Exception If the formula cannot be parsed + */ + public static QueryLanguageParser.Result parseFormula( + @NotNull final TimeLiteralReplacedExpression timeConversionResult, + @NotNull final Map> availableColumns, + @NotNull final Map columnRenames, + @NotNull final Map queryScopeVariables) throws Exception { + return parseFormula(timeConversionResult, availableColumns, columnRenames, queryScopeVariables, true); + } + + /** + * Get the compiled formula for a given formula string. + * + * @param timeConversionResult The formula preprocessed to extract time literals + * @param availableColumns The columns available for use in the formula + * @param columnRenames Outer to inner column name mapping + * @param queryScopeVariables The query scope variables + * @param unboxArguments If true it will unbox the query scope arguments + * @return The parsed formula {@link QueryLanguageParser.Result result} + * @throws Exception If the formula cannot be parsed + */ + public static QueryLanguageParser.Result parseFormula( + @NotNull final TimeLiteralReplacedExpression timeConversionResult, + @NotNull final Map> availableColumns, + @NotNull final Map columnRenames, + @NotNull final Map queryScopeVariables, + final boolean unboxArguments) throws Exception { final Map> possibleVariables = new HashMap<>(); possibleVariables.put("i", int.class); possibleVariables.put("ii", long.class); @@ -81,34 +118,66 @@ public static QueryLanguageParser.Result getCompiledFormula(Map[]> possibleVariableParameterizedTypes = new HashMap<>(); + // Column names get the highest priority. + final BiConsumer> processColumn = (columnName, column) -> { + if (!columnVariables.add(columnName)) { + // this column was renamed + return; + } + + possibleVariables.put(columnName, column.getDataType()); + + final Class compType = column.getComponentType(); + if (compType != null && !compType.isPrimitive()) { + possibleVariableParameterizedTypes.put(columnName, new Class[] {compType}); + } + }; + + // Renames trump the original columns; so they go first. + for (Map.Entry columnRename : columnRenames.entrySet()) { + final String columnName = columnRename.getKey(); + final ColumnDefinition column = availableColumns.get(columnRename.getValue()); + processColumn.accept(columnName, column); + } + + // Now process the original columns. for (ColumnDefinition columnDefinition : availableColumns.values()) { - // add column-vectors - final String columnSuffix = DhFormulaColumn.COLUMN_SUFFIX; - final Class vectorType = DhFormulaColumn.getVectorType(columnDefinition.getDataType()); + processColumn.accept(columnDefinition.getName(), columnDefinition); + } - possibleVariables.put(columnDefinition.getName() + columnSuffix, vectorType); - columnVariables.add(columnDefinition.getName() + columnSuffix); + // Column arrays come between columns and parameters. + final BiConsumer> processColumnArray = (columnName, column) -> { + final String columnArrayName = columnName + COLUMN_SUFFIX; - if (vectorType == ObjectVector.class) { - possibleVariableParameterizedTypes.put(columnDefinition.getName() + columnSuffix, - new Class[] {columnDefinition.getDataType()}); + if (!columnVariables.add(columnArrayName)) { + // Either this is a rename or overloads an existing column name. + return; } - // add columns - columnVariables.add(columnDefinition.getName()); - possibleVariables.put(columnDefinition.getName(), columnDefinition.getDataType()); - final Class compType = columnDefinition.getComponentType(); - if (compType != null && !compType.isPrimitive()) { - possibleVariableParameterizedTypes.put(columnDefinition.getName(), new Class[] {compType}); + final Class vectorType = DhFormulaColumn.getVectorType(column.getDataType()); + possibleVariables.put(columnArrayName, vectorType); + + if (vectorType == ObjectVector.class) { + possibleVariableParameterizedTypes.put(columnArrayName, new Class[] {column.getDataType()}); } + }; + + // Renames still trump the original columns; so they go first. + for (Map.Entry columnRename : columnRenames.entrySet()) { + final String columnName = columnRename.getKey(); + final ColumnDefinition column = availableColumns.get(columnRename.getValue()); + processColumnArray.accept(columnName, column); } - final ExecutionContext context = ExecutionContext.getContext(); - final Map queryScopeVariables = context.getQueryScope().toMap( - (name, value) -> NameValidator.isValidQueryParameterName(name)); + // Now process the original columns. + for (ColumnDefinition columnDefinition : availableColumns.values()) { + processColumnArray.accept(columnDefinition.getName(), columnDefinition); + } + + // Parameters come last. for (Map.Entry param : queryScopeVariables.entrySet()) { if (possibleVariables.containsKey(param.getKey())) { - // skip any existing matches + // Columns and column arrays take precedence over parameters. continue; } @@ -124,22 +193,18 @@ public static QueryLanguageParser.Result getCompiledFormula(Map> classImports = - new HashSet<>(context.getQueryLibrary().getClassImports()); + final QueryLibrary queryLibrary = ExecutionContext.getContext().getQueryLibrary(); + final Set> classImports = new HashSet<>(queryLibrary.getClassImports()); classImports.add(TrackingWritableRowSet.class); classImports.add(WritableColumnSource.class); - return new QueryLanguageParser(timeConversionResult.getConvertedFormula(), - context.getQueryLibrary().getPackageImports(), - classImports, context.getQueryLibrary().getStaticImports(), possibleVariables, - possibleVariableParameterizedTypes, queryScopeVariables, columnVariables) - .getResult(); + return new QueryLanguageParser(timeConversionResult.getConvertedFormula(), queryLibrary.getPackageImports(), + classImports, queryLibrary.getStaticImports(), possibleVariables, possibleVariableParameterizedTypes, + queryScopeVariables, columnVariables, unboxArguments).getResult(); } public static class Result { diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/JavaKernelBuilder.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/JavaKernelBuilder.java index 7eedb5af3f5..bdfea574d9c 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/JavaKernelBuilder.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/codegen/JavaKernelBuilder.java @@ -5,10 +5,11 @@ import io.deephaven.engine.context.QueryCompiler; import io.deephaven.engine.context.ExecutionContext; -import io.deephaven.util.SafeCloseable; +import io.deephaven.engine.context.QueryCompilerRequest; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; +import io.deephaven.util.CompletionStageFuture; import io.deephaven.vector.Vector; import io.deephaven.engine.context.QueryScopeParam; -import io.deephaven.engine.table.impl.perf.QueryPerformanceRecorder; import io.deephaven.engine.table.impl.select.Formula; import io.deephaven.engine.table.impl.select.DhFormulaColumn; import io.deephaven.engine.table.impl.select.FormulaCompilationException; @@ -30,21 +31,38 @@ public class JavaKernelBuilder { private static final String FORMULA_KERNEL_FACTORY_NAME = "__FORMULA_KERNEL_FACTORY"; - public static Result create(String cookedFormulaString, Class returnedType, String timeInstanceVariables, - Map columns, Map> arrays, Map> params) { - final JavaKernelBuilder jkf = new JavaKernelBuilder(cookedFormulaString, returnedType, timeInstanceVariables, - columns, arrays, params); + public static CompletionStageFuture create( + @NotNull final String originalFormulaString, + @NotNull final String cookedFormulaString, + @NotNull final Class returnedType, + @NotNull final String timeInstanceVariables, + @NotNull final Map columns, + @NotNull final Map> arrays, + @NotNull final Map> params, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { + final JavaKernelBuilder jkf = new JavaKernelBuilder( + originalFormulaString, cookedFormulaString, returnedType, timeInstanceVariables, columns, arrays, + params); final String classBody = jkf.generateKernelClassBody(); - final Class clazz = compileFormula(cookedFormulaString, classBody, "Formula"); - final FormulaKernelFactory fkf; - try { - fkf = (FormulaKernelFactory) clazz.getField(FORMULA_KERNEL_FACTORY_NAME).get(null); - } catch (ReflectiveOperationException e) { - throw new FormulaCompilationException("Formula compilation error for: " + cookedFormulaString, e); - } - return new Result(classBody, clazz, fkf); + + return compilationRequestProcessor.submit(QueryCompilerRequest.builder() + .description("FormulaKernel: " + originalFormulaString) + .className("Formula") + .classBody(classBody) + .packageNameRoot(QueryCompiler.FORMULA_PREFIX) + .build()).thenApply(clazz -> { + final FormulaKernelFactory fkf; + try { + fkf = (FormulaKernelFactory) clazz.getField(FORMULA_KERNEL_FACTORY_NAME).get(null); + } catch (ReflectiveOperationException e) { + throw new FormulaCompilationException( + "Formula compilation error for: " + cookedFormulaString, e); + } + return new Result(classBody, clazz, fkf); + }); } + private final String originalFormulaString; private final String cookedFormulaString; private final Class returnedType; private final String timeInstanceVariables; @@ -62,8 +80,15 @@ public static Result create(String cookedFormulaString, Class returnedType, S */ private final Map> params; - private JavaKernelBuilder(String cookedFormulaString, Class returnedType, String timeInstanceVariables, - Map columns, Map> arrays, Map> params) { + private JavaKernelBuilder( + @NotNull final String originalFormulaString, + @NotNull final String cookedFormulaString, + @NotNull final Class returnedType, + @NotNull final String timeInstanceVariables, + @NotNull final Map columns, + @NotNull final Map> arrays, + @NotNull final Map> params) { + this.originalFormulaString = originalFormulaString; this.cookedFormulaString = cookedFormulaString; this.returnedType = returnedType; this.timeInstanceVariables = timeInstanceVariables; @@ -214,7 +239,7 @@ private CodeGenerator generateApplyFormulaPerItem(final TypeAnalyzer ta) { null); g.replace("ARGS", makeCommaSeparatedList(args)); g.replace("FORMULA_STRING", ta.wrapWithCastIfNecessary(cookedFormulaString)); - final String joinedFormulaString = QueryCompiler.createEscapedJoinedString(cookedFormulaString); + final String joinedFormulaString = QueryCompiler.createEscapedJoinedString(originalFormulaString); g.replace("JOINED_FORMULA_STRING", joinedFormulaString); g.replace("EXCEPTION_TYPE", FormulaEvaluationException.class.getCanonicalName()); return g.freeze(); @@ -257,23 +282,16 @@ private List visitFormulaParameters( return results; } - @SuppressWarnings("SameParameterValue") - private static Class compileFormula(final String what, final String classBody, final String className) { - // System.out.printf("compileFormula: formulaString is %s. Code is...%n%s%n", what, classBody); - try (final SafeCloseable ignored = QueryPerformanceRecorder.getInstance().getCompilationNugget(what)) { - // Compilation needs to take place with elevated privileges, but the created object should not have them. - final QueryCompiler compiler = ExecutionContext.getContext().getQueryCompiler(); - return compiler.compile(className, classBody, QueryCompiler.FORMULA_PREFIX); - } - } - public static class Result { public final String classBody; public final Class clazz; public final FormulaKernelFactory formulaKernelFactory; - public Result(String classBody, Class clazz, FormulaKernelFactory formulaKernelFactory) { + public Result( + @NotNull final String classBody, + @NotNull final Class clazz, + @NotNull final FormulaKernelFactory formulaKernelFactory) { this.classBody = classBody; this.clazz = clazz; this.formulaKernelFactory = formulaKernelFactory; diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/formula/FormulaFactory.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/formula/FormulaFactory.java index 8712bf97279..20f2906ab0c 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/formula/FormulaFactory.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/formula/FormulaFactory.java @@ -12,6 +12,7 @@ public interface FormulaFactory { Formula createFormula( + String columnName, TrackingRowSet rowSet, boolean initLazyMap, Map columnsToData, QueryScopeParam... params); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/python/FormulaColumnPython.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/python/FormulaColumnPython.java index cc84b09739d..18842058fc0 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/python/FormulaColumnPython.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/python/FormulaColumnPython.java @@ -5,17 +5,21 @@ import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.context.QueryScopeParam; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; +import io.deephaven.util.CompletionStageFuture; import io.deephaven.vector.Vector; import io.deephaven.engine.table.impl.select.AbstractFormulaColumn; import io.deephaven.engine.table.impl.select.SelectColumn; import io.deephaven.engine.table.impl.select.formula.FormulaKernel; import io.deephaven.engine.table.impl.select.formula.FormulaKernelFactory; import io.deephaven.engine.table.impl.select.formula.FormulaSourceDescriptor; +import org.jetbrains.annotations.NotNull; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.Supplier; import static io.deephaven.datastructures.util.CollectionUtil.ZERO_LENGTH_STRING_ARRAY; @@ -25,8 +29,7 @@ public class FormulaColumnPython extends AbstractFormulaColumn implements FormulaKernelFactory { @SuppressWarnings("unused") // called from python - public static FormulaColumnPython create(String columnName, - DeephavenCompatibleFunction dcf) { + public static FormulaColumnPython create(String columnName, DeephavenCompatibleFunction dcf) { return new FormulaColumnPython(columnName, dcf); } @@ -39,13 +42,16 @@ private FormulaColumnPython(String columnName, } @Override - public final List initDef(Map> columnDefinitionMap) { + public final List initDef( + @NotNull final Map> columnDefinitionMap, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationRequestProcessor) { if (formulaFactory != null) { validateColumnDefinition(columnDefinitionMap); } else { returnedType = dcf.getReturnedType(); applyUsedVariables(columnDefinitionMap, new LinkedHashSet<>(dcf.getColumnNames()), Map.of()); - formulaFactory = createKernelFormulaFactory(this); + formulaFactory = createKernelFormulaFactory(CompletionStageFuture.completedFuture(this)); } return usedColumns; diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/UpdateByOperatorFactory.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/UpdateByOperatorFactory.java index 9a2a295e1e7..76cb79d8119 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/UpdateByOperatorFactory.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/UpdateByOperatorFactory.java @@ -9,7 +9,9 @@ import io.deephaven.engine.table.ColumnDefinition; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.FormulaColumn; +import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; import io.deephaven.engine.table.impl.updateby.delta.*; import io.deephaven.engine.table.impl.updateby.em.*; import io.deephaven.engine.table.impl.updateby.emstd.*; @@ -35,6 +37,7 @@ import java.math.MathContext; import java.time.Instant; import java.util.*; +import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -82,7 +85,10 @@ final Collection getOutputColumns(@NotNull final Collection getOperators(@NotNull final Collection specs) { - final OperationVisitor v = new OperationVisitor(); + final QueryCompilerRequestProcessor.BatchProcessor compilationProcessor = + new QueryCompilerRequestProcessor.BatchProcessor(); + + final OperationVisitor v = new OperationVisitor(compilationProcessor); specs.forEach(s -> s.walk(v)); // Do we have a combined rolling group operator to create? @@ -90,6 +96,8 @@ final Collection getOperators(@NotNull final Collection, UpdateByOperation.Visitor { + private final Supplier> queryScopeVariables; + private final QueryCompilerRequestProcessor compilationProcessor; private final List ops = new ArrayList<>(); private MatchPair[] pairs; @@ -286,6 +296,12 @@ private class OperationVisitor implements UpdateBySpec.Visitor, UpdateByOp RollingGroupSpec rollingGroupSpec; MatchPair[] rollingGroupPairs; + OperationVisitor( + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { + this.queryScopeVariables = SelectAndViewAnalyzer.newQueryScopeVariableSupplier(); + this.compilationProcessor = compilationProcessor; + } + /** * Check if the supplied type is one of the supported time types. * @@ -1361,47 +1377,47 @@ private UpdateByOperator makeRollingFormulaOperator(@NotNull final MatchPair pai return new BooleanRollingFormulaOperator(pair, affectingColumns, rs.revWindowScale().timestampCol(), prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), - formulaColumnMap, tableDef); + formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); } else if (csType == byte.class || csType == Byte.class) { return new ByteRollingFormulaOperator(pair, affectingColumns, rs.revWindowScale().timestampCol(), prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), - formulaColumnMap, tableDef); + formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); } else if (csType == char.class || csType == Character.class) { return new CharRollingFormulaOperator(pair, affectingColumns, rs.revWindowScale().timestampCol(), prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), - formulaColumnMap, tableDef); + formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); } else if (csType == short.class || csType == Short.class) { return new ShortRollingFormulaOperator(pair, affectingColumns, rs.revWindowScale().timestampCol(), prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), - formulaColumnMap, tableDef); + formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); } else if (csType == int.class || csType == Integer.class) { return new IntRollingFormulaOperator(pair, affectingColumns, rs.revWindowScale().timestampCol(), prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), - formulaColumnMap, tableDef); + formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); } else if (csType == long.class || csType == Long.class) { return new LongRollingFormulaOperator(pair, affectingColumns, rs.revWindowScale().timestampCol(), prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), - formulaColumnMap, tableDef); + formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); } else if (csType == float.class || csType == Float.class) { return new FloatRollingFormulaOperator(pair, affectingColumns, rs.revWindowScale().timestampCol(), prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), - formulaColumnMap, tableDef); + formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); } else if (csType == double.class || csType == Double.class) { return new DoubleRollingFormulaOperator(pair, affectingColumns, rs.revWindowScale().timestampCol(), prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), - formulaColumnMap, tableDef); + formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); } return new ObjectRollingFormulaOperator<>(pair, affectingColumns, rs.revWindowScale().timestampCol(), prevWindowScaleUnits, fwdWindowScaleUnits, rs.formula(), rs.paramToken(), - formulaColumnMap, tableDef); + formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); } } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/BaseRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/BaseRollingFormulaOperator.java index 51be9f20d03..2ccd10ea1e7 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/BaseRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/BaseRollingFormulaOperator.java @@ -7,6 +7,7 @@ import io.deephaven.engine.rowset.RowSet; import io.deephaven.engine.table.*; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.FormulaColumn; import io.deephaven.engine.table.impl.select.FormulaUtil; import io.deephaven.engine.table.impl.sources.ArrayBackedColumnSource; @@ -24,6 +25,7 @@ import java.util.Collections; import java.util.Map; import java.util.function.IntConsumer; +import java.util.function.Supplier; abstract class BaseRollingFormulaOperator extends UpdateByOperator { protected final String PARAM_COLUMN_NAME = "__PARAM_COLUMN__"; @@ -92,7 +94,9 @@ public BaseRollingFormulaOperator( @NotNull final String formula, @NotNull final String paramToken, @NotNull final Map, FormulaColumn> formulaColumnMap, - @NotNull final TableDefinition tableDef) { + @NotNull final TableDefinition tableDef, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, true); this.formulaColumnMap = formulaColumnMap; this.tableDef = tableDef; @@ -110,7 +114,8 @@ public BaseRollingFormulaOperator( final ColumnDefinition inputColumnDefinition = ColumnDefinition .fromGenericType(PARAM_COLUMN_NAME, inputVectorType, inputColumnType); - tmp.initDef(Collections.singletonMap(PARAM_COLUMN_NAME, inputColumnDefinition)); + tmp.initDef(Collections.singletonMap(PARAM_COLUMN_NAME, inputColumnDefinition), queryScopeVariables, + compilationProcessor); return tmp; }); } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/BooleanRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/BooleanRollingFormulaOperator.java index c5dd517bd8b..1ba61fb9cb9 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/BooleanRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/BooleanRollingFormulaOperator.java @@ -13,6 +13,7 @@ import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.FormulaColumn; import io.deephaven.engine.table.impl.sources.SingleValueColumnSource; import io.deephaven.engine.table.impl.updateby.UpdateByOperator; @@ -25,6 +26,7 @@ import java.util.Collections; import java.util.Map; import java.util.function.IntConsumer; +import java.util.function.Supplier; import static io.deephaven.util.QueryConstants.NULL_INT; @@ -155,8 +157,11 @@ public BooleanRollingFormulaOperator( @NotNull final String formula, @NotNull final String paramToken, @NotNull final Map, FormulaColumn> formulaColumnMap, - @NotNull final TableDefinition tableDef) { - super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, paramToken, formulaColumnMap, tableDef); + @NotNull final TableDefinition tableDef, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { + super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, + paramToken, formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); } protected BooleanRollingFormulaOperator( diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ByteRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ByteRollingFormulaOperator.java index 966f8a78d4b..461ed73f3cf 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ByteRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ByteRollingFormulaOperator.java @@ -1,3 +1,6 @@ +/** + * Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending + */ /* * --------------------------------------------------------------------------------------------------------------------- * AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY - for any changes edit CharRollingFormulaOperator and regenerate @@ -18,6 +21,7 @@ import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.FormulaColumn; import io.deephaven.engine.table.impl.sources.ReinterpretUtils; import io.deephaven.engine.table.impl.sources.SingleValueColumnSource; @@ -30,6 +34,7 @@ import java.util.Collections; import java.util.Map; import java.util.function.IntConsumer; +import java.util.function.Supplier; import static io.deephaven.util.QueryConstants.NULL_INT; @@ -159,11 +164,14 @@ public ByteRollingFormulaOperator( @NotNull final String formula, @NotNull final String paramToken, @NotNull final Map, FormulaColumn> formulaColumnMap, - @NotNull final TableDefinition tableDef + @NotNull final TableDefinition tableDef, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor // region extra-constructor-args // endregion extra-constructor-args ) { - super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, paramToken, formulaColumnMap, tableDef); + super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, + paramToken, formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); // region constructor // endregion constructor } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/CharRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/CharRollingFormulaOperator.java index bc9464acbc7..8c5f2f1c696 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/CharRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/CharRollingFormulaOperator.java @@ -1,3 +1,6 @@ +/** + * Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending + */ package io.deephaven.engine.table.impl.updateby.rollingformula; import io.deephaven.base.ringbuffer.CharRingBuffer; @@ -13,6 +16,7 @@ import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.FormulaColumn; import io.deephaven.engine.table.impl.sources.ReinterpretUtils; import io.deephaven.engine.table.impl.sources.SingleValueColumnSource; @@ -25,6 +29,7 @@ import java.util.Collections; import java.util.Map; import java.util.function.IntConsumer; +import java.util.function.Supplier; import static io.deephaven.util.QueryConstants.NULL_INT; @@ -154,11 +159,14 @@ public CharRollingFormulaOperator( @NotNull final String formula, @NotNull final String paramToken, @NotNull final Map, FormulaColumn> formulaColumnMap, - @NotNull final TableDefinition tableDef + @NotNull final TableDefinition tableDef, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor // region extra-constructor-args // endregion extra-constructor-args ) { - super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, paramToken, formulaColumnMap, tableDef); + super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, + paramToken, formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); // region constructor // endregion constructor } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/DoubleRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/DoubleRollingFormulaOperator.java index 5f65bbf71e5..7871a3ece94 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/DoubleRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/DoubleRollingFormulaOperator.java @@ -1,3 +1,6 @@ +/** + * Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending + */ /* * --------------------------------------------------------------------------------------------------------------------- * AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY - for any changes edit CharRollingFormulaOperator and regenerate @@ -18,6 +21,7 @@ import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.FormulaColumn; import io.deephaven.engine.table.impl.sources.ReinterpretUtils; import io.deephaven.engine.table.impl.sources.SingleValueColumnSource; @@ -30,6 +34,7 @@ import java.util.Collections; import java.util.Map; import java.util.function.IntConsumer; +import java.util.function.Supplier; import static io.deephaven.util.QueryConstants.NULL_INT; @@ -159,11 +164,14 @@ public DoubleRollingFormulaOperator( @NotNull final String formula, @NotNull final String paramToken, @NotNull final Map, FormulaColumn> formulaColumnMap, - @NotNull final TableDefinition tableDef + @NotNull final TableDefinition tableDef, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor // region extra-constructor-args // endregion extra-constructor-args ) { - super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, paramToken, formulaColumnMap, tableDef); + super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, + paramToken, formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); // region constructor // endregion constructor } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/FloatRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/FloatRollingFormulaOperator.java index c350d61cd4a..3351e6de51d 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/FloatRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/FloatRollingFormulaOperator.java @@ -1,3 +1,6 @@ +/** + * Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending + */ /* * --------------------------------------------------------------------------------------------------------------------- * AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY - for any changes edit CharRollingFormulaOperator and regenerate @@ -18,6 +21,7 @@ import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.FormulaColumn; import io.deephaven.engine.table.impl.sources.ReinterpretUtils; import io.deephaven.engine.table.impl.sources.SingleValueColumnSource; @@ -30,6 +34,7 @@ import java.util.Collections; import java.util.Map; import java.util.function.IntConsumer; +import java.util.function.Supplier; import static io.deephaven.util.QueryConstants.NULL_INT; @@ -159,11 +164,14 @@ public FloatRollingFormulaOperator( @NotNull final String formula, @NotNull final String paramToken, @NotNull final Map, FormulaColumn> formulaColumnMap, - @NotNull final TableDefinition tableDef + @NotNull final TableDefinition tableDef, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor // region extra-constructor-args // endregion extra-constructor-args ) { - super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, paramToken, formulaColumnMap, tableDef); + super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, + paramToken, formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); // region constructor // endregion constructor } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/IntRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/IntRollingFormulaOperator.java index 3bbfaa8d2f4..6d8314f5b9c 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/IntRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/IntRollingFormulaOperator.java @@ -1,3 +1,6 @@ +/** + * Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending + */ /* * --------------------------------------------------------------------------------------------------------------------- * AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY - for any changes edit CharRollingFormulaOperator and regenerate @@ -18,6 +21,7 @@ import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.FormulaColumn; import io.deephaven.engine.table.impl.sources.ReinterpretUtils; import io.deephaven.engine.table.impl.sources.SingleValueColumnSource; @@ -30,6 +34,7 @@ import java.util.Collections; import java.util.Map; import java.util.function.IntConsumer; +import java.util.function.Supplier; import static io.deephaven.util.QueryConstants.NULL_INT; @@ -159,11 +164,14 @@ public IntRollingFormulaOperator( @NotNull final String formula, @NotNull final String paramToken, @NotNull final Map, FormulaColumn> formulaColumnMap, - @NotNull final TableDefinition tableDef + @NotNull final TableDefinition tableDef, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor // region extra-constructor-args // endregion extra-constructor-args ) { - super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, paramToken, formulaColumnMap, tableDef); + super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, + paramToken, formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); // region constructor // endregion constructor } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/LongRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/LongRollingFormulaOperator.java index 9af89aa516f..ba62610a6eb 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/LongRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/LongRollingFormulaOperator.java @@ -1,3 +1,6 @@ +/** + * Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending + */ /* * --------------------------------------------------------------------------------------------------------------------- * AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY - for any changes edit CharRollingFormulaOperator and regenerate @@ -18,6 +21,7 @@ import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.FormulaColumn; import io.deephaven.engine.table.impl.sources.ReinterpretUtils; import io.deephaven.engine.table.impl.sources.SingleValueColumnSource; @@ -30,6 +34,7 @@ import java.util.Collections; import java.util.Map; import java.util.function.IntConsumer; +import java.util.function.Supplier; import static io.deephaven.util.QueryConstants.NULL_INT; @@ -159,11 +164,14 @@ public LongRollingFormulaOperator( @NotNull final String formula, @NotNull final String paramToken, @NotNull final Map, FormulaColumn> formulaColumnMap, - @NotNull final TableDefinition tableDef + @NotNull final TableDefinition tableDef, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor // region extra-constructor-args // endregion extra-constructor-args ) { - super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, paramToken, formulaColumnMap, tableDef); + super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, + paramToken, formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); // region constructor // endregion constructor } diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ObjectRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ObjectRollingFormulaOperator.java index 63375c30bd0..83a49083693 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ObjectRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ObjectRollingFormulaOperator.java @@ -13,6 +13,7 @@ import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.FormulaColumn; import io.deephaven.engine.table.impl.sources.SingleValueColumnSource; import io.deephaven.engine.table.impl.updateby.UpdateByOperator; @@ -24,6 +25,7 @@ import java.util.Collections; import java.util.Map; import java.util.function.IntConsumer; +import java.util.function.Supplier; import static io.deephaven.util.QueryConstants.NULL_INT; @@ -154,8 +156,11 @@ public ObjectRollingFormulaOperator( @NotNull final String formula, @NotNull final String paramToken, @NotNull final Map, FormulaColumn> formulaColumnMap, - @NotNull final TableDefinition tableDef) { - super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, paramToken, formulaColumnMap, tableDef); + @NotNull final TableDefinition tableDef, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) { + super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, + paramToken, formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); } protected ObjectRollingFormulaOperator( diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ShortRollingFormulaOperator.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ShortRollingFormulaOperator.java index ebd9052f0b2..39f578e693b 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ShortRollingFormulaOperator.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/updateby/rollingformula/ShortRollingFormulaOperator.java @@ -1,3 +1,6 @@ +/** + * Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending + */ /* * --------------------------------------------------------------------------------------------------------------------- * AUTO-GENERATED CLASS - DO NOT EDIT MANUALLY - for any changes edit CharRollingFormulaOperator and regenerate @@ -18,6 +21,7 @@ import io.deephaven.engine.table.ColumnSource; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.MatchPair; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import io.deephaven.engine.table.impl.select.FormulaColumn; import io.deephaven.engine.table.impl.sources.ReinterpretUtils; import io.deephaven.engine.table.impl.sources.SingleValueColumnSource; @@ -30,6 +34,7 @@ import java.util.Collections; import java.util.Map; import java.util.function.IntConsumer; +import java.util.function.Supplier; import static io.deephaven.util.QueryConstants.NULL_INT; @@ -159,11 +164,14 @@ public ShortRollingFormulaOperator( @NotNull final String formula, @NotNull final String paramToken, @NotNull final Map, FormulaColumn> formulaColumnMap, - @NotNull final TableDefinition tableDef + @NotNull final TableDefinition tableDef, + @NotNull final Supplier> queryScopeVariables, + @NotNull final QueryCompilerRequestProcessor compilationProcessor // region extra-constructor-args // endregion extra-constructor-args ) { - super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, paramToken, formulaColumnMap, tableDef); + super(pair, affectingColumns, timestampColumnName, reverseWindowScaleUnits, forwardWindowScaleUnits, formula, + paramToken, formulaColumnMap, tableDef, queryScopeVariables, compilationProcessor); // region constructor // endregion constructor } diff --git a/engine/table/src/main/java/io/deephaven/engine/util/DynamicCompileUtils.java b/engine/table/src/main/java/io/deephaven/engine/util/DynamicCompileUtils.java index 6991153b155..89132e6f790 100644 --- a/engine/table/src/main/java/io/deephaven/engine/util/DynamicCompileUtils.java +++ b/engine/table/src/main/java/io/deephaven/engine/util/DynamicCompileUtils.java @@ -5,6 +5,8 @@ import io.deephaven.engine.context.ExecutionContext; import io.deephaven.engine.context.QueryCompiler; +import io.deephaven.engine.context.QueryCompilerRequest; +import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor; import java.util.*; import java.util.function.Supplier; @@ -20,7 +22,7 @@ public static Supplier compileSimpleFunction(final Class res public static Supplier compileSimpleStatement(final Class resultType, final String code, final String... imports) { - final List importClasses = new ArrayList<>(); + final List> importClasses = new ArrayList<>(); for (final String importString : imports) { try { importClasses.add(Class.forName(importString)); @@ -33,14 +35,14 @@ public static Supplier compileSimpleStatement(final Class re } public static Supplier compileSimpleFunction(final Class resultType, final String code, - final Collection imports, final Collection staticImports) { + final Collection> imports, final Collection> staticImports) { final StringBuilder classBody = new StringBuilder(); classBody.append("import ").append(resultType.getName()).append(";\n"); - for (final Class im : imports) { + for (final Class im : imports) { classBody.append("import ").append(im.getName()).append(";\n"); } - for (final Class sim : staticImports) { + for (final Class sim : staticImports) { classBody.append("import static ").append(sim.getName()).append(".*;\n"); } @@ -52,8 +54,13 @@ public static Supplier compileSimpleFunction(final Class res classBody.append(" }\n"); classBody.append("}\n"); - final Class partitionClass = ExecutionContext.getContext().getQueryCompiler() - .compile("Function", classBody.toString(), QueryCompiler.FORMULA_PREFIX); + final Class partitionClass = ExecutionContext.getContext().getQueryCompiler().compile( + QueryCompilerRequest.builder() + .description("Simple Function: " + code) + .className("Function") + .classBody(classBody.toString()) + .packageNameRoot(QueryCompiler.FORMULA_PREFIX) + .build()); try { // noinspection unchecked @@ -63,7 +70,7 @@ public static Supplier compileSimpleFunction(final Class res } } - public static Class getClassThroughCompilation(final String object) { + public static Class getClassThroughCompilation(final String object) { final StringBuilder classBody = new StringBuilder(); classBody.append("public class $CLASSNAME$ implements ").append(Supplier.class.getCanonicalName()) .append("{ \n"); @@ -71,12 +78,17 @@ public static Class getClassThroughCompilation(final String object) { classBody.append(" public Class get() { return ").append(object).append(".class; }\n"); classBody.append("}\n"); - final Class partitionClass = ExecutionContext.getContext().getQueryCompiler() - .compile("Function", classBody.toString(), QueryCompiler.FORMULA_PREFIX); + final Class partitionClass = ExecutionContext.getContext().getQueryCompiler().compile( + QueryCompilerRequest.builder() + .description("Formula: return " + object + ".class") + .className("Function") + .classBody(classBody.toString()) + .packageNameRoot(QueryCompiler.FORMULA_PREFIX) + .build()); try { // noinspection unchecked - return ((Supplier) partitionClass.newInstance()).get(); + return ((Supplier>) partitionClass.newInstance()).get(); } catch (InstantiationException | IllegalAccessException e) { throw new RuntimeException("Could not instantiate function.", e); } diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereParallelTest.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereParallelTest.java index d60ba8f6795..2f7af80baea 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereParallelTest.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableWhereParallelTest.java @@ -18,6 +18,8 @@ import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.function.Supplier; import static org.junit.Assert.assertEquals; @@ -54,7 +56,10 @@ public List getColumnArrays() { } @Override - public void init(TableDefinition tableDefinition) {} + public void init( + @NotNull final TableDefinition tableDefinition, + @NotNull final Supplier> queryScopeSupplier, + @NotNull final QueryCompilerRequestProcessor compilationProcessor) {} @NotNull @Override diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/lang/TestQueryLanguageParser.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/lang/TestQueryLanguageParser.java index 13f2e3c5aa1..bc211f2a0ac 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/lang/TestQueryLanguageParser.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/lang/TestQueryLanguageParser.java @@ -12,6 +12,7 @@ import io.deephaven.engine.context.*; import io.deephaven.engine.table.Table; import io.deephaven.engine.table.impl.lang.QueryLanguageParser.QueryLanguageParseException; +import io.deephaven.engine.table.impl.select.analyzers.SelectAndViewAnalyzer; import io.deephaven.engine.testutil.ControlledUpdateGraph; import io.deephaven.engine.util.PyCallableWrapper; import io.deephaven.util.QueryConstants; @@ -3178,7 +3179,7 @@ private void check(String expression, String resultExpression, Class resultTy final Map possibleParams; final QueryScope queryScope = ExecutionContext.getContext().getQueryScope(); if (!(queryScope instanceof PoisonedQueryScope)) { - possibleParams = queryScope.toMap((name, value) -> NameValidator.isValidQueryParameterName(name)); + possibleParams = SelectAndViewAnalyzer.newQueryScopeVariableSupplier().get(); } else { possibleParams = null; } diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/select/FormulaKernelSample.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/select/FormulaKernelSample.java index 505bad205b0..d6a5c6abcc1 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/select/FormulaKernelSample.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/select/FormulaKernelSample.java @@ -106,7 +106,7 @@ private long applyFormulaPerItem(long II, long ii, int I, int i) { try { return plus(plus(multiply(I, II), multiply(q.intValue(), ii)), II_.get(longCast(minus(i, 1)))); } catch (java.lang.Exception __e) { - throw new io.deephaven.engine.table.impl.select.FormulaEvaluationException("In formula: " + "plus(plus(multiply(I, II), multiply(q.intValue(), ii)), II_.get(longCast(minus(i, 1))))", __e); + throw new io.deephaven.engine.table.impl.select.FormulaEvaluationException("In formula: " + "I * II + q * ii + II_[i - 1]", __e); } } diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/select/FormulaSample.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/select/FormulaSample.java index 897f8697869..f24fd283dbc 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/select/FormulaSample.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/select/FormulaSample.java @@ -73,6 +73,7 @@ public class FormulaSample extends io.deephaven.engine.table.impl.select.Formula { public static final io.deephaven.engine.table.impl.select.formula.FormulaFactory __FORMULA_FACTORY = FormulaSample::new; + private final String __columnName; private final io.deephaven.engine.table.ColumnSource II; private final io.deephaven.engine.table.ColumnSource I; private final io.deephaven.vector.LongVector II_; @@ -80,11 +81,13 @@ public class FormulaSample extends io.deephaven.engine.table.impl.select.Formula private final Map __lazyResultCache; - public FormulaSample(final TrackingRowSet __rowSet, + public FormulaSample(final String __columnName, + final TrackingRowSet __rowSet, final boolean __lazy, final java.util.Map __columnsToData, final io.deephaven.engine.context.QueryScopeParam... __params) { super(__rowSet); + this.__columnName = __columnName; II = __columnsToData.get("II"); I = __columnsToData.get("I"); II_ = new io.deephaven.engine.table.impl.vector.LongVectorColumnWrapper(__columnsToData.get("II"), __rowSet); @@ -190,7 +193,7 @@ private long applyFormulaPerItem(int i, long ii, long II, int I) { try { return plus(plus(multiply(I, II), multiply(q.intValue(), ii)), II_.get(longCast(minus(i, 1)))); } catch (java.lang.Exception __e) { - throw new io.deephaven.engine.table.impl.select.FormulaEvaluationException("In formula: Value = " + "plus(plus(multiply(I, II), multiply(q.intValue(), ii)), II_.get(longCast(minus(i, 1))))", __e); + throw new io.deephaven.engine.table.impl.select.FormulaEvaluationException("In formula: " + __columnName + " = " + "I * II + q * ii + II_[i - 1]", __e); } } diff --git a/engine/table/src/test/java/io/deephaven/engine/util/TestCompileSimpleFunction.java b/engine/table/src/test/java/io/deephaven/engine/util/TestCompileSimpleFunction.java index fea7cee27b1..9c9971aff15 100644 --- a/engine/table/src/test/java/io/deephaven/engine/util/TestCompileSimpleFunction.java +++ b/engine/table/src/test/java/io/deephaven/engine/util/TestCompileSimpleFunction.java @@ -18,7 +18,7 @@ public void testNotString() { DynamicCompileUtils.compileSimpleFunction(String.class, "return 7"); TestCase.fail("Should never have reached this statement."); } catch (RuntimeException e) { - TestCase.assertTrue(e.getMessage().contains("int cannot be converted to String")); + TestCase.assertTrue(e.getMessage().contains("int cannot be converted to java.lang.String")); } } } diff --git a/engine/time/src/main/java/io/deephaven/time/TimeLiteralReplacedExpression.java b/engine/time/src/main/java/io/deephaven/time/TimeLiteralReplacedExpression.java index 012eb43d6a2..c26293cb789 100644 --- a/engine/time/src/main/java/io/deephaven/time/TimeLiteralReplacedExpression.java +++ b/engine/time/src/main/java/io/deephaven/time/TimeLiteralReplacedExpression.java @@ -8,8 +8,6 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import static io.deephaven.util.QueryConstants.NULL_LONG; - // TODO: Move? /** diff --git a/extensions/performance/src/main/java/io/deephaven/engine/table/impl/util/PerformanceQueriesGeneral.java b/extensions/performance/src/main/java/io/deephaven/engine/table/impl/util/PerformanceQueriesGeneral.java index 7943037b5af..41eaa2c08e2 100644 --- a/extensions/performance/src/main/java/io/deephaven/engine/table/impl/util/PerformanceQueriesGeneral.java +++ b/extensions/performance/src/main/java/io/deephaven/engine/table/impl/util/PerformanceQueriesGeneral.java @@ -337,15 +337,13 @@ public static TreeTable queryPerformanceAsTreeTable(@NotNull final Table qpl) { public static TreeTable queryOperationPerformanceAsTreeTable( @NotNull final Table qpl, @NotNull final Table qopl) { - // TODO (https://github.com/deephaven/deephaven-core/issues/4814): use NULL_INT for ParentOperationNumber and - // Depth once we can prevent any compilation or at least reduce multiple usages to a single formula Table mergeWithAggKeys = TableTools.merge( qpl.updateView( "EvalKey = Long.toString(EvaluationNumber)", "ParentEvalKey = ParentEvaluationNumber == null ? null : (Long.toString(ParentEvaluationNumber))", "OperationNumber = NULL_INT", - "ParentOperationNumber = OperationNumber", - "Depth = OperationNumber", + "ParentOperationNumber = NULL_INT", + "Depth = NULL_INT", "CallerLine = (String) null", "IsCompilation = NULL_BOOLEAN", "InputSizeLong = NULL_LONG"),