Skip to content

Commit

Permalink
Merge remote-tracking branch 'refs/remotes/not-napoleon/esql-to-date-…
Browse files Browse the repository at this point in the history 
…nanos-docs' into esql-to-date-nanos-docs
  • Loading branch information
@not-napoleon
not-napoleon committed Jan 14, 2025
2 parents 41e76dd + c17c8b6 commit 6b8c1fa
Show file tree
Hide file tree
Showing 722 changed files with 11,933 additions and 6,588 deletions.
25 changes: 15 additions & 10 deletions benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import org.elasticsearch.core.TimeValue;
import org.elasticsearch.xpack.esql.core.expression.Expression;
import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
import org.elasticsearch.xpack.esql.core.expression.FoldContext;
import org.elasticsearch.xpack.esql.core.expression.Literal;
import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern;
import org.elasticsearch.xpack.esql.core.tree.Source;
Expand Down Expand Up @@ -71,12 +72,11 @@ public class EvalBenchmark {
BigArrays.NON_RECYCLING_INSTANCE
);

private static final FoldContext FOLD_CONTEXT = FoldContext.small();

private static final int BLOCK_LENGTH = 8 * 1024;

static final DriverContext driverContext = new DriverContext(
BigArrays.NON_RECYCLING_INSTANCE,
BlockFactory.getInstance(new NoopCircuitBreaker("noop"), BigArrays.NON_RECYCLING_INSTANCE)
);
static final DriverContext driverContext = new DriverContext(BigArrays.NON_RECYCLING_INSTANCE, blockFactory);

static {
// Smoke test all the expected values and force loading subclasses more like prod
Expand Down Expand Up @@ -114,18 +114,20 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
return switch (operation) {
case "abs" -> {
FieldAttribute longField = longField();
yield EvalMapper.toEvaluator(new Abs(Source.EMPTY, longField), layout(longField)).get(driverContext);
yield EvalMapper.toEvaluator(FOLD_CONTEXT, new Abs(Source.EMPTY, longField), layout(longField)).get(driverContext);
}
case "add" -> {
FieldAttribute longField = longField();
yield EvalMapper.toEvaluator(
FOLD_CONTEXT,
new Add(Source.EMPTY, longField, new Literal(Source.EMPTY, 1L, DataType.LONG)),
layout(longField)
).get(driverContext);
}
case "add_double" -> {
FieldAttribute doubleField = doubleField();
yield EvalMapper.toEvaluator(
FOLD_CONTEXT,
new Add(Source.EMPTY, doubleField, new Literal(Source.EMPTY, 1D, DataType.DOUBLE)),
layout(doubleField)
).get(driverContext);
Expand All @@ -140,7 +142,8 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
lhs = new Add(Source.EMPTY, lhs, new Literal(Source.EMPTY, 1L, DataType.LONG));
rhs = new Add(Source.EMPTY, rhs, new Literal(Source.EMPTY, 1L, DataType.LONG));
}
yield EvalMapper.toEvaluator(new Case(Source.EMPTY, condition, List.of(lhs, rhs)), layout(f1, f2)).get(driverContext);
yield EvalMapper.toEvaluator(FOLD_CONTEXT, new Case(Source.EMPTY, condition, List.of(lhs, rhs)), layout(f1, f2))
.get(driverContext);
}
case "date_trunc" -> {
FieldAttribute timestamp = new FieldAttribute(
Expand All @@ -149,35 +152,37 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
new EsField("timestamp", DataType.DATETIME, Map.of(), true)
);
yield EvalMapper.toEvaluator(
FOLD_CONTEXT,
new DateTrunc(Source.EMPTY, new Literal(Source.EMPTY, Duration.ofHours(24), DataType.TIME_DURATION), timestamp),
layout(timestamp)
).get(driverContext);
}
case "equal_to_const" -> {
FieldAttribute longField = longField();
yield EvalMapper.toEvaluator(
FOLD_CONTEXT,
new Equals(Source.EMPTY, longField, new Literal(Source.EMPTY, 100_000L, DataType.LONG)),
layout(longField)
).get(driverContext);
}
case "long_equal_to_long" -> {
FieldAttribute lhs = longField();
FieldAttribute rhs = longField();
yield EvalMapper.toEvaluator(new Equals(Source.EMPTY, lhs, rhs), layout(lhs, rhs)).get(driverContext);
yield EvalMapper.toEvaluator(FOLD_CONTEXT, new Equals(Source.EMPTY, lhs, rhs), layout(lhs, rhs)).get(driverContext);
}
case "long_equal_to_int" -> {
FieldAttribute lhs = longField();
FieldAttribute rhs = intField();
yield EvalMapper.toEvaluator(new Equals(Source.EMPTY, lhs, rhs), layout(lhs, rhs)).get(driverContext);
yield EvalMapper.toEvaluator(FOLD_CONTEXT, new Equals(Source.EMPTY, lhs, rhs), layout(lhs, rhs)).get(driverContext);
}
case "mv_min", "mv_min_ascending" -> {
FieldAttribute longField = longField();
yield EvalMapper.toEvaluator(new MvMin(Source.EMPTY, longField), layout(longField)).get(driverContext);
yield EvalMapper.toEvaluator(FOLD_CONTEXT, new MvMin(Source.EMPTY, longField), layout(longField)).get(driverContext);
}
case "rlike" -> {
FieldAttribute keywordField = keywordField();
RLike rlike = new RLike(Source.EMPTY, keywordField, new RLikePattern(".ar"));
yield EvalMapper.toEvaluator(rlike, layout(keywordField)).get(driverContext);
yield EvalMapper.toEvaluator(FOLD_CONTEXT, rlike, layout(keywordField)).get(driverContext);
}
default -> throw new UnsupportedOperationException();
};
Expand Down
2 changes: 1 addition & 1 deletion .../integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGitAwareGradleFuncTest.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ abstract class AbstractGitAwareGradleFuncTest extends AbstractGradleFuncTest {

def setup() {
remoteGitRepo = new File(setupGitRemote(), '.git')
"git clone ${remoteGitRepo.absolutePath} cloned".execute(Collections.emptyList(), testProjectDir.root).waitFor()
execute("git clone ${remoteGitRepo.absolutePath} cloned", testProjectDir.root)
buildFile = new File(testProjectDir.root, 'cloned/build.gradle')
settingsFile = new File(testProjectDir.root, 'cloned/settings.gradle')
}
Expand Down
2 changes: 1 addition & 1 deletion ...s/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ abstract class AbstractGradleFuncTest extends Specification {
.forwardOutput()
), configurationCacheCompatible,
buildApiRestrictionsDisabled)
).withArguments(arguments.collect { it.toString() })
).withArguments(arguments.collect { it.toString() } + "--full-stacktrace")
}

def assertOutputContains(String givenOutput, String expected) {
Expand Down
50 changes: 5 additions & 45 deletions distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,20 +187,12 @@ static String agentCommandLineOption(Path agentJar, Path tmpPropertiesFile) {
static void extractSecureSettings(SecureSettings secrets, Map<String, String> propertiesMap) {
final Set<String> settingNames = secrets.getSettingNames();
for (String key : List.of("api_key", "secret_token")) {
for (String prefix : List.of("telemetry.", "tracing.apm.")) {
if (settingNames.contains(prefix + key)) {
if (propertiesMap.containsKey(key)) {
throw new IllegalStateException(
Strings.format("Duplicate telemetry setting: [telemetry.%s] and [tracing.apm.%s]", key, key)
);
}

try (SecureString token = secrets.getString(prefix + key)) {
propertiesMap.put(key, token.toString());
}
String prefix = "telemetry.";
if (settingNames.contains(prefix + key)) {
try (SecureString token = secrets.getString(prefix + key)) {
propertiesMap.put(key, token.toString());
}
}

}
}

Expand All @@ -227,44 +219,12 @@ private static Map<String, String> extractDynamicSettings(Map<String, String> pr
static Map<String, String> extractApmSettings(Settings settings) throws UserException {
final Map<String, String> propertiesMap = new HashMap<>();

// tracing.apm.agent. is deprecated by telemetry.agent.
final String telemetryAgentPrefix = "telemetry.agent.";
final String deprecatedTelemetryAgentPrefix = "tracing.apm.agent.";

final Settings telemetryAgentSettings = settings.getByPrefix(telemetryAgentPrefix);
telemetryAgentSettings.keySet().forEach(key -> propertiesMap.put(key, String.valueOf(telemetryAgentSettings.get(key))));

final Settings apmAgentSettings = settings.getByPrefix(deprecatedTelemetryAgentPrefix);
for (String key : apmAgentSettings.keySet()) {
if (propertiesMap.containsKey(key)) {
throw new IllegalStateException(
Strings.format(
"Duplicate telemetry setting: [%s%s] and [%s%s]",
telemetryAgentPrefix,
key,
deprecatedTelemetryAgentPrefix,
key
)
);
}
propertiesMap.put(key, String.valueOf(apmAgentSettings.get(key)));
}

StringJoiner globalLabels = extractGlobalLabels(telemetryAgentPrefix, propertiesMap, settings);
if (globalLabels.length() == 0) {
globalLabels = extractGlobalLabels(deprecatedTelemetryAgentPrefix, propertiesMap, settings);
} else {
StringJoiner tracingGlobalLabels = extractGlobalLabels(deprecatedTelemetryAgentPrefix, propertiesMap, settings);
if (tracingGlobalLabels.length() != 0) {
throw new IllegalArgumentException(
"Cannot have global labels with tracing.agent prefix ["
+ globalLabels
+ "] and telemetry.apm.agent prefix ["
+ tracingGlobalLabels
+ "]"
);
}
}
if (globalLabels.length() > 0) {
propertiesMap.put("global_labels", globalLabels.toString());
}
Expand All @@ -274,7 +234,7 @@ static Map<String, String> extractApmSettings(Settings settings) throws UserExce
if (propertiesMap.containsKey(key)) {
throw new UserException(
ExitCodes.CONFIG,
"Do not set a value for [tracing.apm.agent." + key + "], as this is configured automatically by Elasticsearch"
"Do not set a value for [telemetry.agent." + key + "], as this is configured automatically by Elasticsearch"
);
}
}
Expand Down
5 changes: 4 additions & 1 deletion ...ibution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.EsExecutors;
import org.elasticsearch.core.Booleans;
import org.elasticsearch.core.UpdateForV9;
import org.elasticsearch.jdk.RuntimeVersionFeature;

Expand All @@ -26,7 +27,9 @@ final class SystemJvmOptions {
static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, String> sysprops) {
String distroType = sysprops.get("es.distribution.type");
boolean isHotspot = sysprops.getOrDefault("sun.management.compiler", "").contains("HotSpot");
boolean useEntitlements = Boolean.parseBoolean(sysprops.getOrDefault("es.entitlements.enabled", "false"));
boolean entitlementsExplicitlyEnabled = Booleans.parseBoolean(sysprops.getOrDefault("es.entitlements.enabled", "false"));
// java 24+ only supports entitlements, but it may be enabled on earlier versions explicitly
boolean useEntitlements = RuntimeVersionFeature.isSecurityManagerAvailable() == false || entitlementsExplicitlyEnabled;
return Stream.of(
Stream.of(
/*
Expand Down
Loading

0 comments on commit 6b8c1fa

Please sign in to comment.