From 8726fd40ad5aa22a0a570d72b1d6be0190e87aba Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Tue, 31 Aug 2021 11:50:19 +0200 Subject: [PATCH 01/16] delete CloneContext.java CtxCloneable.java and related functionality --- .../models/config/ThreadPoolDefinition.java | 4 +- .../forms/managed/AbsoluteFormQueryPlan.java | 20 ++--- .../forms/managed/EntityDateQueryPlan.java | 30 +++----- .../models/forms/managed/FormQueryPlan.java | 76 +++++++++---------- .../forms/managed/RelativeFormQueryPlan.java | 34 ++++----- .../namespaces/specific/ExecuteForm.java | 29 +------ .../namespaces/specific/ExecuteQuery.java | 10 +-- .../conquery/models/query/QueryExecutor.java | 6 +- .../conquery/models/query/QueryJob.java | 8 +- .../filter/CollectionNotEmptyFilterNode.java | 6 -- .../models/query/filter/RangeFilterNode.java | 6 -- .../filter/event/DateDistanceFilterNode.java | 6 -- .../filter/event/MultiSelectFilterNode.java | 7 -- .../filter/event/PrefixTextFilterNode.java | 6 -- .../query/filter/event/SelectFilterNode.java | 6 -- .../event/number/DecimalFilterNode.java | 6 -- .../event/number/IntegerFilterNode.java | 6 -- .../filter/event/number/MoneyFilterNode.java | 6 -- .../filter/event/number/RealFilterNode.java | 6 -- .../queryplan/ArrayConceptQueryPlan.java | 25 ++---- .../query/queryplan/ConceptQueryPlan.java | 22 +----- .../query/queryplan/DateAggregator.java | 14 +--- .../query/queryplan/EventIterating.java | 2 + .../models/query/queryplan/QPNode.java | 3 +- .../models/query/queryplan/QPParentNode.java | 15 ---- .../models/query/queryplan/QueryPlan.java | 4 +- .../query/queryplan/SecondaryIdQueryPlan.java | 25 +++--- .../query/queryplan/TableExportQueryPlan.java | 11 ++- .../queryplan/aggregators/Aggregator.java | 3 +- .../aggregators/ColumnAggregator.java | 6 +- .../DistinctValuesWrapperAggregator.java | 6 -- .../MultiDistinctValuesWrapperAggregator.java | 6 -- .../aggregators/SingleColumnAggregator.java | 6 +- .../specific/ConstantValueAggregator.java | 6 -- .../aggregators/specific/CountAggregator.java | 6 -- .../CountQuartersOfDateRangeAggregator.java | 6 -- .../CountQuartersOfDatesAggregator.java | 6 -- .../specific/DateDistanceAggregator.java | 6 -- .../specific/DateUnionAggregator.java | 9 --- .../specific/DurationSumAggregator.java | 6 -- .../specific/EventDateUnionAggregator.java | 6 -- .../specific/EventDurationSumAggregator.java | 6 -- .../specific/ExistsAggregator.java | 10 --- .../specific/MultiSelectAggregator.java | 6 -- .../specific/PrefixTextAggregator.java | 6 -- .../specific/QuarterAggregator.java | 8 +- .../specific/QuartersInYearAggregator.java | 6 -- .../specific/SelectAggregator.java | 6 -- .../specific/SpecialDateUnion.java | 6 -- .../diffsum/DecimalDiffSumAggregator.java | 6 -- .../diffsum/IntegerDiffSumAggregator.java | 6 -- .../diffsum/MoneyDiffSumAggregator.java | 6 -- .../diffsum/RealDiffSumAggregator.java | 6 -- .../specific/sum/DecimalSumAggregator.java | 6 -- .../specific/sum/IntegerSumAggregator.java | 6 -- .../specific/sum/MoneySumAggregator.java | 6 -- .../specific/sum/RealSumAggregator.java | 6 -- .../specific/value/AllValuesAggregator.java | 6 -- .../specific/value/FirstValueAggregator.java | 6 -- .../specific/value/LastValueAggregator.java | 6 -- .../specific/value/RandomValueAggregator.java | 6 -- .../query/queryplan/clone/CloneContext.java | 41 ---------- .../query/queryplan/clone/CtxCloneable.java | 5 -- .../query/queryplan/filter/FilterNode.java | 3 +- .../query/queryplan/specific/AndNode.java | 14 ---- .../query/queryplan/specific/ConceptNode.java | 6 -- .../specific/DateRestrictingNode.java | 5 -- .../queryplan/specific/ExternalNode.java | 10 +-- .../query/queryplan/specific/FiltersNode.java | 22 ------ .../models/query/queryplan/specific/Leaf.java | 14 +--- .../queryplan/specific/NegatingNode.java | 16 ++-- .../query/queryplan/specific/OrNode.java | 10 +-- .../queryplan/specific/ValidityDateNode.java | 6 -- .../specific/temporal/SampledNode.java | 11 +-- .../specific/temporal/TemporalQueryNode.java | 10 +-- .../conquery/io/result/ResultTestUtil.java | 11 +-- 76 files changed, 137 insertions(+), 650 deletions(-) delete mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/queryplan/clone/CloneContext.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/queryplan/clone/CtxCloneable.java diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ThreadPoolDefinition.java b/backend/src/main/java/com/bakdata/conquery/models/config/ThreadPoolDefinition.java index 4765ac846c..681820867f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ThreadPoolDefinition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ThreadPoolDefinition.java @@ -14,9 +14,9 @@ @Getter @Setter @ToString public class ThreadPoolDefinition { @Min(0) - private int minThreads = Runtime.getRuntime().availableProcessors(); + private int minThreads = 1; @Min(1) - private int maxThreads = Runtime.getRuntime().availableProcessors(); + private int maxThreads = 2; private boolean allowCoreThreadTimeOut = false; private Duration keepAliveTime = Duration.seconds(60); private Duration shutdownTime = Duration.hours(1); diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/AbsoluteFormQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/AbsoluteFormQueryPlan.java index 4a460cceac..dacf33beb3 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/AbsoluteFormQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/AbsoluteFormQueryPlan.java @@ -1,24 +1,28 @@ package com.bakdata.conquery.models.forms.managed; +import java.util.Optional; + import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.QueryPlan; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.query.results.MultilineEntityResult; import lombok.Getter; import lombok.RequiredArgsConstructor; -import java.util.Optional; - @Getter @RequiredArgsConstructor public class AbsoluteFormQueryPlan implements QueryPlan { private final QueryPlan query; private final FormQueryPlan subPlan; - + + @Override + public void init(QueryExecutionContext ctxt, Entity entity) { + + } + @Override public Optional execute(QueryExecutionContext ctx, Entity entity) { @@ -31,14 +35,6 @@ public Optional execute(QueryExecutionContext ctx, Entity return subPlan.execute(ctx, entity); } - @Override - public AbsoluteFormQueryPlan clone(CloneContext ctx) { - return new AbsoluteFormQueryPlan( - query.clone(ctx), - subPlan.clone(ctx) - ); - } - @Override public boolean isOfInterest(Entity entity) { return query.isOfInterest(entity); diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/EntityDateQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/EntityDateQueryPlan.java index eb64ced785..14c6433fc5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/EntityDateQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/EntityDateQueryPlan.java @@ -1,5 +1,9 @@ package com.bakdata.conquery.models.forms.managed; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.common.daterange.CDateRange; @@ -10,16 +14,10 @@ import com.bakdata.conquery.models.query.queryplan.DateAggregator; import com.bakdata.conquery.models.query.queryplan.QueryPlan; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.query.results.MultilineEntityResult; import lombok.RequiredArgsConstructor; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.function.Function; - /** * Implementation of the QueryPlan for an {@link EntityDateQuery}. */ @@ -32,9 +30,13 @@ public class EntityDateQueryPlan implements QueryPlan { private final List resolutionsAndAlignments; private final CDateRange dateRestriction; - private Function validityDateCollector; - @Override + @Override + public void init(QueryExecutionContext ctxt, Entity entity) { + + } + + @Override public Optional execute(QueryExecutionContext ctx, Entity entity) { // Don't set the query date aggregator here because the subqueries should set their aggregator independently @@ -64,17 +66,7 @@ public Optional execute(QueryExecutionContext ctx, Entity return resolutionQuery.execute(ctx, entity); } - @Override - public EntityDateQueryPlan clone(CloneContext ctx) { - return new EntityDateQueryPlan( - query.clone(ctx), - features.clone(ctx), - resolutionsAndAlignments, - dateRestriction - ); - } - - @Override + @Override public boolean isOfInterest(Entity entity) { return query.isOfInterest(entity); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/FormQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/FormQueryPlan.java index 0121285526..19c5b254c5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/FormQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/FormQueryPlan.java @@ -15,7 +15,6 @@ import com.bakdata.conquery.models.query.queryplan.DateAggregator; import com.bakdata.conquery.models.query.queryplan.QueryPlan; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; import com.bakdata.conquery.models.query.results.MultilineEntityResult; import com.bakdata.conquery.models.query.results.SinglelineEntityResult; import com.bakdata.conquery.util.QueryUtils; @@ -28,21 +27,21 @@ public class FormQueryPlan implements QueryPlan { private final ArrayConceptQueryPlan features; private final int constantCount; private final List subPlans = new ArrayList<>(); - + public FormQueryPlan(List dateContexts, ArrayConceptQueryPlan features) { this.dateContexts = dateContexts; this.features = features; - + if (dateContexts.size() <= 0) { // There is nothing to do for this FormQueryPlan but we will return an empty result when its executed constantCount = 3; return; } - + // Either all date contexts have an relative event date or none has one boolean withRelativeEventdate = dateContexts.get(0).getEventDate() != null; - for(DateContext dateContext : dateContexts) { - if((dateContext.getEventDate() == null) == withRelativeEventdate) { + for (DateContext dateContext : dateContexts) { + if ((dateContext.getEventDate() == null) == withRelativeEventdate) { throw new IllegalStateException("Queryplan has absolute AND relative date contexts. Only one kind is allowed."); } } @@ -53,7 +52,7 @@ public FormQueryPlan(List dateContexts, ArrayConceptQueryPlan featu public Optional execute(QueryExecutionContext ctx, Entity entity) { - features.init(ctx,entity); + init(ctx, entity); if (!isOfInterest(entity)) { // If the entity is not covered by the query generate a basic result line with constants but without features @@ -61,76 +60,75 @@ public Optional execute(QueryExecutionContext ctx, Entity } List resultValues = new ArrayList<>(dateContexts.size()); - - for(DateContext dateContext : dateContexts) { - - CloneContext clCtx = new CloneContext(ctx.getStorage()); - - ArrayConceptQueryPlan subPlan = features.clone(clCtx); - subPlans.add(subPlan); - + + for (DateContext dateContext : dateContexts) { CDateSet dateRestriction = CDateSet.create(ctx.getDateRestriction()); dateRestriction.retainAll(dateContext.getDateRange()); // Reference the dates per sub-query, don't accumulate dates of all sub-queries - ctx = QueryUtils.determineDateAggregatorForContext(ctx, subPlan::getValidityDateAggregator); - Optional subResult = subPlan.execute(ctx.withDateRestriction(dateRestriction), entity); - - if(subResult.isEmpty()) { + QueryExecutionContext innerContext = QueryUtils.determineDateAggregatorForContext(ctx, features::getValidityDateAggregator) + .withDateRestriction(dateRestriction); + + features.init(innerContext, entity); + + Optional subResult = features.execute(innerContext, entity); + + if (subResult.isEmpty()) { resultValues.addAll(createResultForNotContained(entity, dateContext).listResultLines()); continue; } - + resultValues.addAll( - ResultModifier.modify( - subResult.get(), - ResultModifier.existAggValuesSetterFor(subPlan.getAggregators(), OptionalInt.of(0)).unaryAndThen(v->addConstants(v, dateContext)) - ).listResultLines() + ResultModifier.modify( + subResult.get(), + ResultModifier.existAggValuesSetterFor(features.getAggregators(), OptionalInt.of(0)).unaryAndThen(v -> addConstants(v, dateContext)) + ).listResultLines() ); } - + return Optional.of(new MultilineEntityResult(entity.getId(), resultValues)); } + @Override + public void init(QueryExecutionContext ctx, Entity entity) { + features.init(ctx, entity); + } + private MultilineEntityResult createResultForNotContained(Entity entity, DateContext dateContext) { List result = new ArrayList<>(); result.add(new Object[features.getAggregatorSize()]); - return ResultModifier.modify(new MultilineEntityResult(entity.getId(), result), ResultModifier.existAggValuesSetterFor(getAggregators(), OptionalInt.of(0)).unaryAndThen(v->addConstants(v, dateContext))); + return ResultModifier.modify(new MultilineEntityResult(entity.getId(), result), ResultModifier.existAggValuesSetterFor(getAggregators(), OptionalInt.of(0)) + .unaryAndThen(v -> addConstants(v, dateContext))); } - + public List> getAggregators() { return features.getAggregators(); } - + private Object[] addConstants(Object[] values, DateContext dateContext) { Object[] result = new Object[values.length + constantCount]; System.arraycopy(values, 0, result, constantCount, values.length); - - if(dateContext == null) { + + if (dateContext == null) { return result; } - + //add resolution indicator result[0] = dateContext.getSubdivisionMode().toString(); //add index value result[1] = dateContext.getIndex(); // add event date - if(dateContext.getEventDate() != null) { + if (dateContext.getEventDate() != null) { result[2] = dateContext.getEventDate().toEpochDay(); } //add date range at [2] or [3] result[getDateRangeResultPosition()] = dateContext.getDateRange(); - + return result; } private int getDateRangeResultPosition() { - return constantCount-1; - } - - @Override - public FormQueryPlan clone(CloneContext ctx) { - return new FormQueryPlan(dateContexts, features.clone(ctx)); + return constantCount - 1; } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQueryPlan.java index 5bd2901173..7673f4befb 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQueryPlan.java @@ -1,20 +1,25 @@ package com.bakdata.conquery.models.forms.managed; -import java.util.*; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.OptionalInt; import com.bakdata.conquery.apiv1.forms.FeatureGroup; import com.bakdata.conquery.apiv1.forms.IndexPlacement; import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; +import com.bakdata.conquery.apiv1.query.concept.specific.temporal.TemporalSampler; import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.error.ConqueryError; import com.bakdata.conquery.models.forms.util.DateContext; import com.bakdata.conquery.models.forms.util.ResultModifier; import com.bakdata.conquery.models.query.QueryExecutionContext; -import com.bakdata.conquery.apiv1.query.concept.specific.temporal.TemporalSampler; import com.bakdata.conquery.models.query.entity.Entity; -import com.bakdata.conquery.models.query.queryplan.*; +import com.bakdata.conquery.models.query.queryplan.ArrayConceptQueryPlan; +import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; +import com.bakdata.conquery.models.query.queryplan.DateAggregator; +import com.bakdata.conquery.models.query.queryplan.QueryPlan; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.query.results.MultilineEntityResult; import com.google.common.collect.ImmutableList; @@ -47,6 +52,11 @@ public class RelativeFormQueryPlan implements QueryPlan { private final transient List featureSubqueries = new ArrayList<>(); private final transient List outcomeSubqueries = new ArrayList<>(); + @Override + public void init(QueryExecutionContext ctxt, Entity entity) { + + } + @Override public Optional execute(QueryExecutionContext ctx, Entity entity) { @@ -241,22 +251,6 @@ public List> getAggregators() { return ImmutableList.copyOf(Iterables.concat(featurePlan.getAggregators(),outcomePlan.getAggregators())); } - @Override - public RelativeFormQueryPlan clone(CloneContext ctx) { - RelativeFormQueryPlan copy = new RelativeFormQueryPlan( - query.clone(ctx), - featurePlan.clone(ctx), - outcomePlan.clone(ctx), - indexSelector, - indexPlacement, - timeCountBefore, - timeCountAfter, - timeUnit, - resolutionsAndAlignmentMap - ); - return copy; - } - @Override public boolean isOfInterest(Entity entity) { return query.isOfInterest(entity) || featurePlan.isOfInterest(entity) || outcomePlan.isOfInterest(entity); diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ExecuteForm.java b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ExecuteForm.java index 78bbfadece..791fcd749c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ExecuteForm.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ExecuteForm.java @@ -1,21 +1,15 @@ package com.bakdata.conquery.models.messages.namespaces.specific; -import static com.bakdata.conquery.models.error.ConqueryError.asConqueryError; - import java.util.Map; import java.util.Map.Entry; -import java.util.stream.Collectors; import com.bakdata.conquery.apiv1.query.Query; import com.bakdata.conquery.io.cps.CPSType; -import com.bakdata.conquery.models.error.ConqueryError; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.messages.namespaces.NamespacedMessage; import com.bakdata.conquery.models.messages.namespaces.WorkerMessage; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.QueryExecutor; -import com.bakdata.conquery.models.query.QueryPlanContext; -import com.bakdata.conquery.models.query.queryplan.QueryPlan; import com.bakdata.conquery.models.query.results.FormShardResult; import com.bakdata.conquery.models.query.results.ShardResult; import com.bakdata.conquery.models.worker.Worker; @@ -58,29 +52,8 @@ public void react(Worker worker) throws Exception { queryExecutor.unsetQueryCancelled(formId); - - Map> plans = null; - // Generate query plans for this execution. For ManagedQueries this is only one plan. - // For ManagedForms there might be multiple plans, which originate from ManagedQueries. - // The results are send directly to these ManagesQueries - try { - final QueryPlanContext queryPlanContext = new QueryPlanContext(worker); - - plans = queries.entrySet().stream() - .collect(Collectors.toMap(Entry::getKey, entry -> entry.getValue().createQueryPlan(queryPlanContext))); - - } - catch (Exception e) { - ConqueryError err = asConqueryError(e); - log.warn("Failed to create query plans for {}.", formId, err); - ShardResult result = createResult(worker, null); - - queryExecutor.sendFailureToManagerNode(result, err); - return; - } - // Execute all plans. - for (Entry> entry : plans.entrySet()) { + for (Entry entry : queries.entrySet()) { ShardResult result = createResult(worker, entry.getKey()); final QueryExecutionContext subQueryContext = new QueryExecutionContext(formId, queryExecutor, worker.getStorage(), worker.getBucketManager()); diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ExecuteQuery.java b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ExecuteQuery.java index 49d203fb61..cfb634b498 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ExecuteQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ExecuteQuery.java @@ -11,7 +11,6 @@ import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.QueryExecutor; import com.bakdata.conquery.models.query.QueryPlanContext; -import com.bakdata.conquery.models.query.queryplan.QueryPlan; import com.bakdata.conquery.models.query.results.ShardResult; import com.bakdata.conquery.models.worker.Worker; import lombok.AllArgsConstructor; @@ -36,7 +35,6 @@ public class ExecuteQuery extends WorkerMessage { private final Query query; - private ShardResult createShardResult(Worker worker) { final ShardResult result = new ShardResult(id, worker.getInfo().getId()); @@ -54,15 +52,13 @@ public void react(Worker worker) throws Exception { queryExecutor.unsetQueryCancelled(executionId); - - QueryPlan plan; final ShardResult result = createShardResult(worker); // Generate query plans for this execution. For ManagedQueries this is only one plan. - // For ManagedForms there might be multiple plans, which originate from ManagedQueries. // The results are send directly to these ManagesQueries try { - plan = query.createQueryPlan(new QueryPlanContext(worker)); + // Assert that we can create the queryPlan + query.createQueryPlan(new QueryPlanContext(worker)); } catch (Exception e) { ConqueryError err = asConqueryError(e); @@ -74,7 +70,7 @@ public void react(Worker worker) throws Exception { final QueryExecutionContext executionContext = new QueryExecutionContext(executionId, queryExecutor, worker.getStorage(), worker.getBucketManager()); - queryExecutor.execute(plan, executionContext, result); + queryExecutor.execute(query, executionContext, result); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/QueryExecutor.java b/backend/src/main/java/com/bakdata/conquery/models/query/QueryExecutor.java index 179f23e74a..fc2a6be2b3 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/QueryExecutor.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/QueryExecutor.java @@ -17,6 +17,7 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.stream.Collectors; +import com.bakdata.conquery.apiv1.query.Query; import com.bakdata.conquery.models.error.ConqueryError; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.query.entity.Entity; @@ -54,8 +55,9 @@ public void sendFailureToManagerNode(ShardResult result, ConqueryError error) { result.finish(Collections.emptyList(), Optional.of(error), worker); } - public boolean execute(QueryPlan queryPlan, QueryExecutionContext executionContext, ShardResult result) { + public boolean execute(Query query, QueryExecutionContext executionContext, ShardResult result) { Collection entities = executionContext.getBucketManager().getEntities().values(); + ThreadLocal> plan = ThreadLocal.withInitial(() -> query.createQueryPlan(new QueryPlanContext(worker))); if (entities.isEmpty()) { log.warn("Entities for query are empty"); @@ -64,7 +66,7 @@ public boolean execute(QueryPlan queryPlan, QueryExecutionContext executionCo try { final List>> futures = entities.stream() - .map(entity -> new QueryJob(executionContext, queryPlan, entity)) + .map(entity -> new QueryJob(executionContext, plan, entity)) .map(job -> CompletableFuture.supplyAsync(job, executor)) .collect(Collectors.toList()); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/QueryJob.java b/backend/src/main/java/com/bakdata/conquery/models/query/QueryJob.java index af67c1dd22..c251a6446a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/QueryJob.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/QueryJob.java @@ -6,7 +6,6 @@ import com.bakdata.conquery.models.error.ConqueryError; import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.QueryPlan; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; import com.bakdata.conquery.models.query.results.EntityResult; import lombok.RequiredArgsConstructor; @@ -14,7 +13,7 @@ public class QueryJob implements Supplier> { private final QueryExecutionContext ctx; - private final QueryPlan plan; + private final ThreadLocal> plan; private final Entity entity; @Override @@ -25,9 +24,8 @@ public Optional get() { } try { - CloneContext cCtx = new CloneContext(ctx.getStorage()); - QueryPlan queryPlan = plan.clone(cCtx); - + QueryPlan queryPlan = plan.get(); + return queryPlan.execute(ctx, entity); } catch (ConqueryError e) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/filter/CollectionNotEmptyFilterNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/filter/CollectionNotEmptyFilterNode.java index 4dd748de81..53c62461b4 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/filter/CollectionNotEmptyFilterNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/filter/CollectionNotEmptyFilterNode.java @@ -3,7 +3,6 @@ import java.util.Collection; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; /** @@ -15,11 +14,6 @@ public CollectionNotEmptyFilterNode(Aggregator> aggregator) { super(aggregator, null); } - @Override - public CollectionNotEmptyFilterNode doClone(CloneContext context) { - return new CollectionNotEmptyFilterNode<>(getAggregator().doClone(context)); - } - @Override public boolean isContained() { return !getAggregator().getAggregationResult().isEmpty(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/filter/RangeFilterNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/filter/RangeFilterNode.java index dd522a4045..1c54cd1a62 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/filter/RangeFilterNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/filter/RangeFilterNode.java @@ -2,7 +2,6 @@ import com.bakdata.conquery.models.common.IRange; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; /** * Entity is included, when the result of the aggregator is contained in the range. @@ -14,11 +13,6 @@ public RangeFilterNode(IRange filterValue, Aggregator aggregator) super(aggregator, filterValue); } - @Override - public RangeFilterNode doClone(CloneContext context) { - return new RangeFilterNode<>(filterValue, getAggregator().doClone(context)); - } - @Override public boolean isContained() { return filterValue.contains(getAggregator().getAggregationResult()); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/DateDistanceFilterNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/DateDistanceFilterNode.java index d52a555006..b5029128fe 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/DateDistanceFilterNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/DateDistanceFilterNode.java @@ -12,7 +12,6 @@ import com.bakdata.conquery.models.datasets.Table; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.query.QueryExecutionContext; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; import com.bakdata.conquery.models.query.queryplan.filter.EventFilterNode; import lombok.Getter; import lombok.Setter; @@ -46,11 +45,6 @@ public void nextTable(QueryExecutionContext ctx, Table currentTable) { } } - @Override - public DateDistanceFilterNode doClone(CloneContext ctx) { - return new DateDistanceFilterNode(getColumn(), unit, filterValue); - } - @Override public boolean checkEvent(Bucket bucket, int event) { if (!bucket.has(event, getColumn())) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/MultiSelectFilterNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/MultiSelectFilterNode.java index cfd8955c26..09d9a3cf3b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/MultiSelectFilterNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/MultiSelectFilterNode.java @@ -11,7 +11,6 @@ import com.bakdata.conquery.models.datasets.Table; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.events.stores.root.StringStore; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; import com.bakdata.conquery.models.query.queryplan.filter.EventFilterNode; import lombok.Getter; import lombok.Setter; @@ -92,12 +91,6 @@ public boolean checkEvent(Bucket bucket, int event) { return false; } - @Override - public MultiSelectFilterNode doClone(CloneContext ctx) { - // We reuse the cache - return new MultiSelectFilterNode(getColumn(), filterValue, selectedValuesCache); - } - @Override public boolean isOfInterest(Bucket bucket) { for (String selected : getFilterValue()) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/PrefixTextFilterNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/PrefixTextFilterNode.java index c53ae2a861..69d7bc7cc5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/PrefixTextFilterNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/PrefixTextFilterNode.java @@ -8,7 +8,6 @@ import com.bakdata.conquery.models.datasets.Table; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.events.stores.root.StringStore; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; import com.bakdata.conquery.models.query.queryplan.filter.EventFilterNode; import lombok.Getter; import lombok.Setter; @@ -30,11 +29,6 @@ public PrefixTextFilterNode(Column column, String filterValue) { this.column = column; } - @Override - public PrefixTextFilterNode doClone(CloneContext ctx) { - return new PrefixTextFilterNode(getColumn(), filterValue); - } - @Override public void nextBlock(Bucket bucket) { store = (StringStore) bucket.getStore(getColumn()); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/SelectFilterNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/SelectFilterNode.java index 09c601cafe..3bbba741c2 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/SelectFilterNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/SelectFilterNode.java @@ -8,7 +8,6 @@ import com.bakdata.conquery.models.datasets.Table; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.events.stores.root.StringStore; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; import com.bakdata.conquery.models.query.queryplan.filter.EventFilterNode; import lombok.Getter; import lombok.Setter; @@ -36,11 +35,6 @@ public void nextBlock(Bucket bucket) { selectedId = ((StringStore) bucket.getStore(getColumn())).getId(filterValue); } - @Override - public SelectFilterNode doClone(CloneContext ctx) { - return new SelectFilterNode(getColumn(), filterValue); - } - @Override public boolean checkEvent(Bucket bucket, int event) { if (selectedId == -1 || !bucket.has(event, getColumn())) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/number/DecimalFilterNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/number/DecimalFilterNode.java index 481bc9c32e..8cb472cf97 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/number/DecimalFilterNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/number/DecimalFilterNode.java @@ -5,7 +5,6 @@ import com.bakdata.conquery.models.common.Range; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.events.Bucket; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; public class DecimalFilterNode extends NumberFilterNode> { @@ -13,11 +12,6 @@ public DecimalFilterNode(Column column, Range filterValue) { super(column, filterValue); } - @Override - public DecimalFilterNode doClone(CloneContext ctx) { - return new DecimalFilterNode(getColumn(), filterValue); - } - @Override public boolean contains(Bucket bucket, int event) { return getFilterValue().contains(bucket.getDecimal(event, getColumn())); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/number/IntegerFilterNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/number/IntegerFilterNode.java index 257d4d893a..ab22eb9f71 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/number/IntegerFilterNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/number/IntegerFilterNode.java @@ -3,7 +3,6 @@ import com.bakdata.conquery.models.common.Range; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.events.Bucket; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; public class IntegerFilterNode extends NumberFilterNode { @@ -11,11 +10,6 @@ public IntegerFilterNode(Column column, Range.LongRange filterValue) { super(column, filterValue); } - @Override - public IntegerFilterNode doClone(CloneContext ctx) { - return new IntegerFilterNode(getColumn(), filterValue); - } - @Override public boolean contains(Bucket bucket, int event) { return getFilterValue().contains(bucket.getInteger(event, getColumn())); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/number/MoneyFilterNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/number/MoneyFilterNode.java index cccf573e16..1236848d4a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/number/MoneyFilterNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/number/MoneyFilterNode.java @@ -3,7 +3,6 @@ import com.bakdata.conquery.models.common.Range; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.events.Bucket; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; public class MoneyFilterNode extends NumberFilterNode { @@ -11,11 +10,6 @@ public MoneyFilterNode(Column column, Range.LongRange filterValue) { super(column, filterValue); } - @Override - public MoneyFilterNode doClone(CloneContext ctx) { - return new MoneyFilterNode(getColumn(), filterValue); - } - @Override public boolean contains(Bucket bucket, int event) { return getFilterValue().contains(bucket.getMoney(event, getColumn())); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/number/RealFilterNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/number/RealFilterNode.java index 036ac64fed..578d0d4758 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/number/RealFilterNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/number/RealFilterNode.java @@ -3,7 +3,6 @@ import com.bakdata.conquery.models.common.Range; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.events.Bucket; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; public class RealFilterNode extends NumberFilterNode { @@ -11,11 +10,6 @@ public RealFilterNode(Column column, Range.DoubleRange filterValue) { super(column, filterValue); } - @Override - public RealFilterNode doClone(CloneContext ctx) { - return new RealFilterNode(getColumn(), filterValue); - } - @Override public boolean contains(Bucket bucket, int event) { return getFilterValue().contains(bucket.getReal(event, getColumn())); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ArrayConceptQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ArrayConceptQueryPlan.java index d887f11972..32ce40a885 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ArrayConceptQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ArrayConceptQueryPlan.java @@ -5,17 +5,16 @@ import java.util.Optional; import java.util.OptionalInt; +import com.bakdata.conquery.apiv1.query.ArrayConceptQuery; +import com.bakdata.conquery.apiv1.query.ConceptQuery; import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.datasets.Table; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.forms.util.ResultModifier; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.QueryPlanContext; -import com.bakdata.conquery.apiv1.query.ArrayConceptQuery; -import com.bakdata.conquery.apiv1.query.ConceptQuery; import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; import com.bakdata.conquery.models.query.results.SinglelineEntityResult; import com.bakdata.conquery.util.QueryUtils; import lombok.Getter; @@ -47,18 +46,6 @@ public boolean isOfInterest(Bucket bucket) { return false; } - @Override - public ArrayConceptQueryPlan clone(CloneContext ctx) { - List childPlanClones = new ArrayList<>(); - for (ConceptQueryPlan child : childPlans) { - childPlanClones.add(child.clone(ctx)); - } - ArrayConceptQueryPlan aqClone = new ArrayConceptQueryPlan(generateDateAggregation); - aqClone.childPlans = new ArrayList<>(childPlanClones); - initDateAggregator(aqClone.validityDateAggregator, aqClone.childPlans); - return aqClone; - } - /** * Helper function to add child queries. This takes care of the SpecialDateUnion * union handling. It acts as a gate keeper, so all child queries either have a @@ -88,7 +75,6 @@ private static void initDateAggregator(DateAggregator validityDateAggregator, Li } public void init(QueryExecutionContext ctx, Entity entity) { - childPlans.forEach(plan -> plan.init(entity, ctx)); } @Override @@ -109,7 +95,8 @@ public Optional execute(QueryExecutionContext ctx, Entit // Start with 1 for aggregator values if dateSet needs to be added to the result final int resultOffset = generateDateAggregation ? 1 : 0; int resultInsertIdx = resultOffset; - boolean notContainedInChildQueries = true; + boolean containedInChildQueries = false; + for (ConceptQueryPlan child : childPlans) { Optional result = child.execute(ctx, entity); @@ -129,7 +116,7 @@ public Optional execute(QueryExecutionContext ctx, Entit SinglelineEntityResult singleLineResult = result.get(); // Mark this result line as contained. - notContainedInChildQueries = false; + containedInChildQueries = true; int copyLength = calculateCopyLength(singleLineResult); System.arraycopy(singleLineResult.getValues(), resultOffset, resultValues, resultInsertIdx, copyLength); @@ -138,7 +125,7 @@ public Optional execute(QueryExecutionContext ctx, Entit // aggregators. resultInsertIdx = nextIndex(resultInsertIdx, child); } - if (notContainedInChildQueries) { + if (!containedInChildQueries) { // None of the subqueries contained an result return Optional.empty(); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ConceptQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ConceptQueryPlan.java index 4c04f7a9c9..1b1c2a1dc0 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ConceptQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ConceptQueryPlan.java @@ -14,7 +14,6 @@ import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; import com.bakdata.conquery.models.query.results.SinglelineEntityResult; import com.bakdata.conquery.util.QueryUtils; import lombok.Getter; @@ -44,23 +43,6 @@ public ConceptQueryPlan(boolean generateDateAggregator) { } } - @Override - public ConceptQueryPlan clone(CloneContext ctx) { - checkRequiredTables(ctx.getStorage()); - - // We set the date aggregator if needed by manually in the following for loop - ConceptQueryPlan clone = new ConceptQueryPlan(false); - clone.setChild(ctx.clone(child)); - - for (Aggregator agg : aggregators) { - clone.aggregators.add(ctx.clone(agg)); - } - - clone.dateAggregator = ctx.clone(dateAggregator); - clone.setRequiredTables(this.getRequiredTables()); - return clone; - } - protected void checkRequiredTables(ModificationShieldedWorkerStorage storage) { if (requiredTables.get() != null) { return; @@ -77,7 +59,7 @@ protected void checkRequiredTables(ModificationShieldedWorkerStorage storage) { } } - public void init(Entity entity, QueryExecutionContext ctx) { + public void init(QueryExecutionContext ctx, Entity entity) { this.entity = entity; child.init(entity, ctx); } @@ -108,7 +90,7 @@ public Optional execute(QueryExecutionContext ctx, Entit return Optional.empty(); } - init(entity, ctx); + init(ctx, entity); if(!isOfInterest(entity)){ return Optional.empty(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregator.java index 840bdf3e61..857acb13d1 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregator.java @@ -8,7 +8,6 @@ import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; import lombok.RequiredArgsConstructor; /** @@ -62,18 +61,7 @@ public ResultType getResultType() { return new ResultType.ListT(ResultType.DateRangeT.INSTANCE); } - @Override - public Aggregator doClone(CloneContext ctx) { - DateAggregator clone = new DateAggregator(action); - Set> clonedChildren = new HashSet<>(); - for (Aggregator sibling : children) { - clonedChildren.add(ctx.clone(sibling)); - } - clone.children = clonedChildren; - return clone; - } - - public boolean hasChildren() { + public boolean hasChildren() { return !children.isEmpty(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/EventIterating.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/EventIterating.java index ca3be7734f..286d857c8d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/EventIterating.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/EventIterating.java @@ -17,6 +17,8 @@ default Set collectRequiredTables() { this.collectRequiredTables(out); return out; } + + public void init(QueryExecutionContext ctx); default void nextTable(QueryExecutionContext ctx, Table currentTable) {} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QPNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QPNode.java index dff71729ab..d092a2b70d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QPNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QPNode.java @@ -10,13 +10,12 @@ import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CtxCloneable; import lombok.AccessLevel; import lombok.Getter; import lombok.Setter; @Getter(AccessLevel.PROTECTED) @Setter(AccessLevel.PROTECTED) -public abstract class QPNode implements EventIterating, CtxCloneable { +public abstract class QPNode implements EventIterating { protected QueryExecutionContext context; protected Entity entity; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QPParentNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QPParentNode.java index 0cf629417d..286db843be 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QPParentNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QPParentNode.java @@ -1,10 +1,8 @@ package com.bakdata.conquery.models.query.queryplan; -import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; -import java.util.Map.Entry; import java.util.Set; import com.bakdata.conquery.models.common.CDateSet; @@ -13,9 +11,7 @@ import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; import com.google.common.base.Preconditions; -import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.ImmutableListMultimap; import com.google.common.collect.ListMultimap; import lombok.AccessLevel; @@ -131,17 +127,6 @@ public String toString() { return super.toString() + "[children = " + children + "]"; } - protected Pair, ListMultimap> createClonedFields(CloneContext ctx) { - List clones = new ArrayList<>(getChildren()); - clones.replaceAll(ctx::clone); - - ArrayListMultimap cloneMap = ArrayListMultimap.create(childMap); - - for (Entry> e : cloneMap.asMap().entrySet()) { - ((List) e.getValue()).replaceAll(ctx::clone); - } - return Pair.of(clones, cloneMap); - } @Override public Collection> getDateAggregators() { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QueryPlan.java index afeb5d28b5..f0480a9ce8 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QueryPlan.java @@ -6,13 +6,12 @@ import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; import com.bakdata.conquery.models.query.results.EntityResult; import org.jetbrains.annotations.NotNull; public interface QueryPlan { - QueryPlan clone(CloneContext ctx); + void init(QueryExecutionContext ctxt, Entity entity); Optional execute(QueryExecutionContext ctx, Entity entity); @@ -20,5 +19,4 @@ public interface QueryPlan { @NotNull Optional> getValidityDateAggregator(); - } \ No newline at end of file diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java index 7c66e677fc..e4db85faa2 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java @@ -7,6 +7,7 @@ import java.util.Optional; import java.util.Set; +import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.SecondaryIdDescription; @@ -14,10 +15,8 @@ import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.identifiable.ids.specific.SecondaryIdDescriptionId; import com.bakdata.conquery.models.query.QueryExecutionContext; -import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; import com.bakdata.conquery.models.query.results.MultilineEntityResult; import com.bakdata.conquery.util.QueryUtils; import lombok.Getter; @@ -47,6 +46,8 @@ public class SecondaryIdQueryPlan implements QueryPlan { private Map childPerKey = new HashMap<>(); + + /** * This is the same execution as a typical ConceptQueryPlan. The difference * is that this method will create a new cloned child for each distinct @@ -60,8 +61,7 @@ public Optional execute(QueryExecutionContext ctx, Entity return Optional.empty(); } - query.checkRequiredTables(ctx.getStorage()); - query.init(entity, ctx); + init(ctx, entity); if (!query.isOfInterest(entity)) { return Optional.empty(); @@ -97,6 +97,14 @@ public Optional execute(QueryExecutionContext ctx, Entity return Optional.of(new MultilineEntityResult(entity.getId(), result)); } + @Override + public void init(QueryExecutionContext ctx, Entity entity) { + query.checkRequiredTables(ctx.getStorage()); + query.init(ctx, entity); + + childPerKey.clear(); + } + private void executeQueriesWithSecondaryId(QueryExecutionContext ctx, Entity entity, Column secondaryIdColumnId) { @@ -186,11 +194,11 @@ private boolean isOfInterest(Bucket bucket) { */ private ConceptQueryPlan createChild(Column secondaryIdColumn, QueryExecutionContext currentContext, Bucket currentBucket) { - ConceptQueryPlan plan = query.clone(new CloneContext(currentContext.getStorage())); + ConceptQueryPlan plan = query; QueryExecutionContext context = QueryUtils.determineDateAggregatorForContext(currentContext, plan::getValidityDateAggregator); - plan.init(query.getEntity(), context); + plan.init(context, query.getEntity()); plan.nextTable(context, secondaryIdColumn.getTable()); plan.isOfInterest(currentBucket); plan.nextBlock(currentBucket); @@ -198,11 +206,6 @@ private ConceptQueryPlan createChild(Column secondaryIdColumn, QueryExecutionCon return plan; } - @Override - public QueryPlan clone(CloneContext ctx) { - return new SecondaryIdQueryPlan(query.clone(ctx), secondaryId, tablesWithSecondaryId, tablesWithoutSecondaryId); - } - @Override public boolean isOfInterest(Entity entity) { return query.isOfInterest(entity); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/TableExportQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/TableExportQueryPlan.java index a1b5d0c3b4..b8791beee1 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/TableExportQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/TableExportQueryPlan.java @@ -15,7 +15,6 @@ import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.query.results.MultilineEntityResult; import lombok.Getter; @@ -33,11 +32,6 @@ public class TableExportQueryPlan implements QueryPlan { private final List tables; private final Map positions; - @Override - public QueryPlan clone(CloneContext ctx) { - return new TableExportQueryPlan(subPlan.clone(ctx), dateRange, tables, positions); - } - @Override public boolean isOfInterest(Entity entity) { return subPlan.isOfInterest(entity); @@ -49,6 +43,11 @@ public Optional> getValidityDateAggregator() { return Optional.empty(); } + @Override + public void init(QueryExecutionContext ctxt, Entity entity) { + // Does nothing + } + @Override public Optional execute(QueryExecutionContext ctx, Entity entity) { Optional result = subPlan.execute(ctx, entity); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/Aggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/Aggregator.java index 695773dc06..2723046719 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/Aggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/Aggregator.java @@ -2,7 +2,6 @@ import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.query.queryplan.EventIterating; -import com.bakdata.conquery.models.query.queryplan.clone.CtxCloneable; import com.fasterxml.jackson.annotation.JsonIgnore; /** @@ -18,7 +17,7 @@ * * @param Java result type after aggregation. */ -public interface Aggregator extends CtxCloneable>, EventIterating { +public interface Aggregator extends EventIterating { T getAggregationResult(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/ColumnAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/ColumnAggregator.java index 9f4874f9ef..15ae09c6fc 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/ColumnAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/ColumnAggregator.java @@ -5,7 +5,6 @@ import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.Table; import com.bakdata.conquery.models.events.Bucket; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; /** * Base class for aggregators acting on columns. @@ -28,10 +27,7 @@ public void collectRequiredTables(Set
out) { public String toString(){ return getClass().getSimpleName(); } - - public ColumnAggregator clone(CloneContext ctx) { - return ctx.clone(this); - } + /** * Skip all buckets where none of the required columns have values. diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/DistinctValuesWrapperAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/DistinctValuesWrapperAggregator.java index fc974ac52e..d800d51e1c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/DistinctValuesWrapperAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/DistinctValuesWrapperAggregator.java @@ -6,7 +6,6 @@ import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; import lombok.Getter; import org.apache.commons.lang3.ArrayUtils; @@ -48,11 +47,6 @@ public void acceptEvent(Bucket bucket, int event) { } } - @Override - public Aggregator doClone(CloneContext ctx) { - return new DistinctValuesWrapperAggregator<>(aggregator.clone(ctx), column); - } - @Override public ResultType getResultType() { return aggregator.getResultType(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/MultiDistinctValuesWrapperAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/MultiDistinctValuesWrapperAggregator.java index 23ab8131a0..a420450cc7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/MultiDistinctValuesWrapperAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/MultiDistinctValuesWrapperAggregator.java @@ -8,7 +8,6 @@ import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; import lombok.Getter; import org.apache.commons.lang3.ArrayUtils; @@ -53,11 +52,6 @@ public void acceptEvent(Bucket bucket, int event) { } } - @Override - public Aggregator doClone(CloneContext ctx) { - return new MultiDistinctValuesWrapperAggregator<>(aggregator.clone(ctx), columns); - } - @Override public ResultType getResultType() { return ResultType.IntegerT.INSTANCE; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/SingleColumnAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/SingleColumnAggregator.java index e244d4e686..08633b4092 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/SingleColumnAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/SingleColumnAggregator.java @@ -8,7 +8,6 @@ import com.bakdata.conquery.io.jackson.serializer.NsIdRef; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.Table; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.Setter; @@ -36,8 +35,5 @@ public final void collectRequiredTables(Set
out) { out.add(getColumn().getTable()); } - @Override - public SingleColumnAggregator clone(CloneContext ctx) { - return ctx.clone(this); - } + } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/ConstantValueAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/ConstantValueAggregator.java index c6532b6972..aa3177d814 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/ConstantValueAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/ConstantValueAggregator.java @@ -3,7 +3,6 @@ import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.ToString; @@ -18,11 +17,6 @@ public class ConstantValueAggregator implements Aggregator { private final Object value; private final ResultType type; - - @Override - public ConstantValueAggregator doClone(CloneContext ctx) { - return this; - } @Override public Object getAggregationResult() { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountAggregator.java index d620c3915a..2f75780d00 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountAggregator.java @@ -4,7 +4,6 @@ import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.query.queryplan.aggregators.SingleColumnAggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; /** * Aggregator counting the number of present values in a column. @@ -29,11 +28,6 @@ public Long getAggregationResult() { return count > 0 ? count : null; } - @Override - public CountAggregator doClone(CloneContext ctx) { - return new CountAggregator(getColumn()); - } - @Override public ResultType getResultType() { return ResultType.IntegerT.INSTANCE; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountQuartersOfDateRangeAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountQuartersOfDateRangeAggregator.java index ea9edd6cb5..9725d68083 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountQuartersOfDateRangeAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountQuartersOfDateRangeAggregator.java @@ -14,7 +14,6 @@ import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.queryplan.aggregators.SingleColumnAggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; import it.unimi.dsi.fastutil.ints.IntOpenHashSet; import it.unimi.dsi.fastutil.ints.IntSet; @@ -33,11 +32,6 @@ public CountQuartersOfDateRangeAggregator(Column column) { super(column); } - @Override - public CountQuartersOfDateRangeAggregator doClone(CloneContext ctx) { - return new CountQuartersOfDateRangeAggregator(getColumn()); - } - @Override public void nextTable(QueryExecutionContext ctx, Table currentTable) { dateRestriction = ctx.getDateRestriction(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountQuartersOfDatesAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountQuartersOfDatesAggregator.java index 8d6a25fb1f..16109e5e48 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountQuartersOfDatesAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountQuartersOfDatesAggregator.java @@ -8,7 +8,6 @@ import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.query.queryplan.aggregators.SingleColumnAggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; import it.unimi.dsi.fastutil.ints.IntOpenHashSet; import it.unimi.dsi.fastutil.ints.IntSet; @@ -38,11 +37,6 @@ public Long getAggregationResult() { return quarters.isEmpty() ? null : (long) quarters.size(); } - @Override - public CountQuartersOfDatesAggregator doClone(CloneContext ctx) { - return new CountQuartersOfDatesAggregator(getColumn()); - } - @Override public ResultType getResultType() { return ResultType.IntegerT.INSTANCE; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DateDistanceAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DateDistanceAggregator.java index d17c356052..f07326be97 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DateDistanceAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DateDistanceAggregator.java @@ -10,7 +10,6 @@ import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.queryplan.aggregators.SingleColumnAggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; /** * Aggregator, returning the min duration in the column, relative to the end of date restriction. @@ -38,11 +37,6 @@ public void nextTable(QueryExecutionContext ctx, Table currentTable) { } } - @Override - public DateDistanceAggregator doClone(CloneContext ctx) { - return new DateDistanceAggregator(getColumn(), unit); - } - @Override public Long getAggregationResult() { return result != Long.MAX_VALUE || hit ? result : null; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DateUnionAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DateUnionAggregator.java index 05d8e9b692..f30986991d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DateUnionAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DateUnionAggregator.java @@ -1,7 +1,5 @@ package com.bakdata.conquery.models.query.queryplan.aggregators.specific; -import java.util.Collection; - import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.datasets.Column; @@ -9,9 +7,7 @@ import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.query.QueryExecutionContext; -import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.SingleColumnAggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; /** * Aggregator, listing all days present. @@ -45,11 +41,6 @@ public void acceptEvent(Bucket bucket, int event) { set.maskedAdd(value, dateRestriction); } - @Override - public DateUnionAggregator doClone(CloneContext ctx) { - return new DateUnionAggregator(getColumn()); - } - @Override public CDateSet getAggregationResult() { return set; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DurationSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DurationSumAggregator.java index e26bcd9e93..1e22caa1b5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DurationSumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DurationSumAggregator.java @@ -8,7 +8,6 @@ import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.queryplan.aggregators.SingleColumnAggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; /** * Aggregator, counting the number of days present. @@ -43,11 +42,6 @@ public void acceptEvent(Bucket bucket, int event) { set.maskedAdd(value, dateRestriction); } - @Override - public DurationSumAggregator doClone(CloneContext ctx) { - return new DurationSumAggregator(getColumn()); - } - @Override public Long getAggregationResult() { return set.isEmpty() ? null : set.countDays(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDateUnionAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDateUnionAggregator.java index f43441ab5c..3b27e20559 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDateUnionAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDateUnionAggregator.java @@ -9,7 +9,6 @@ import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; import lombok.RequiredArgsConstructor; /** @@ -42,11 +41,6 @@ public void nextTable(QueryExecutionContext ctx, Table currentTable) { Aggregator.super.nextTable(ctx, currentTable); } - @Override - public Aggregator doClone(CloneContext ctx) { - return new EventDateUnionAggregator(requiredTables); - } - @Override public CDateSet getAggregationResult() { return set; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDurationSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDurationSumAggregator.java index fdab4bdcae..b2fdb8662b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDurationSumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDurationSumAggregator.java @@ -12,7 +12,6 @@ import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; /** * Aggregator, counting the number of days present. @@ -54,11 +53,6 @@ public void acceptEvent(Bucket bucket, int event) { set.maskedAdd(value, dateRestriction); } - @Override - public EventDurationSumAggregator doClone(CloneContext ctx) { - return new EventDurationSumAggregator(); - } - @Override public Long getAggregationResult() { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/ExistsAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/ExistsAggregator.java index 1b54439805..08accdbfc2 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/ExistsAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/ExistsAggregator.java @@ -7,7 +7,6 @@ import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.query.queryplan.QPNode; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; import lombok.RequiredArgsConstructor; import lombok.ToString; @@ -39,15 +38,6 @@ public void collectRequiredTables(Set
requiredTables) { requiredTables.addAll(this.requiredTables); } - @Override - public ExistsAggregator doClone(CloneContext ctx) { - final ExistsAggregator existsAggregator = new ExistsAggregator(requiredTables); - - existsAggregator.setReference(ctx.clone(reference)); - - return existsAggregator; - } - @Override public ResultType getResultType() { return ResultType.BooleanT.INSTANCE; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/MultiSelectAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/MultiSelectAggregator.java index 294120ede7..b4961748b3 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/MultiSelectAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/MultiSelectAggregator.java @@ -8,7 +8,6 @@ import com.bakdata.conquery.models.events.stores.root.StringStore; import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.query.queryplan.aggregators.SingleColumnAggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; /** * Aggregator counting the occurrence of multiple values. @@ -65,11 +64,6 @@ public Map getAggregationResult() { return out.isEmpty() ? null : out; } - @Override - public MultiSelectAggregator doClone(CloneContext ctx) { - return new MultiSelectAggregator(getColumn(), selection); - } - @Override public ResultType getResultType() { return ResultType.StringT.INSTANCE; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/PrefixTextAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/PrefixTextAggregator.java index 31916d7a9d..f0c21306bd 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/PrefixTextAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/PrefixTextAggregator.java @@ -7,7 +7,6 @@ import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.query.queryplan.aggregators.SingleColumnAggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; /** * Aggregator, returning all values of a column, beginning with a specified value. @@ -43,11 +42,6 @@ public Set getAggregationResult() { return entries.isEmpty() ? null : entries; } - @Override - public PrefixTextAggregator doClone(CloneContext ctx) { - return new PrefixTextAggregator(getColumn(), prefix); - } - @Override public ResultType getResultType() { return ResultType.StringT.INSTANCE; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuarterAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuarterAggregator.java index 6bf7d7b013..d4d429bfa8 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuarterAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuarterAggregator.java @@ -3,6 +3,7 @@ import java.time.LocalDate; import java.util.OptionalInt; +import com.bakdata.conquery.apiv1.query.concept.specific.temporal.TemporalSampler; import com.bakdata.conquery.models.common.CDate; import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.common.QuarterUtils; @@ -12,9 +13,7 @@ import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.query.QueryExecutionContext; -import com.bakdata.conquery.apiv1.query.concept.specific.temporal.TemporalSampler; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; import lombok.Data; /** @@ -55,11 +54,6 @@ public void acceptEvent(Bucket bucket, int event) { set.maskedAdd(value, dateRestriction); } - @Override - public QuarterAggregator doClone(CloneContext ctx) { - return new QuarterAggregator(sampler); - } - @Override public String getAggregationResult() { if (set.isEmpty()) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuartersInYearAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuartersInYearAggregator.java index e8e24a55e5..40fed04568 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuartersInYearAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuartersInYearAggregator.java @@ -11,7 +11,6 @@ import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.query.queryplan.aggregators.SingleColumnAggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; import it.unimi.dsi.fastutil.ints.Int2ObjectMap; import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; @@ -64,11 +63,6 @@ public Long getAggregationResult() { return max; } - @Override - public QuartersInYearAggregator doClone(CloneContext ctx) { - return new QuartersInYearAggregator(getColumn()); - } - @Override public ResultType getResultType() { return ResultType.IntegerT.INSTANCE; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SelectAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SelectAggregator.java index a621f34103..eae73349f3 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SelectAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SelectAggregator.java @@ -5,7 +5,6 @@ import com.bakdata.conquery.models.events.stores.root.StringStore; import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.query.queryplan.aggregators.SingleColumnAggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; /** @@ -49,11 +48,6 @@ public Long getAggregationResult() { return hits > 0 ? hits : null; } - @Override - public SelectAggregator doClone(CloneContext ctx) { - return new SelectAggregator(getColumn(), selected); - } - @Override public ResultType getResultType() { return ResultType.IntegerT.INSTANCE; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SpecialDateUnion.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SpecialDateUnion.java index 6760dbd326..27217ae1d7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SpecialDateUnion.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SpecialDateUnion.java @@ -7,7 +7,6 @@ import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; import lombok.RequiredArgsConstructor; /** @@ -48,11 +47,6 @@ public void merge(CDateSet other){ set.addAll(other); } - @Override - public SpecialDateUnion doClone(CloneContext ctx) { - return new SpecialDateUnion(); - } - @Override public CDateSet getAggregationResult() { return set; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/DecimalDiffSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/DecimalDiffSumAggregator.java index 749e0b7a67..029517b5cc 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/DecimalDiffSumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/DecimalDiffSumAggregator.java @@ -6,7 +6,6 @@ import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.query.queryplan.aggregators.ColumnAggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; import lombok.Getter; /** @@ -27,11 +26,6 @@ public DecimalDiffSumAggregator(Column addend, Column subtrahend) { this.subtrahendColumn = subtrahend; } - @Override - public DecimalDiffSumAggregator doClone(CloneContext ctx) { - return new DecimalDiffSumAggregator(getAddendColumn(), getSubtrahendColumn()); - } - @Override public Column[] getRequiredColumns() { return new Column[]{getAddendColumn(), getSubtrahendColumn()}; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/IntegerDiffSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/IntegerDiffSumAggregator.java index 964ddf22c9..ebf97e359a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/IntegerDiffSumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/IntegerDiffSumAggregator.java @@ -4,7 +4,6 @@ import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.query.queryplan.aggregators.ColumnAggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; import lombok.Getter; /** @@ -25,11 +24,6 @@ public IntegerDiffSumAggregator(Column addend, Column subtrahend) { this.sum = 0L; } - @Override - public IntegerDiffSumAggregator doClone(CloneContext ctx) { - return new IntegerDiffSumAggregator(getAddendColumn(), getSubtrahendColumn()); - } - @Override public Column[] getRequiredColumns() { return new Column[]{getAddendColumn(), getSubtrahendColumn()}; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/MoneyDiffSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/MoneyDiffSumAggregator.java index 313a314545..f8a32c46f2 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/MoneyDiffSumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/MoneyDiffSumAggregator.java @@ -4,7 +4,6 @@ import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.query.queryplan.aggregators.ColumnAggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; import lombok.Getter; /** @@ -24,11 +23,6 @@ public MoneyDiffSumAggregator(Column addend, Column subtrahend) { this.subtrahendColumn = subtrahend; } - @Override - public MoneyDiffSumAggregator doClone(CloneContext ctx) { - return new MoneyDiffSumAggregator(getAddendColumn(), getSubtrahendColumn()); - } - @Override public Column[] getRequiredColumns() { return new Column[]{getAddendColumn(), getSubtrahendColumn()}; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/RealDiffSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/RealDiffSumAggregator.java index 8c07301fe0..34ffd3d5f0 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/RealDiffSumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/RealDiffSumAggregator.java @@ -4,7 +4,6 @@ import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.query.queryplan.aggregators.ColumnAggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; import lombok.Getter; /** @@ -25,11 +24,6 @@ public RealDiffSumAggregator(Column addend, Column subtrahend) { this.subtrahendColumn = subtrahend; } - @Override - public RealDiffSumAggregator doClone(CloneContext ctx) { - return new RealDiffSumAggregator(getAddendColumn(), getSubtrahendColumn()); - } - @Override public Column[] getRequiredColumns() { return new Column[]{getAddendColumn(), getSubtrahendColumn()}; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/DecimalSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/DecimalSumAggregator.java index 5e94c7e776..287d8592a9 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/DecimalSumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/DecimalSumAggregator.java @@ -6,7 +6,6 @@ import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.query.queryplan.aggregators.SingleColumnAggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; /** * Aggregator implementing a sum over {@code column}, for decimal columns. @@ -20,11 +19,6 @@ public DecimalSumAggregator(Column column) { super(column); } - @Override - public DecimalSumAggregator doClone(CloneContext ctx) { - return new DecimalSumAggregator(getColumn()); - } - @Override public void acceptEvent(Bucket bucket, int event) { if (!bucket.has(event, getColumn())) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/IntegerSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/IntegerSumAggregator.java index 6acc6b3fc9..41f9aad46d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/IntegerSumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/IntegerSumAggregator.java @@ -4,7 +4,6 @@ import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.query.queryplan.aggregators.SingleColumnAggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; /** * Aggregator implementing a sum over {@code column}, for Integer columns. @@ -18,11 +17,6 @@ public IntegerSumAggregator(Column column) { super(column); } - @Override - public IntegerSumAggregator doClone(CloneContext ctx) { - return new IntegerSumAggregator(getColumn()); - } - @Override public void acceptEvent(Bucket bucket, int event) { if (!bucket.has(event, getColumn())) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/MoneySumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/MoneySumAggregator.java index eb25f21ab4..838eb7abcc 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/MoneySumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/MoneySumAggregator.java @@ -4,7 +4,6 @@ import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.query.queryplan.aggregators.SingleColumnAggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; /** * Aggregator implementing a sum over {@code column}, for money columns. @@ -18,11 +17,6 @@ public MoneySumAggregator(Column column) { super(column); } - @Override - public MoneySumAggregator doClone(CloneContext ctx) { - return new MoneySumAggregator(getColumn()); - } - @Override public void acceptEvent(Bucket bucket, int event) { if (!bucket.has(event, getColumn())) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/RealSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/RealSumAggregator.java index dad8605d00..7887878a20 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/RealSumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/RealSumAggregator.java @@ -4,7 +4,6 @@ import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.query.queryplan.aggregators.SingleColumnAggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; /** * Aggregator implementing a sum over {@code column}, for real columns. @@ -18,11 +17,6 @@ public RealSumAggregator(Column column) { super(column); } - @Override - public RealSumAggregator doClone(CloneContext ctx) { - return new RealSumAggregator(getColumn()); - } - @Override public void acceptEvent(Bucket bucket, int event) { if (!bucket.has(event, getColumn())) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/AllValuesAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/AllValuesAggregator.java index 671c18f5cc..ef4e22f00e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/AllValuesAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/AllValuesAggregator.java @@ -7,7 +7,6 @@ import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.query.queryplan.aggregators.SingleColumnAggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; /** * Aggregator gathering all unique values in a column, into a Set. @@ -34,11 +33,6 @@ public Set getAggregationResult() { return entries.isEmpty() ? null : entries; } - @Override - public AllValuesAggregator doClone(CloneContext ctx) { - return new AllValuesAggregator<>(getColumn()); - } - @Override public ResultType getResultType() { return new ResultType.ListT(ResultType.resolveResultType(column.getType())); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/FirstValueAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/FirstValueAggregator.java index 290c7382e1..5e1594b69c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/FirstValueAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/FirstValueAggregator.java @@ -8,7 +8,6 @@ import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.queryplan.aggregators.SingleColumnAggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; import lombok.extern.slf4j.Slf4j; /** @@ -77,11 +76,6 @@ public VALUE getAggregationResult() { return (VALUE) selectedBucket.createScriptValue(selectedEvent.getAsInt(), getColumn()); } - @Override - public FirstValueAggregator doClone(CloneContext ctx) { - return new FirstValueAggregator(getColumn()); - } - @Override public ResultType getResultType() { return ResultType.resolveResultType(getColumn().getType()); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/LastValueAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/LastValueAggregator.java index 0a1cee85e1..433034af99 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/LastValueAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/LastValueAggregator.java @@ -8,7 +8,6 @@ import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.queryplan.aggregators.SingleColumnAggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; import lombok.extern.slf4j.Slf4j; /** @@ -78,11 +77,6 @@ public VALUE getAggregationResult() { return (VALUE) selectedBucket.createScriptValue(selectedEvent.getAsInt(), getColumn()); } - @Override - public LastValueAggregator doClone(CloneContext ctx) { - return new LastValueAggregator(getColumn()); - } - @Override public ResultType getResultType() { return ResultType.resolveResultType(getColumn().getType()); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/RandomValueAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/RandomValueAggregator.java index 01024158e7..7c63dec2e2 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/RandomValueAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/RandomValueAggregator.java @@ -6,7 +6,6 @@ import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.query.queryplan.aggregators.SingleColumnAggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; /** * Aggregator, returning a random value of a column. @@ -59,11 +58,6 @@ public VALUE getAggregationResult() { return (VALUE) bucket.createScriptValue(event, getColumn()); } - @Override - public RandomValueAggregator doClone(CloneContext ctx) { - return new RandomValueAggregator(getColumn()); - } - @Override public ResultType getResultType() { return ResultType.resolveResultType(getColumn().getType()); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/clone/CloneContext.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/clone/CloneContext.java deleted file mode 100644 index 08bff95700..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/clone/CloneContext.java +++ /dev/null @@ -1,41 +0,0 @@ -package com.bakdata.conquery.models.query.queryplan.clone; - -import java.util.IdentityHashMap; - -import com.bakdata.conquery.io.storage.ModificationShieldedWorkerStorage; -import lombok.Getter; -import lombok.RequiredArgsConstructor; - -@RequiredArgsConstructor -public class CloneContext { - @Getter - private final ModificationShieldedWorkerStorage storage; - private final IdentityHashMap, CtxCloneable> cloneCache = new IdentityHashMap<>(); - - /** - * Force a clone into the cache (which might not actually be a clone). - */ - public > T inject(T orig, T obj) { - - CtxCloneable old = cloneCache.put(orig, obj); - - if (old != null) { - throw new IllegalStateException(); - } - - return obj; - } - - - public > T clone(T obj) { - CtxCloneable value = cloneCache.get(obj); - if(value == null) { - value = obj.doClone(this); - CtxCloneable old = cloneCache.put(obj, value); - if(old != null) { - throw new IllegalStateException("Object was cloned twice"); - } - } - return (T) value; - } -} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/clone/CtxCloneable.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/clone/CtxCloneable.java deleted file mode 100644 index 63f74656a5..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/clone/CtxCloneable.java +++ /dev/null @@ -1,5 +0,0 @@ -package com.bakdata.conquery.models.query.queryplan.clone; - -public interface CtxCloneable> { - SELF doClone(CloneContext ctx); -} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/filter/FilterNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/filter/FilterNode.java index f845611fda..bb19a4b516 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/filter/FilterNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/filter/FilterNode.java @@ -1,13 +1,12 @@ package com.bakdata.conquery.models.query.queryplan.filter; import com.bakdata.conquery.models.query.queryplan.EventIterating; -import com.bakdata.conquery.models.query.queryplan.clone.CtxCloneable; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.Setter; @AllArgsConstructor -public abstract class FilterNode implements EventIterating, CtxCloneable> { +public abstract class FilterNode implements EventIterating { @Setter @Getter protected FILTER_VALUE filterValue; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/AndNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/AndNode.java index 55f644cc27..684b22ed30 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/AndNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/AndNode.java @@ -4,14 +4,9 @@ import java.util.Collection; import java.util.List; -import com.bakdata.conquery.models.datasets.Table; import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; -import com.bakdata.conquery.models.query.queryplan.DateAggregator; import com.bakdata.conquery.models.query.queryplan.QPNode; import com.bakdata.conquery.models.query.queryplan.QPParentNode; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; -import com.google.common.collect.ListMultimap; -import org.apache.commons.lang3.tuple.Pair; public class AndNode extends QPParentNode { @@ -19,15 +14,6 @@ public AndNode(List children, DateAggregationAction action) { super(children, action); } - private AndNode(List children, ListMultimap childMap, DateAggregator dateAggregator) { - super(children, childMap, dateAggregator); - } - - @Override - public QPNode doClone(CloneContext ctx) { - Pair, ListMultimap> fields = createClonedFields(ctx); - return new AndNode(fields.getLeft(), fields.getRight(), ctx.clone(getDateAggregator())); - } @Override public boolean isContained() { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ConceptNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ConceptNode.java index 1eaf3b212b..656cd21a8f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ConceptNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ConceptNode.java @@ -15,7 +15,6 @@ import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.QPChainNode; import com.bakdata.conquery.models.query.queryplan.QPNode; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; import lombok.Getter; @Getter @@ -131,11 +130,6 @@ public boolean isContained() { return getChild().isContained(); } - @Override - public QPNode doClone(CloneContext ctx) { - return new ConceptNode(ctx.clone(getChild()), concepts, requiredBits, table, selectedSecondaryId); - } - @Override public void collectRequiredTables(Set
requiredTables) { super.collectRequiredTables(requiredTables); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/DateRestrictingNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/DateRestrictingNode.java index 62a62dfcf7..5b8866520b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/DateRestrictingNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/DateRestrictingNode.java @@ -12,7 +12,6 @@ import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.queryplan.QPChainNode; import com.bakdata.conquery.models.query.queryplan.QPNode; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; import lombok.Getter; import lombok.Setter; @@ -78,8 +77,4 @@ public boolean isContained() { return getChild().isContained(); } - @Override - public QPNode doClone(CloneContext ctx) { - return new DateRestrictingNode(restriction, ctx.clone(getChild())); - } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ExternalNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ExternalNode.java index 7ca3894cb6..9b4a64efb7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ExternalNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ExternalNode.java @@ -14,7 +14,6 @@ import com.bakdata.conquery.models.query.queryplan.QPNode; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.SpecialDateUnion; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; import lombok.Getter; import lombok.NonNull; @@ -38,14 +37,7 @@ public void init(Entity entity, QueryExecutionContext context) { super.init(entity, context); contained = includedEntities.get(entity.getId()); } - - @Override - public ExternalNode doClone(CloneContext ctx) { - ExternalNode node = new ExternalNode(table, includedEntities); - node.dateUnion = ctx.clone(dateUnion); - return node; - } - + @Override public void nextTable(QueryExecutionContext ctx, Table currentTable) { if(contained != null) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/FiltersNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/FiltersNode.java index 5aceb31c79..0acf87c81a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/FiltersNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/FiltersNode.java @@ -12,7 +12,6 @@ import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.QPNode; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; import com.bakdata.conquery.models.query.queryplan.filter.EventFilterNode; import com.bakdata.conquery.models.query.queryplan.filter.FilterNode; import lombok.AccessLevel; @@ -113,27 +112,6 @@ public Collection> getDateAggregators() { return eventDateAggregators; } - @Override - public FiltersNode doClone(CloneContext ctx) { - final FiltersNode clone = new FiltersNode(); - - List> filters = new ArrayList<>(this.filters); - filters.replaceAll(ctx::clone); - - clone.setFilters(filters); - - List> eventFilters = new ArrayList<>(this.eventFilters); - eventFilters.replaceAll(ctx::clone); - clone.setEventFilters(eventFilters); - - List> aggregators = new ArrayList<>(this.aggregators); - aggregators.replaceAll(ctx::clone); - - clone.setAggregators(aggregators); - - return clone; - } - @Override public void collectRequiredTables(Set
requiredTables) { super.collectRequiredTables(requiredTables); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/Leaf.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/Leaf.java index f47d378430..7426be5d02 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/Leaf.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/Leaf.java @@ -1,24 +1,18 @@ package com.bakdata.conquery.models.query.queryplan.specific; +import java.util.Collection; +import java.util.Collections; + import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.QPNode; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; - -import java.util.Collection; -import java.util.Collections; public class Leaf extends QPNode { private boolean triggered = false; - - @Override - public QPNode doClone(CloneContext ctx) { - return new Leaf(); - } - + @Override public void acceptEvent(Bucket bucket, int event) { triggered = true; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/NegatingNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/NegatingNode.java index 8af0b5e10a..d045539146 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/NegatingNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/NegatingNode.java @@ -1,5 +1,9 @@ package com.bakdata.conquery.models.query.queryplan.specific; +import java.util.Collection; +import java.util.Collections; +import java.util.Set; + import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; @@ -7,13 +11,8 @@ import com.bakdata.conquery.models.query.queryplan.QPChainNode; import com.bakdata.conquery.models.query.queryplan.QPNode; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; import lombok.NonNull; -import java.util.Collection; -import java.util.Collections; -import java.util.Set; - public class NegatingNode extends QPChainNode { private final DateAggregator dateAggregator; @@ -33,12 +32,7 @@ private NegatingNode(@NonNull QPNode child, @NonNull DateAggregator dateAggregat public void acceptEvent(Bucket bucket, int event) { getChild().acceptEvent(bucket, event); } - - @Override - public NegatingNode doClone(CloneContext ctx) { - return new NegatingNode(ctx.clone(getChild()), this.dateAggregator != null ? ctx.clone(this.dateAggregator): null); - } - + @Override public boolean isContained() { return !getChild().isContained(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/OrNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/OrNode.java index 39c8c7972d..53c8ccb0dc 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/OrNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/OrNode.java @@ -9,9 +9,7 @@ import com.bakdata.conquery.models.query.queryplan.DateAggregator; import com.bakdata.conquery.models.query.queryplan.QPNode; import com.bakdata.conquery.models.query.queryplan.QPParentNode; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; import com.google.common.collect.ListMultimap; -import org.apache.commons.lang3.tuple.Pair; public class OrNode extends QPParentNode { @@ -23,13 +21,7 @@ private OrNode(List children, ListMultimap childMap, Date super(children, childMap, dateAggregator); } - - @Override - public QPNode doClone(CloneContext ctx) { - Pair, ListMultimap> fields = createClonedFields(ctx); - return new OrNode(fields.getLeft(), fields.getRight(), getDateAggregator() != null ? ctx.clone(getDateAggregator()): null); - } - + @Override public boolean isContained() { boolean currently = false; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ValidityDateNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ValidityDateNode.java index f68b02d3c3..96e9002c2a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ValidityDateNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ValidityDateNode.java @@ -6,7 +6,6 @@ import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.queryplan.QPChainNode; import com.bakdata.conquery.models.query.queryplan.QPNode; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; public class ValidityDateNode extends QPChainNode { @@ -42,11 +41,6 @@ public boolean isContained() { return getChild().isContained(); } - @Override - public QPNode doClone(CloneContext ctx) { - return new ValidityDateNode(validityDateColumn, ctx.clone(getChild())); - } - @Override public void nextTable(QueryExecutionContext ctx, Table currentTable) { super.nextTable(ctx.withValidityDateColumn(validityDateColumn), currentTable); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/temporal/SampledNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/temporal/SampledNode.java index 6cb02c8393..c6ab8b5cf5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/temporal/SampledNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/temporal/SampledNode.java @@ -6,9 +6,6 @@ import com.bakdata.conquery.apiv1.query.concept.specific.temporal.TemporalSampler; import com.bakdata.conquery.models.query.queryplan.ConceptQueryPlan; import com.bakdata.conquery.models.query.queryplan.QPNode; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; -import com.bakdata.conquery.models.query.queryplan.clone.CtxCloneable; - import lombok.AllArgsConstructor; import lombok.Getter; import lombok.NoArgsConstructor; @@ -20,7 +17,7 @@ */ @AllArgsConstructor @NoArgsConstructor @Getter @Setter -public class SampledNode implements CtxCloneable { +public class SampledNode { /** * A query plan which should be sampled to a singe day. @@ -32,9 +29,5 @@ public class SampledNode implements CtxCloneable { */ @NotNull @Valid private TemporalSampler sampler; - - @Override - public SampledNode doClone(CloneContext ctx) { - return new SampledNode(child.clone(ctx), sampler); - } + } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/temporal/TemporalQueryNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/temporal/TemporalQueryNode.java index 6124f32ac1..1d1899c32a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/temporal/TemporalQueryNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/temporal/TemporalQueryNode.java @@ -13,7 +13,6 @@ import com.bakdata.conquery.models.query.queryplan.QPNode; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.SpecialDateUnion; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; import lombok.Getter; /** @@ -50,11 +49,6 @@ public TemporalQueryNode(SampledNode reference, SampledNode preceding, Precedenc this.dateUnion = dateUnion; } - @Override - public QPNode doClone(CloneContext ctx) { - return new TemporalQueryNode(ctx.clone((SampledNode) reference), ctx.clone((SampledNode) preceding), matcher, ctx.clone(dateUnion)); - } - /** * Collects required tables of {@link #reference} and {@link #preceding} into {@code out}. * @@ -75,8 +69,8 @@ public void collectRequiredTables(Set
out) { public void init(Entity entity, QueryExecutionContext context) { super.init(entity, context); - reference.getChild().init(entity, context); - preceding.getChild().init(entity, context); + reference.getChild().init(context, entity); + preceding.getChild().init(context, entity); } /** diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java b/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java index 7d1c838fad..2ba9214b25 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java @@ -1,20 +1,18 @@ package com.bakdata.conquery.io.result; +import java.util.List; + import com.bakdata.conquery.models.datasets.concepts.select.Select; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.forms.util.DateContext; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; -import com.bakdata.conquery.models.query.queryplan.clone.CloneContext; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.query.results.MultilineEntityResult; import com.bakdata.conquery.models.query.results.SinglelineEntityResult; import lombok.experimental.UtilityClass; import org.jetbrains.annotations.NotNull; -import java.time.LocalDate; -import java.util.List; - @UtilityClass public class ResultTestUtil { @@ -64,11 +62,6 @@ public TypedSelectDummy(ResultType resultType) { public Aggregator createAggregator() { return new Aggregator() { - @Override - public Aggregator doClone(CloneContext ctx) { - throw new UnsupportedOperationException(); - } - @Override public void acceptEvent(Bucket bucket, int event) { throw new UnsupportedOperationException(); From bd56564c32a316287c66853b031e5c74ca514d28 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Tue, 31 Aug 2021 14:01:09 +0200 Subject: [PATCH 02/16] properly init all Aggregators before execution --- .../filter/AggregationResultFilterNode.java | 9 ++++++++- .../filter/event/MultiSelectFilterNode.java | 1 + .../queryplan/ArrayConceptQueryPlan.java | 8 +++++--- .../query/queryplan/DateAggregator.java | 10 +++++++++- .../query/queryplan/EventIterating.java | 2 +- .../DistinctValuesWrapperAggregator.java | 8 ++++++++ .../MultiDistinctValuesWrapperAggregator.java | 8 ++++++++ .../specific/ConstantValueAggregator.java | 7 +++++++ .../aggregators/specific/CountAggregator.java | 7 +++++++ .../CountQuartersOfDateRangeAggregator.java | 6 ++++++ .../CountQuartersOfDatesAggregator.java | 7 +++++++ .../specific/DateDistanceAggregator.java | 7 +++++++ .../specific/DateUnionAggregator.java | 8 +++++++- .../specific/DurationSumAggregator.java | 6 ++++++ .../specific/EventDateUnionAggregator.java | 8 +++++++- .../specific/EventDurationSumAggregator.java | 6 ++++++ .../specific/ExistsAggregator.java | 12 ++++++++++-- .../specific/MultiSelectAggregator.java | 12 ++++++++++-- .../specific/PrefixTextAggregator.java | 9 ++++++++- .../specific/QuarterAggregator.java | 6 ++++++ .../specific/QuartersInYearAggregator.java | 7 +++++++ .../specific/SelectAggregator.java | 7 +++++++ .../specific/SpecialDateUnion.java | 8 +++++++- .../diffsum/DecimalDiffSumAggregator.java | 19 ++++++++++++++----- .../diffsum/IntegerDiffSumAggregator.java | 9 +++++++++ .../diffsum/MoneyDiffSumAggregator.java | 19 ++++++++++++++----- .../diffsum/RealDiffSumAggregator.java | 19 ++++++++++++++----- .../specific/sum/DecimalSumAggregator.java | 8 ++++++++ .../specific/sum/IntegerSumAggregator.java | 8 ++++++++ .../specific/sum/MoneySumAggregator.java | 9 +++++++++ .../specific/sum/RealSumAggregator.java | 9 +++++++++ .../specific/value/AllValuesAggregator.java | 9 ++++++++- .../specific/value/FirstValueAggregator.java | 8 ++++++++ .../specific/value/LastValueAggregator.java | 8 ++++++++ .../specific/value/RandomValueAggregator.java | 9 +++++++++ .../queryplan/filter/EventFilterNode.java | 12 ++++++++++-- .../query/queryplan/specific/FiltersNode.java | 10 ++++++++++ .../conquery/io/result/ResultTestUtil.java | 9 ++++++++- 38 files changed, 301 insertions(+), 33 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/filter/AggregationResultFilterNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/filter/AggregationResultFilterNode.java index 56852c69b2..399e44f88f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/filter/AggregationResultFilterNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/filter/AggregationResultFilterNode.java @@ -5,13 +5,15 @@ import com.bakdata.conquery.models.datasets.Table; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.query.QueryExecutionContext; +import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.filter.FilterNode; import lombok.Getter; /** * Abstract class for filter nodes acting on aggregation results. - * @param Type of the Aggregator + * + * @param Type of the Aggregator * @param Type of the used FilterValue */ public abstract class AggregationResultFilterNode, FILTER_VALUE> extends FilterNode { @@ -19,6 +21,11 @@ public abstract class AggregationResultFilterNode requiredTables) { requiredTables.add(column.getTable()); } + } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ArrayConceptQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ArrayConceptQueryPlan.java index 32ce40a885..76f10eb255 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ArrayConceptQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ArrayConceptQueryPlan.java @@ -30,7 +30,7 @@ public class ArrayConceptQueryPlan implements QueryPlan public static final int VALIDITY_DATE_POSITION = 0; private List childPlans; @ToString.Exclude - private boolean generateDateAggregation = false; + private boolean generateDateAggregation; private final DateAggregator validityDateAggregator = new DateAggregator(DateAggregationAction.MERGE); public ArrayConceptQueryPlan(boolean generateDateAggregation) { @@ -64,7 +64,7 @@ public void addChildPlans(List childQueries, QueryPlanContext cont } if (generateDateAggregation) { - initDateAggregator(this.validityDateAggregator, childPlans); + initDateAggregator(validityDateAggregator, childPlans); } } @@ -74,7 +74,9 @@ private static void initDateAggregator(DateAggregator validityDateAggregator, Li } } + @Override public void init(QueryExecutionContext ctx, Entity entity) { + childPlans.forEach(child -> child.init(ctx,entity)); } @Override @@ -91,7 +93,7 @@ public Optional execute(QueryExecutionContext ctx, Entit } - Object[] resultValues = new Object[this.getAggregatorSize()]; + Object[] resultValues = new Object[getAggregatorSize()]; // Start with 1 for aggregator values if dateSet needs to be added to the result final int resultOffset = generateDateAggregation ? 1 : 0; int resultInsertIdx = resultOffset; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregator.java index 857acb13d1..2a0809bf1c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregator.java @@ -7,6 +7,8 @@ import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; +import com.bakdata.conquery.models.query.QueryExecutionContext; +import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import lombok.RequiredArgsConstructor; @@ -37,7 +39,13 @@ public void registerAll(Collection> children) { this.children.addAll(children); } - @Override + @Override + public void init(Entity entity, QueryExecutionContext context) { + //TODO don't think this is needed? + children.forEach(child -> init(entity, context)); + } + + @Override public void acceptEvent(Bucket bucket, int event) { throw new UnsupportedOperationException("This Aggregator uses the result of its siblings and does not accept events"); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/EventIterating.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/EventIterating.java index 286d857c8d..ca680ea42c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/EventIterating.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/EventIterating.java @@ -18,7 +18,7 @@ default Set
collectRequiredTables() { return out; } - public void init(QueryExecutionContext ctx); + public void init(Entity entity, QueryExecutionContext context); default void nextTable(QueryExecutionContext ctx, Table currentTable) {} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/DistinctValuesWrapperAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/DistinctValuesWrapperAggregator.java index d800d51e1c..129cdc6ab8 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/DistinctValuesWrapperAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/DistinctValuesWrapperAggregator.java @@ -6,6 +6,8 @@ import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; +import com.bakdata.conquery.models.query.QueryExecutionContext; +import com.bakdata.conquery.models.query.entity.Entity; import lombok.Getter; import org.apache.commons.lang3.ArrayUtils; @@ -36,6 +38,12 @@ public Column[] getRequiredColumns() { return ArrayUtils.add(aggregator.getRequiredColumns(), getColumn()); } + @Override + public void init(Entity entity, QueryExecutionContext context) { + aggregator.init(entity,context); + observed.clear(); + } + @Override public void acceptEvent(Bucket bucket, int event) { if(!bucket.has(event,getColumn())){ diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/MultiDistinctValuesWrapperAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/MultiDistinctValuesWrapperAggregator.java index a420450cc7..e00a013594 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/MultiDistinctValuesWrapperAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/MultiDistinctValuesWrapperAggregator.java @@ -8,6 +8,8 @@ import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; +import com.bakdata.conquery.models.query.QueryExecutionContext; +import com.bakdata.conquery.models.query.entity.Entity; import lombok.Getter; import org.apache.commons.lang3.ArrayUtils; @@ -37,6 +39,12 @@ public Column[] getRequiredColumns() { return ArrayUtils.addAll(aggregator.getRequiredColumns(), getColumns()); } + @Override + public void init(Entity entity, QueryExecutionContext context) { + observed.clear(); + aggregator.init(entity, context); + } + @Override public void acceptEvent(Bucket bucket, int event) { List tuple = new ArrayList<>(columns.length); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/ConstantValueAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/ConstantValueAggregator.java index aa3177d814..8de392a88c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/ConstantValueAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/ConstantValueAggregator.java @@ -2,6 +2,8 @@ import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; +import com.bakdata.conquery.models.query.QueryExecutionContext; +import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import lombok.Getter; import lombok.RequiredArgsConstructor; @@ -23,6 +25,11 @@ public Object getAggregationResult() { return value; } + @Override + public void init(Entity entity, QueryExecutionContext context) { + + } + @Override public void acceptEvent(Bucket bucket, int event) {} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountAggregator.java index 2f75780d00..04b0fda1d4 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountAggregator.java @@ -3,6 +3,8 @@ import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; +import com.bakdata.conquery.models.query.QueryExecutionContext; +import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.SingleColumnAggregator; /** @@ -16,6 +18,11 @@ public CountAggregator(Column column) { super(column); } + @Override + public void init(Entity entity, QueryExecutionContext context) { + count = 0; + } + @Override public void acceptEvent(Bucket bucket, int event) { if (bucket.has(event, getColumn())) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountQuartersOfDateRangeAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountQuartersOfDateRangeAggregator.java index 9725d68083..bdfa52b4df 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountQuartersOfDateRangeAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountQuartersOfDateRangeAggregator.java @@ -13,6 +13,7 @@ import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.query.QueryExecutionContext; +import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.SingleColumnAggregator; import it.unimi.dsi.fastutil.ints.IntOpenHashSet; import it.unimi.dsi.fastutil.ints.IntSet; @@ -32,6 +33,11 @@ public CountQuartersOfDateRangeAggregator(Column column) { super(column); } + @Override + public void init(Entity entity, QueryExecutionContext context) { + quarters.clear(); + } + @Override public void nextTable(QueryExecutionContext ctx, Table currentTable) { dateRestriction = ctx.getDateRestriction(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountQuartersOfDatesAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountQuartersOfDatesAggregator.java index 16109e5e48..eb46f95e18 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountQuartersOfDatesAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountQuartersOfDatesAggregator.java @@ -7,6 +7,8 @@ import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; +import com.bakdata.conquery.models.query.QueryExecutionContext; +import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.SingleColumnAggregator; import it.unimi.dsi.fastutil.ints.IntOpenHashSet; import it.unimi.dsi.fastutil.ints.IntSet; @@ -22,6 +24,11 @@ public CountQuartersOfDatesAggregator(Column column) { super(column); } + @Override + public void init(Entity entity, QueryExecutionContext context) { + quarters.clear(); + } + @Override public void acceptEvent(Bucket bucket, int event) { if (!bucket.has(event, getColumn())) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DateDistanceAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DateDistanceAggregator.java index f07326be97..bce9aeed6a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DateDistanceAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DateDistanceAggregator.java @@ -9,6 +9,7 @@ import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.query.QueryExecutionContext; +import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.SingleColumnAggregator; /** @@ -27,6 +28,12 @@ public DateDistanceAggregator(Column column, ChronoUnit unit) { this.unit = unit; } + @Override + public void init(Entity entity, QueryExecutionContext context) { + hit = false; + result = Long.MAX_VALUE; + } + @Override public void nextTable(QueryExecutionContext ctx, Table currentTable) { if(ctx.getDateRestriction().isAll() || ctx.getDateRestriction().isEmpty()){ diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DateUnionAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DateUnionAggregator.java index f30986991d..a42c7d4dc5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DateUnionAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DateUnionAggregator.java @@ -7,6 +7,7 @@ import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.query.QueryExecutionContext; +import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.SingleColumnAggregator; /** @@ -21,6 +22,11 @@ public DateUnionAggregator(Column column) { super(column); } + @Override + public void init(Entity entity, QueryExecutionContext context) { + set.clear(); + } + @Override public void nextTable(QueryExecutionContext ctx, Table currentTable) { dateRestriction = ctx.getDateRestriction(); @@ -43,7 +49,7 @@ public void acceptEvent(Bucket bucket, int event) { @Override public CDateSet getAggregationResult() { - return set; + return CDateSet.create(set.asRanges()); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DurationSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DurationSumAggregator.java index 1e22caa1b5..285bb01b29 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DurationSumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DurationSumAggregator.java @@ -7,6 +7,7 @@ import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.query.QueryExecutionContext; +import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.SingleColumnAggregator; /** @@ -21,6 +22,11 @@ public DurationSumAggregator(Column column) { super(column); } + @Override + public void init(Entity entity, QueryExecutionContext context) { + set.clear(); + } + @Override public void nextTable(QueryExecutionContext ctx, Table currentTable) { dateRestriction = ctx.getDateRestriction(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDateUnionAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDateUnionAggregator.java index 3b27e20559..85c3522b8f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDateUnionAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDateUnionAggregator.java @@ -8,6 +8,7 @@ import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.query.QueryExecutionContext; +import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import lombok.RequiredArgsConstructor; @@ -30,6 +31,11 @@ public void collectRequiredTables(Set
requiredTables) { requiredTables.addAll(this.requiredTables); } + @Override + public void init(Entity entity, QueryExecutionContext context) { + set.clear(); + } + @Override public void nextTable(QueryExecutionContext ctx, Table currentTable) { validityDateColumn = ctx.getValidityDateColumn(); @@ -43,7 +49,7 @@ public void nextTable(QueryExecutionContext ctx, Table currentTable) { @Override public CDateSet getAggregationResult() { - return set; + return CDateSet.create(set.asRanges()); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDurationSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDurationSumAggregator.java index b2fdb8662b..e3491e7406 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDurationSumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDurationSumAggregator.java @@ -11,6 +11,7 @@ import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.query.QueryExecutionContext; +import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; /** @@ -26,6 +27,11 @@ public class EventDurationSumAggregator implements Aggregator { @CheckForNull private Column validityDateColumn; + @Override + public void init(Entity entity, QueryExecutionContext context) { + set.clear(); + } + @Override public void nextTable(QueryExecutionContext ctx, Table currentTable) { dateRestriction = ctx.getDateRestriction(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/ExistsAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/ExistsAggregator.java index 08accdbfc2..2014e36c1f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/ExistsAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/ExistsAggregator.java @@ -5,6 +5,8 @@ import com.bakdata.conquery.models.datasets.Table; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; +import com.bakdata.conquery.models.query.QueryExecutionContext; +import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.QPNode; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import lombok.RequiredArgsConstructor; @@ -13,14 +15,15 @@ /** * Helper Aggregator, returning if it was used at least once. */ -@RequiredArgsConstructor @ToString(of = {"requiredTables"}) +@RequiredArgsConstructor +@ToString(of = "requiredTables") public class ExistsAggregator implements Aggregator { private final Set
requiredTables; public void setReference(QPNode ref) { - this.reference = ref; + reference = ref; } private QPNode reference; @@ -38,6 +41,11 @@ public void collectRequiredTables(Set
requiredTables) { requiredTables.addAll(this.requiredTables); } + @Override + public void init(Entity entity, QueryExecutionContext context) { + + } + @Override public ResultType getResultType() { return ResultType.BooleanT.INSTANCE; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/MultiSelectAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/MultiSelectAggregator.java index b4961748b3..70061c2ebb 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/MultiSelectAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/MultiSelectAggregator.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.models.query.queryplan.aggregators.specific; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -7,6 +8,8 @@ import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.events.stores.root.StringStore; import com.bakdata.conquery.models.externalservice.ResultType; +import com.bakdata.conquery.models.query.QueryExecutionContext; +import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.SingleColumnAggregator; /** @@ -25,6 +28,11 @@ public MultiSelectAggregator(Column column, String[] selection) { this.hits = new int[selection.length]; } + @Override + public void init(Entity entity, QueryExecutionContext context) { + Arrays.fill(hits, 0); + } + @Override public void nextBlock(Bucket bucket) { StringStore type = (StringStore) bucket.getStore(getColumn()); @@ -57,7 +65,7 @@ public Map getAggregationResult() { for (int i = 0; i < hits.length; i++) { int hit = hits[i]; if (hit > 0) { - out.merge(selection[i], hit, (a,b)->a+b); + out.merge(selection[i], hit, Integer::sum); } } @@ -72,7 +80,7 @@ public ResultType getResultType() { @Override public boolean isOfInterest(Bucket bucket) { for (String selected : selection) { - if(((StringStore) bucket.getStores()[getColumn().getPosition()]).getId(selected) == -1) { + if (((StringStore) bucket.getStores()[getColumn().getPosition()]).getId(selected) == -1) { return false; } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/PrefixTextAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/PrefixTextAggregator.java index f0c21306bd..ad6cacdad3 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/PrefixTextAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/PrefixTextAggregator.java @@ -6,6 +6,8 @@ import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; +import com.bakdata.conquery.models.query.QueryExecutionContext; +import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.SingleColumnAggregator; /** @@ -21,6 +23,11 @@ public PrefixTextAggregator(Column column, String prefix) { this.prefix = prefix; } + @Override + public void init(Entity entity, QueryExecutionContext context) { + entries.clear(); + } + @Override public void acceptEvent(Bucket bucket, int event) { if (!bucket.has(event, getColumn())) { @@ -39,7 +46,7 @@ public void acceptEvent(Bucket bucket, int event) { @Override public Set getAggregationResult() { - return entries.isEmpty() ? null : entries; + return entries.isEmpty() ? null : Set.copyOf(entries); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuarterAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuarterAggregator.java index d4d429bfa8..bf88299551 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuarterAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuarterAggregator.java @@ -13,6 +13,7 @@ import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.query.QueryExecutionContext; +import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import lombok.Data; @@ -33,6 +34,11 @@ public QuarterAggregator(TemporalSampler sampler) { this.sampler = sampler; } + @Override + public void init(Entity entity, QueryExecutionContext context) { + set.clear(); + } + @Override public void nextTable(QueryExecutionContext ctx, Table currentTable) { column = ctx.getValidityDateColumn(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuartersInYearAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuartersInYearAggregator.java index 40fed04568..883b37dfb5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuartersInYearAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuartersInYearAggregator.java @@ -10,6 +10,8 @@ import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; +import com.bakdata.conquery.models.query.QueryExecutionContext; +import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.SingleColumnAggregator; import it.unimi.dsi.fastutil.ints.Int2ObjectMap; import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; @@ -25,6 +27,11 @@ public QuartersInYearAggregator(Column column) { super(column); } + @Override + public void init(Entity entity, QueryExecutionContext context) { + quartersInYear.clear(); + } + @Override public void acceptEvent(Bucket bucket, int event) { if (!bucket.has(event, getColumn())) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SelectAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SelectAggregator.java index eae73349f3..898202e48a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SelectAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SelectAggregator.java @@ -4,6 +4,8 @@ import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.events.stores.root.StringStore; import com.bakdata.conquery.models.externalservice.ResultType; +import com.bakdata.conquery.models.query.QueryExecutionContext; +import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.SingleColumnAggregator; @@ -21,6 +23,11 @@ public SelectAggregator(Column column, String selected) { this.selected = selected; } + @Override + public void init(Entity entity, QueryExecutionContext context) { + hits = 0; + } + @Override public void nextBlock(Bucket bucket) { selectedId = ((StringStore) bucket.getStore(getColumn())).getId(selected); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SpecialDateUnion.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SpecialDateUnion.java index 27217ae1d7..ef842a8087 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SpecialDateUnion.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SpecialDateUnion.java @@ -6,6 +6,7 @@ import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.query.QueryExecutionContext; +import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import lombok.RequiredArgsConstructor; @@ -21,6 +22,11 @@ public class SpecialDateUnion implements Aggregator { private CDateSet dateRestriction; + @Override + public void init(Entity entity, QueryExecutionContext context) { + set.clear(); + } + @Override public void nextTable(QueryExecutionContext ctx, Table table) { currentColumn = ctx.getValidityDateColumn(); @@ -49,7 +55,7 @@ public void merge(CDateSet other){ @Override public CDateSet getAggregationResult() { - return set; + return CDateSet.create(set.asRanges()); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/DecimalDiffSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/DecimalDiffSumAggregator.java index 029517b5cc..58a875357f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/DecimalDiffSumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/DecimalDiffSumAggregator.java @@ -5,6 +5,8 @@ import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; +import com.bakdata.conquery.models.query.QueryExecutionContext; +import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.ColumnAggregator; import lombok.Getter; @@ -13,17 +15,24 @@ */ public class DecimalDiffSumAggregator extends ColumnAggregator { - private boolean hit = false; + private boolean hit; @Getter - private Column addendColumn; + private final Column addendColumn; @Getter - private Column subtrahendColumn; + private final Column subtrahendColumn; private BigDecimal sum = BigDecimal.ZERO; + @Override + public void init(Entity entity, QueryExecutionContext context) { + hit = false; + sum = BigDecimal.ZERO; + } + + public DecimalDiffSumAggregator(Column addend, Column subtrahend) { - this.addendColumn = addend; - this.subtrahendColumn = subtrahend; + addendColumn = addend; + subtrahendColumn = subtrahend; } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/IntegerDiffSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/IntegerDiffSumAggregator.java index ebf97e359a..5826b78f4d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/IntegerDiffSumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/IntegerDiffSumAggregator.java @@ -3,6 +3,8 @@ import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; +import com.bakdata.conquery.models.query.QueryExecutionContext; +import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.ColumnAggregator; import lombok.Getter; @@ -24,6 +26,13 @@ public IntegerDiffSumAggregator(Column addend, Column subtrahend) { this.sum = 0L; } + @Override + public void init(Entity entity, QueryExecutionContext context) { + hit = false; + sum = 0; + } + + @Override public Column[] getRequiredColumns() { return new Column[]{getAddendColumn(), getSubtrahendColumn()}; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/MoneyDiffSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/MoneyDiffSumAggregator.java index f8a32c46f2..7bc80856c2 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/MoneyDiffSumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/MoneyDiffSumAggregator.java @@ -3,6 +3,8 @@ import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; +import com.bakdata.conquery.models.query.QueryExecutionContext; +import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.ColumnAggregator; import lombok.Getter; @@ -12,17 +14,24 @@ public class MoneyDiffSumAggregator extends ColumnAggregator { @Getter - private Column addendColumn; + private final Column addendColumn; @Getter - private Column subtrahendColumn; - private long sum = 0L; + private final Column subtrahendColumn; + private long sum; private boolean hit; public MoneyDiffSumAggregator(Column addend, Column subtrahend) { - this.addendColumn = addend; - this.subtrahendColumn = subtrahend; + addendColumn = addend; + subtrahendColumn = subtrahend; } + @Override + public void init(Entity entity, QueryExecutionContext context) { + hit = false; + sum = 0; + } + + @Override public Column[] getRequiredColumns() { return new Column[]{getAddendColumn(), getSubtrahendColumn()}; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/RealDiffSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/RealDiffSumAggregator.java index 34ffd3d5f0..3def598b59 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/RealDiffSumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/RealDiffSumAggregator.java @@ -3,6 +3,8 @@ import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; +import com.bakdata.conquery.models.query.QueryExecutionContext; +import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.ColumnAggregator; import lombok.Getter; @@ -12,18 +14,25 @@ public class RealDiffSumAggregator extends ColumnAggregator { @Getter - private Column addendColumn; + private final Column addendColumn; @Getter - private Column subtrahendColumn; + private final Column subtrahendColumn; - private double sum = 0; + private double sum; private boolean hit; public RealDiffSumAggregator(Column addend, Column subtrahend) { - this.addendColumn = addend; - this.subtrahendColumn = subtrahend; + addendColumn = addend; + subtrahendColumn = subtrahend; } + @Override + public void init(Entity entity, QueryExecutionContext context) { + hit = false; + sum = 0; + } + + @Override public Column[] getRequiredColumns() { return new Column[]{getAddendColumn(), getSubtrahendColumn()}; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/DecimalSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/DecimalSumAggregator.java index 287d8592a9..e83a7e446d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/DecimalSumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/DecimalSumAggregator.java @@ -5,6 +5,8 @@ import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; +import com.bakdata.conquery.models.query.QueryExecutionContext; +import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.SingleColumnAggregator; /** @@ -19,6 +21,12 @@ public DecimalSumAggregator(Column column) { super(column); } + @Override + public void init(Entity entity, QueryExecutionContext context) { + hit = false; + sum = BigDecimal.ZERO; + } + @Override public void acceptEvent(Bucket bucket, int event) { if (!bucket.has(event, getColumn())) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/IntegerSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/IntegerSumAggregator.java index 41f9aad46d..7fa39d3973 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/IntegerSumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/IntegerSumAggregator.java @@ -3,6 +3,8 @@ import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; +import com.bakdata.conquery.models.query.QueryExecutionContext; +import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.SingleColumnAggregator; /** @@ -17,6 +19,12 @@ public IntegerSumAggregator(Column column) { super(column); } + @Override + public void init(Entity entity, QueryExecutionContext context) { + hit = false; + sum = 0; + } + @Override public void acceptEvent(Bucket bucket, int event) { if (!bucket.has(event, getColumn())) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/MoneySumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/MoneySumAggregator.java index 838eb7abcc..2180dad86c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/MoneySumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/MoneySumAggregator.java @@ -3,6 +3,8 @@ import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; +import com.bakdata.conquery.models.query.QueryExecutionContext; +import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.SingleColumnAggregator; /** @@ -17,6 +19,13 @@ public MoneySumAggregator(Column column) { super(column); } + @Override + public void init(Entity entity, QueryExecutionContext context) { + hit = false; + sum = 0; + } + + @Override public void acceptEvent(Bucket bucket, int event) { if (!bucket.has(event, getColumn())) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/RealSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/RealSumAggregator.java index 7887878a20..0c8cd1a9cb 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/RealSumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/RealSumAggregator.java @@ -3,6 +3,8 @@ import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; +import com.bakdata.conquery.models.query.QueryExecutionContext; +import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.SingleColumnAggregator; /** @@ -17,6 +19,13 @@ public RealSumAggregator(Column column) { super(column); } + @Override + public void init(Entity entity, QueryExecutionContext context) { + hit = false; + sum = 0; + } + + @Override public void acceptEvent(Bucket bucket, int event) { if (!bucket.has(event, getColumn())) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/AllValuesAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/AllValuesAggregator.java index ef4e22f00e..bbfec5ada1 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/AllValuesAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/AllValuesAggregator.java @@ -6,6 +6,8 @@ import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; +import com.bakdata.conquery.models.query.QueryExecutionContext; +import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.SingleColumnAggregator; /** @@ -21,6 +23,11 @@ public AllValuesAggregator(Column column) { super(column); } + @Override + public void init(Entity entity, QueryExecutionContext context) { + entries.clear(); + } + @Override public void acceptEvent(Bucket bucket, int event) { if (bucket.has(event, getColumn())) { @@ -30,7 +37,7 @@ public void acceptEvent(Bucket bucket, int event) { @Override public Set getAggregationResult() { - return entries.isEmpty() ? null : entries; + return entries.isEmpty() ? null : Set.copyOf(entries); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/FirstValueAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/FirstValueAggregator.java index 5e1594b69c..1d3fae3870 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/FirstValueAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/FirstValueAggregator.java @@ -7,6 +7,7 @@ import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.query.QueryExecutionContext; +import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.SingleColumnAggregator; import lombok.extern.slf4j.Slf4j; @@ -28,6 +29,13 @@ public FirstValueAggregator(Column column) { super(column); } + @Override + public void init(Entity entity, QueryExecutionContext context) { + selectedEvent = OptionalInt.empty(); + date = Integer.MAX_VALUE; + selectedBucket = null; + } + @Override public void nextTable(QueryExecutionContext ctx, Table currentTable) { validityDateColumn = ctx.getValidityDateColumn(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/LastValueAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/LastValueAggregator.java index 433034af99..fb2ab7672c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/LastValueAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/LastValueAggregator.java @@ -7,6 +7,7 @@ import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.query.QueryExecutionContext; +import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.SingleColumnAggregator; import lombok.extern.slf4j.Slf4j; @@ -28,6 +29,13 @@ public LastValueAggregator(Column column) { super(column); } + @Override + public void init(Entity entity, QueryExecutionContext context) { + selectedEvent = OptionalInt.empty(); + date = Integer.MIN_VALUE; + selectedBucket = null; + } + @Override public void nextTable(QueryExecutionContext ctx, Table currentTable) { validityDateColumn = ctx.getValidityDateColumn(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/RandomValueAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/RandomValueAggregator.java index 7c63dec2e2..26b6f39be0 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/RandomValueAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/RandomValueAggregator.java @@ -5,6 +5,8 @@ import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; +import com.bakdata.conquery.models.query.QueryExecutionContext; +import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.SingleColumnAggregator; /** @@ -23,6 +25,13 @@ public RandomValueAggregator(Column column) { super(column); } + @Override + public void init(Entity entity, QueryExecutionContext context) { + event = -1; + nValues = 0; + bucket = null; + } + /** * length of sequence = m, but not known at event of sampling *

diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/filter/EventFilterNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/filter/EventFilterNode.java index e97963cb95..2d1410d2b2 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/filter/EventFilterNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/filter/EventFilterNode.java @@ -1,6 +1,8 @@ package com.bakdata.conquery.models.query.queryplan.filter; import com.bakdata.conquery.models.events.Bucket; +import com.bakdata.conquery.models.query.QueryExecutionContext; +import com.bakdata.conquery.models.query.entity.Entity; public abstract class EventFilterNode extends FilterNode { @@ -17,8 +19,14 @@ public final void acceptEvent(Bucket bucket, int event) { hit = true; } - public final boolean isContained(){ + public final boolean isContained() { return hit; } - + + @Override + public void init(Entity entity, QueryExecutionContext context) { + hit = false; + } + + } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/FiltersNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/FiltersNode.java index 0acf87c81a..a5bb012f2a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/FiltersNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/FiltersNode.java @@ -41,6 +41,16 @@ public class FiltersNode extends QPNode { @Setter(AccessLevel.PRIVATE) private List> eventDateAggregators; + @Override + public void init(Entity entity, QueryExecutionContext context) { + super.init(entity, context); + + hit = false; + + filters.forEach(child -> child.init(entity, context)); + aggregators.forEach(child -> child.init(entity, context)); + eventFilters.forEach(child -> child.init(entity, context)); + } public static FiltersNode create(List> filters, List> aggregators, List> eventDateAggregators) { if (filters.isEmpty() && aggregators.isEmpty()) { diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java b/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java index 2ba9214b25..4284aafe84 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java @@ -6,6 +6,8 @@ import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.forms.util.DateContext; +import com.bakdata.conquery.models.query.QueryExecutionContext; +import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.query.results.MultilineEntityResult; @@ -60,7 +62,12 @@ public TypedSelectDummy(ResultType resultType) { @Override public Aggregator createAggregator() { - return new Aggregator() { + return new Aggregator<>() { + + @Override + public void init(Entity entity, QueryExecutionContext context) { + + } @Override public void acceptEvent(Bucket bucket, int event) { From 3b6ae13c5a151b9170f05ce907f2b537a7097513 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Tue, 31 Aug 2021 14:02:16 +0200 Subject: [PATCH 03/16] remove noise --- .../java/com/bakdata/conquery/models/preproc/parser/Parser.java | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/Parser.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/Parser.java index 3a76af5af7..f9a4addd64 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/Parser.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/Parser.java @@ -27,6 +27,7 @@ @Slf4j public abstract class Parser { + @ToString.Exclude private final ConqueryConfig config; private int lines = 0; From 3abfd313bcce4857f3aca4ca7738f3e343a2ccd4 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Tue, 31 Aug 2021 15:25:59 +0200 Subject: [PATCH 04/16] fix SecondaryIdQuery with resetting --- .../apiv1/query/SecondaryIdQuery.java | 11 +--- .../query/queryplan/ConceptQueryPlan.java | 45 ++++----------- .../query/queryplan/SecondaryIdQueryPlan.java | 55 +++++++++++-------- .../queryplan/specific/ExternalNode.java | 1 + .../query/queryplan/specific/FiltersNode.java | 1 + .../SECONDARY_ID/SECONDARY_IDS.test.json | 7 +-- 6 files changed, 51 insertions(+), 69 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java index 81f152cf75..72cae95541 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java @@ -25,7 +25,6 @@ import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; import com.bakdata.conquery.models.query.Visitable; -import com.bakdata.conquery.models.query.queryplan.ConceptQueryPlan; import com.bakdata.conquery.models.query.queryplan.SecondaryIdQueryPlan; import com.bakdata.conquery.models.query.resultinfo.ResultInfoCollector; import com.bakdata.conquery.models.query.resultinfo.SimpleResultInfo; @@ -49,7 +48,7 @@ public class SecondaryIdQuery extends Query { @NotNull protected DateAggregationMode dateAggregationMode = DateAggregationMode.MERGE; - + /** * @apiNote not using {@link ConceptQuery} directly in the API-spec simplifies the API. @@ -68,11 +67,7 @@ public class SecondaryIdQuery extends Query { @Override public SecondaryIdQueryPlan createQueryPlan(QueryPlanContext context) { - context = context.withSelectedSecondaryId(getSecondaryId()); - - final ConceptQueryPlan queryPlan = query.createQueryPlan(context); - - return new SecondaryIdQueryPlan(queryPlan, secondaryId, withSecondaryId, withoutSecondaryId); + return new SecondaryIdQueryPlan(query, context, secondaryId, withSecondaryId, withoutSecondaryId); } @Override @@ -85,7 +80,7 @@ public void collectRequiredQueries(Set> requiredQueries) { public void resolve(QueryResolveContext context) { DateAggregationMode resolvedDateAggregationMode = dateAggregationMode; - if(context.getDateAggregationMode() != null) { + if (context.getDateAggregationMode() != null) { log.trace("Overriding date aggregation mode ({}) with mode from context ({})", dateAggregationMode, context.getDateAggregationMode()); resolvedDateAggregationMode = context.getDateAggregationMode(); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ConceptQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ConceptQueryPlan.java index 1b1c2a1dc0..3b9c5b44d3 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ConceptQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ConceptQueryPlan.java @@ -5,7 +5,6 @@ import java.util.Optional; import java.util.Set; -import com.bakdata.conquery.io.storage.ModificationShieldedWorkerStorage; import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.datasets.Table; @@ -28,9 +27,10 @@ public class ConceptQueryPlan implements QueryPlan { public static final int VALIDITY_DATE_POSITION = 0; + @Getter - @Setter - private ThreadLocal> requiredTables = new ThreadLocal<>(); + private final ThreadLocal> requiredTables = ThreadLocal.withInitial(this::collectRequiredTables); + private QPNode child; @ToString.Exclude protected final List> aggregators = new ArrayList<>(); @@ -38,26 +38,11 @@ public class ConceptQueryPlan implements QueryPlan { private DateAggregator dateAggregator = new DateAggregator(DateAggregationAction.MERGE); public ConceptQueryPlan(boolean generateDateAggregator) { - if (generateDateAggregator){ + if (generateDateAggregator) { aggregators.add(dateAggregator); } } - protected void checkRequiredTables(ModificationShieldedWorkerStorage storage) { - if (requiredTables.get() != null) { - return; - } - - - requiredTables.set(this.collectRequiredTables()); - - // Assert that all tables are actually present - for (Table table : requiredTables.get()) { - if (Dataset.isAllIdsTable(table)) { - continue; - } - } - } public void init(QueryExecutionContext ctx, Entity entity) { this.entity = entity; @@ -68,7 +53,7 @@ public void nextEvent(Bucket bucket, int event) { getChild().acceptEvent(bucket, event); } - protected SinglelineEntityResult result() { + protected SinglelineEntityResult createResult() { Object[] values = new Object[aggregators.size()]; for (int i = 0; i < values.length; i++) { @@ -80,19 +65,13 @@ protected SinglelineEntityResult result() { @Override public Optional execute(QueryExecutionContext ctx, Entity entity) { - + // Only override if none has been set from a higher level ctx = QueryUtils.determineDateAggregatorForContext(ctx, this::getValidityDateAggregator); - checkRequiredTables(ctx.getStorage()); - - if (requiredTables.get().isEmpty()) { - return Optional.empty(); - } - init(ctx, entity); - if(!isOfInterest(entity)){ + if (!isOfInterest(entity)) { return Optional.empty(); } @@ -103,7 +82,7 @@ public Optional execute(QueryExecutionContext ctx, Entit for (Table currentTable : requiredTables.get()) { - if(Dataset.isAllIdsTable(currentTable)){ + if (Dataset.isAllIdsTable(currentTable)) { continue; } @@ -115,7 +94,7 @@ public Optional execute(QueryExecutionContext ctx, Entit for (Bucket bucket : tableBuckets) { - if(bucket == null){ + if (bucket == null) { continue; } @@ -137,7 +116,7 @@ public Optional execute(QueryExecutionContext ctx, Entit } if (isContained()) { - return Optional.of(result()); + return Optional.of(createResult()); } return Optional.empty(); } @@ -164,12 +143,12 @@ public boolean isContained() { @Override public boolean isOfInterest(Entity entity) { - return child.isOfInterest(entity); + return !getRequiredTables().get().isEmpty() && child.isOfInterest(entity); } @Override public Optional> getValidityDateAggregator() { - if(!isAggregateValidityDates()) { + if (!isAggregateValidityDates()) { // The date aggregator was not added to the plan, so we don't collect a validity date return Optional.empty(); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java index e4db85faa2..ac8cfacd37 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java @@ -7,6 +7,7 @@ import java.util.Optional; import java.util.Set; +import com.bakdata.conquery.apiv1.query.ConceptQuery; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.datasets.Column; @@ -15,6 +16,7 @@ import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.identifiable.ids.specific.SecondaryIdDescriptionId; import com.bakdata.conquery.models.query.QueryExecutionContext; +import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.results.MultilineEntityResult; @@ -29,7 +31,7 @@ * one result per distinct value in a {@link SecondaryIdDescriptionId} Column. * * @implNote This class will first execute the Query on all Tables carrying the selected {@link SecondaryIdDescriptionId}. Which will then be joined with all Tables that don't have a {@link SecondaryIdDescriptionId}, or are explicitly excluded (via {@link CQConcept#isExcludeFromSecondaryIdQuery()}. - * + *

* This Query likely uses a lot of memory! */ @RequiredArgsConstructor @@ -38,32 +40,32 @@ public class SecondaryIdQueryPlan implements QueryPlan { public static final int VALIDITY_DATE_POSITION = ConceptQueryPlan.VALIDITY_DATE_POSITION + 1; - private final ConceptQueryPlan query; + private final ConceptQuery query; + private final QueryPlanContext queryPlanContext; + private final SecondaryIdDescription secondaryId; private final Set tablesWithSecondaryId; private final Set

tablesWithoutSecondaryId; - private Map childPerKey = new HashMap<>(); + private final Map childPerKey = new HashMap<>(); + private ConceptQueryPlan queryPlan; /** * This is the same execution as a typical ConceptQueryPlan. The difference * is that this method will create a new cloned child for each distinct * secondaryId it encounters during iteration. + * * @return */ @Override public Optional execute(QueryExecutionContext ctx, Entity entity) { - if (query.getRequiredTables().get().isEmpty()) { - return Optional.empty(); - } - init(ctx, entity); - if (!query.isOfInterest(entity)) { + if (!queryPlan.isOfInterest(entity)) { return Optional.empty(); } @@ -76,7 +78,13 @@ public Optional execute(QueryExecutionContext ctx, Entity executeQueriesWithoutSecondaryId(ctx, entity, currentTable); } + return createResult(entity); + } + /** + * For each secondaryId that is included, create a line, return {@link MultilineEntityResult} containing all results. + */ + private Optional createResult(Entity entity) { List result = new ArrayList<>(childPerKey.values().size()); // Prepend the key (ie the actual SecondaryId) to the result. @@ -86,10 +94,9 @@ public Optional execute(QueryExecutionContext ctx, Entity } // Prepend SecondaryId to result-line. - result.add(ArrayUtils.insert(0, child.getValue().result().getValues(), child.getKey())); + result.add(ArrayUtils.insert(0, child.getValue().createResult().getValues(), child.getKey())); } - if (result.isEmpty()) { return Optional.empty(); } @@ -99,8 +106,8 @@ public Optional execute(QueryExecutionContext ctx, Entity @Override public void init(QueryExecutionContext ctx, Entity entity) { - query.checkRequiredTables(ctx.getStorage()); - query.init(ctx, entity); + queryPlan = query.createQueryPlan(queryPlanContext.withSelectedSecondaryId(secondaryId)); + queryPlan.init(ctx, entity); childPerKey.clear(); } @@ -125,7 +132,7 @@ private void executeQueriesWithSecondaryId(QueryExecutionContext ctx, Entity ent continue; } - if(!isOfInterest(bucket)){ + if (!isOfInterest(bucket)) { continue; } @@ -139,7 +146,7 @@ private void executeQueriesWithSecondaryId(QueryExecutionContext ctx, Entity ent } String key = ((String) bucket.createScriptValue(event, secondaryIdColumnId)); - final ConceptQueryPlan plan = childPerKey.computeIfAbsent(key, k -> this.createChild(secondaryIdColumnId, ctxWithPhase, bucket)); + final ConceptQueryPlan plan = childPerKey.computeIfAbsent(key, k -> createChild(ctxWithPhase, bucket)); plan.nextEvent(bucket, event); } } @@ -170,7 +177,7 @@ private void executeQueriesWithoutSecondaryId(QueryExecutionContext ctx, Entity } private void nextTable(QueryExecutionContext ctx, Table currentTable) { - query.nextTable(ctx, currentTable); + queryPlan.nextTable(ctx, currentTable); for (ConceptQueryPlan c : childPerKey.values()) { QueryExecutionContext context = QueryUtils.determineDateAggregatorForContext(ctx, c::getValidityDateAggregator); c.nextTable(context, currentTable); @@ -178,28 +185,28 @@ private void nextTable(QueryExecutionContext ctx, Table currentTable) { } private void nextBlock(Bucket bucket) { - query.nextBlock(bucket); + queryPlan.nextBlock(bucket); for (ConceptQueryPlan c : childPerKey.values()) { c.nextBlock(bucket); } } private boolean isOfInterest(Bucket bucket) { - return query.isOfInterest(bucket); + return queryPlan.isOfInterest(bucket); } /** * if a new distinct secondaryId was found we create a new clone of the ConceptQueryPlan * and bring it up to speed */ - private ConceptQueryPlan createChild(Column secondaryIdColumn, QueryExecutionContext currentContext, Bucket currentBucket) { + private ConceptQueryPlan createChild(QueryExecutionContext currentContext, Bucket currentBucket) { - ConceptQueryPlan plan = query; + final ConceptQueryPlan plan = query.createQueryPlan(queryPlanContext.withSelectedSecondaryId(secondaryId)); - QueryExecutionContext context = QueryUtils.determineDateAggregatorForContext(currentContext, plan::getValidityDateAggregator); + final QueryExecutionContext context = QueryUtils.determineDateAggregatorForContext(currentContext, plan::getValidityDateAggregator); - plan.init(context, query.getEntity()); - plan.nextTable(context, secondaryIdColumn.getTable()); + plan.init(context, queryPlan.getEntity()); + plan.nextTable(context, currentBucket.getTable()); plan.isOfInterest(currentBucket); plan.nextBlock(currentBucket); @@ -208,12 +215,12 @@ private ConceptQueryPlan createChild(Column secondaryIdColumn, QueryExecutionCon @Override public boolean isOfInterest(Entity entity) { - return query.isOfInterest(entity); + return queryPlan.isOfInterest(entity); } @Override public Optional> getValidityDateAggregator() { - if(!query.isAggregateValidityDates()) { + if (!queryPlan.isAggregateValidityDates()) { return Optional.empty(); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ExternalNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ExternalNode.java index 9b4a64efb7..a173bc58a7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ExternalNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ExternalNode.java @@ -36,6 +36,7 @@ public ExternalNode(Table table, Map includedEntities) { public void init(Entity entity, QueryExecutionContext context) { super.init(entity, context); contained = includedEntities.get(entity.getId()); + dateUnion.init(entity, context); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/FiltersNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/FiltersNode.java index a5bb012f2a..34892ae10c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/FiltersNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/FiltersNode.java @@ -50,6 +50,7 @@ public void init(Entity entity, QueryExecutionContext context) { filters.forEach(child -> child.init(entity, context)); aggregators.forEach(child -> child.init(entity, context)); eventFilters.forEach(child -> child.init(entity, context)); + eventDateAggregators.forEach(child -> child.init(entity, context)); } public static FiltersNode create(List> filters, List> aggregators, List> eventDateAggregators) { diff --git a/backend/src/test/resources/tests/query/SECONDARY_ID/SECONDARY_IDS.test.json b/backend/src/test/resources/tests/query/SECONDARY_ID/SECONDARY_IDS.test.json index 4b805c8d66..bf3ca74c39 100644 --- a/backend/src/test/resources/tests/query/SECONDARY_ID/SECONDARY_IDS.test.json +++ b/backend/src/test/resources/tests/query/SECONDARY_ID/SECONDARY_IDS.test.json @@ -14,7 +14,6 @@ ], "type": "CONCEPT", "excludeFromSecondaryIdQuery" : false, - "label": "vs", "tables": [ { "id": "number.number_connector", @@ -49,11 +48,11 @@ }, "concepts": [ { - "label": "number", + "name": "number", "type": "TREE", "connectors": [ { - "label": "number_connector", + "name": "number_connector", "table": "table1", "validityDates": { "label": "datum", @@ -67,7 +66,7 @@ } }, { - "label": "number_connector2", + "name": "number_connector2", "table": "table12", "validityDates": { "label": "datum", From a4e14e906fa90fe7ee070bb89bccaf413567d81a Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Tue, 31 Aug 2021 16:42:42 +0200 Subject: [PATCH 05/16] Fix special queries --- .../temporal/CQBeforeOrSameTemporalQuery.java | 6 +++--- .../models/forms/managed/AbsoluteFormQueryPlan.java | 1 + .../conquery/models/forms/managed/FormQueryPlan.java | 11 +++++++---- .../conquery/models/forms/util/ResultModifier.java | 2 ++ .../models/query/queryplan/ArrayConceptQueryPlan.java | 2 ++ .../models/query/queryplan/DateAggregator.java | 2 +- .../models/query/queryplan/specific/Leaf.java | 7 +++++++ .../temporal/BeforeOrSameTemporalMatcher.java | 2 +- .../temporal/DaysBeforeOrNeverPrecedenceMatcher.java | 11 ++++------- .../temporal/DaysBeforePrecedenceMatcher.java | 2 +- .../specific/temporal/TemporalQueryNode.java | 10 +++++----- 11 files changed, 34 insertions(+), 22 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/temporal/CQBeforeOrSameTemporalQuery.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/temporal/CQBeforeOrSameTemporalQuery.java index c48bfa74e6..4689175553 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/temporal/CQBeforeOrSameTemporalQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/temporal/CQBeforeOrSameTemporalQuery.java @@ -25,9 +25,9 @@ public QPNode createQueryPlan(QueryPlanContext ctx, ConceptQueryPlan plan) { plan.getDateAggregator().register(dateAggregator); return new TemporalQueryNode( - index.createQueryPlan(ctx), - preceding.createQueryPlan(ctx), - new BeforeOrSameTemporalMatcher(), + index.createQueryPlan(ctx), + preceding.createQueryPlan(ctx), + new BeforeOrSameTemporalMatcher(), dateAggregator ); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/AbsoluteFormQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/AbsoluteFormQueryPlan.java index dacf33beb3..a0301d81d6 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/AbsoluteFormQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/AbsoluteFormQueryPlan.java @@ -29,6 +29,7 @@ public Optional execute(QueryExecutionContext ctx, Entity // Don't set the query date aggregator here because the subqueries should set their aggregator independently Optional preResult = query.execute(ctx, entity); + if (preResult.isEmpty()) { return Optional.empty(); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/FormQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/FormQueryPlan.java index 19c5b254c5..42b39cce66 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/FormQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/FormQueryPlan.java @@ -25,7 +25,9 @@ public class FormQueryPlan implements QueryPlan { private final List dateContexts; private final ArrayConceptQueryPlan features; + private final int constantCount; + private final List subPlans = new ArrayList<>(); public FormQueryPlan(List dateContexts, ArrayConceptQueryPlan features) { @@ -40,6 +42,7 @@ public FormQueryPlan(List dateContexts, ArrayConceptQueryPlan featu // Either all date contexts have an relative event date or none has one boolean withRelativeEventdate = dateContexts.get(0).getEventDate() != null; + for (DateContext dateContext : dateContexts) { if ((dateContext.getEventDate() == null) == withRelativeEventdate) { throw new IllegalStateException("Queryplan has absolute AND relative date contexts. Only one kind is allowed."); @@ -51,7 +54,6 @@ public FormQueryPlan(List dateContexts, ArrayConceptQueryPlan featu @Override public Optional execute(QueryExecutionContext ctx, Entity entity) { - init(ctx, entity); if (!isOfInterest(entity)) { @@ -69,7 +71,6 @@ public Optional execute(QueryExecutionContext ctx, Entity QueryExecutionContext innerContext = QueryUtils.determineDateAggregatorForContext(ctx, features::getValidityDateAggregator) .withDateRestriction(dateRestriction); - features.init(innerContext, entity); Optional subResult = features.execute(innerContext, entity); @@ -81,8 +82,10 @@ public Optional execute(QueryExecutionContext ctx, Entity resultValues.addAll( ResultModifier.modify( subResult.get(), - ResultModifier.existAggValuesSetterFor(features.getAggregators(), OptionalInt.of(0)).unaryAndThen(v -> addConstants(v, dateContext)) - ).listResultLines() + ResultModifier.existAggValuesSetterFor(features.getAggregators(), OptionalInt.of(0)) + .unaryAndThen(v -> addConstants(v, dateContext)) + ) + .listResultLines() ); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/util/ResultModifier.java b/backend/src/main/java/com/bakdata/conquery/models/forms/util/ResultModifier.java index 9816866697..b0a68a799f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/util/ResultModifier.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/util/ResultModifier.java @@ -9,7 +9,9 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.specific.ExistsAggregator; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.util.functions.ChainableUnaryOperator; +import lombok.experimental.UtilityClass; +@UtilityClass public class ResultModifier { /** diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ArrayConceptQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ArrayConceptQueryPlan.java index 76f10eb255..4073ae3d36 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ArrayConceptQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ArrayConceptQueryPlan.java @@ -76,6 +76,7 @@ private static void initDateAggregator(DateAggregator validityDateAggregator, Li @Override public void init(QueryExecutionContext ctx, Entity entity) { + validityDateAggregator.init(entity, ctx); childPlans.forEach(child -> child.init(ctx,entity)); } @@ -127,6 +128,7 @@ public Optional execute(QueryExecutionContext ctx, Entit // aggregators. resultInsertIdx = nextIndex(resultInsertIdx, child); } + if (!containedInChildQueries) { // None of the subqueries contained an result return Optional.empty(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregator.java index 2a0809bf1c..55fa03bc45 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregator.java @@ -42,7 +42,7 @@ public void registerAll(Collection> children) { @Override public void init(Entity entity, QueryExecutionContext context) { //TODO don't think this is needed? - children.forEach(child -> init(entity, context)); + children.forEach(child -> child.init(entity, context)); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/Leaf.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/Leaf.java index 7426be5d02..d395266de3 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/Leaf.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/Leaf.java @@ -5,6 +5,7 @@ import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.events.Bucket; +import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.QPNode; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; @@ -13,6 +14,12 @@ public class Leaf extends QPNode { private boolean triggered = false; + @Override + public void init(Entity entity, QueryExecutionContext context) { + super.init(entity, context); + triggered = false; + } + @Override public void acceptEvent(Bucket bucket, int event) { triggered = true; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/temporal/BeforeOrSameTemporalMatcher.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/temporal/BeforeOrSameTemporalMatcher.java index a13f7d558e..6b3e88a24f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/temporal/BeforeOrSameTemporalMatcher.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/temporal/BeforeOrSameTemporalMatcher.java @@ -18,7 +18,7 @@ public void removePreceding(CDateSet preceding, int sample) { @Override public boolean isContained(OptionalInt reference, OptionalInt preceding) { - if (!preceding.isPresent() || !reference.isPresent()) { + if (preceding.isEmpty() || reference.isEmpty()) { return false; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/temporal/DaysBeforeOrNeverPrecedenceMatcher.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/temporal/DaysBeforeOrNeverPrecedenceMatcher.java index 5772f115bf..458e639992 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/temporal/DaysBeforeOrNeverPrecedenceMatcher.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/temporal/DaysBeforeOrNeverPrecedenceMatcher.java @@ -26,18 +26,15 @@ public void removePreceding(CDateSet preceding, int sample) { @Override public boolean isContained(OptionalInt reference, OptionalInt preceding) { - if (!reference.isPresent()) { + if (reference.isEmpty()) { return false; } - // never - if (!preceding.isPresent()) + if (preceding.isEmpty()) { return true; + } // days before - if ((reference.getAsInt() - preceding.getAsInt()) > days) - return true; - - return false; + return (reference.getAsInt() - preceding.getAsInt()) > days; } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/temporal/DaysBeforePrecedenceMatcher.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/temporal/DaysBeforePrecedenceMatcher.java index 433fadeed7..c53fc17d2c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/temporal/DaysBeforePrecedenceMatcher.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/temporal/DaysBeforePrecedenceMatcher.java @@ -26,7 +26,7 @@ public void removePreceding(CDateSet preceding, int sample) { @Override public boolean isContained(OptionalInt reference, OptionalInt preceding) { - if (!preceding.isPresent() || !reference.isPresent()) { + if (preceding.isEmpty() || reference.isEmpty()) { return false; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/temporal/TemporalQueryNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/temporal/TemporalQueryNode.java index 1d1899c32a..190ca4a9b7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/temporal/TemporalQueryNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/temporal/TemporalQueryNode.java @@ -25,22 +25,22 @@ public class TemporalQueryNode extends QPNode { /** * Matcher to be used when testing for inclusion. */ - private PrecedenceMatcher matcher; + private final PrecedenceMatcher matcher; /** * QueryPlan for the events to be compared to. */ - private SampledNode reference; + private final SampledNode reference; /** * QueryPlan for the events being compared. */ - private SampledNode preceding; + private final SampledNode preceding; /** * The {@link SpecialDateUnion} to be fed with the included dataset. */ - private SpecialDateUnion dateUnion; + private final SpecialDateUnion dateUnion; public TemporalQueryNode(SampledNode reference, SampledNode preceding, PrecedenceMatcher matcher, SpecialDateUnion dateUnion) { this.reference = reference; @@ -63,7 +63,6 @@ public void collectRequiredTables(Set
out) { /** * Initializes the {@link TemporalQueryNode} and its children. * - * @param entity the Entity to be worked on. */ @Override public void init(Entity entity, QueryExecutionContext context) { @@ -71,6 +70,7 @@ public void init(Entity entity, QueryExecutionContext context) { reference.getChild().init(context, entity); preceding.getChild().init(context, entity); + dateUnion.init(entity, context); } /** From ea52064135387cae4fb0a3a736db8f301e106054 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 31 Aug 2021 14:52:14 +0000 Subject: [PATCH 06/16] Update AutoDoc --- docs/Concept JSONs.md | 2 +- docs/REST API JSONs.md | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/Concept JSONs.md b/docs/Concept JSONs.md index 981a7a2c92..6eff536443 100644 --- a/docs/Concept JSONs.md +++ b/docs/Concept JSONs.md @@ -101,7 +101,7 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/GroovyCondition.java#L34) | script | `String` | `null` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/GroovyCondition.java#L35) | script | `String` | `null` | | |

### NOT [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/NotCondition.java#L14-L16) diff --git a/docs/REST API JSONs.md b/docs/REST API JSONs.md index 7a503dc1e4..329444c2fb 100644 --- a/docs/REST API JSONs.md +++ b/docs/REST API JSONs.md @@ -354,7 +354,7 @@ Supported Fields: | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQuery.java#L49) | timeUnit | one of DAYS, QUARTERS, YEARS | ? | | |

-### SECONDARY_ID_QUERY [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java#L37) +### SECONDARY_ID_QUERY [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java#L36)
Details

@@ -365,12 +365,12 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java#L50) | dateAggregationMode | one of NONE, MERGE, INTERSECT, LOGICAL | `"MERGE"` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java#L54-L56) | query | [CONCEPT_QUERY](#CONCEPT_QUERY) | ␀ | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java#L43) | root | [@NotNull CQElement](#Base-CQElement) | `null` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java#L46) | secondaryId | ID of `@NsIdRef @NotNull SecondaryIdDescription` | `null` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java#L60) | withSecondaryId | list of ID of `@NsIdRefCollection Set` | ␀ | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java#L64) | withoutSecondaryId | list of ID of `@NsIdRefCollection Set

` | ␀ | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java#L49) | dateAggregationMode | one of NONE, MERGE, INTERSECT, LOGICAL | `"MERGE"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java#L53-L55) | query | [CONCEPT_QUERY](#CONCEPT_QUERY) | ␀ | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java#L42) | root | [@NotNull CQElement](#Base-CQElement) | `null` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java#L45) | secondaryId | ID of `@NsIdRef @NotNull SecondaryIdDescription` | `null` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java#L59) | withSecondaryId | list of ID of `@NsIdRefCollection Set` | ␀ | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java#L63) | withoutSecondaryId | list of ID of `@NsIdRefCollection Set
` | ␀ | | |

### TABLE_EXPORT [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java#L49-L51) From e9dd9bbf13daa498145724ac0fcf6dfad48fa061 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Tue, 31 Aug 2021 17:18:47 +0200 Subject: [PATCH 07/16] remove unused code --- .../models/forms/managed/FormQueryPlan.java | 27 ++++++++----------- .../query/queryplan/DateAggregator.java | 4 +-- 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/FormQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/FormQueryPlan.java index 42b39cce66..fb533702b2 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/FormQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/FormQueryPlan.java @@ -11,8 +11,6 @@ import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.ArrayConceptQueryPlan; -import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; -import com.bakdata.conquery.models.query.queryplan.DateAggregator; import com.bakdata.conquery.models.query.queryplan.QueryPlan; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.results.MultilineEntityResult; @@ -28,8 +26,6 @@ public class FormQueryPlan implements QueryPlan { private final int constantCount; - private final List subPlans = new ArrayList<>(); - public FormQueryPlan(List dateContexts, ArrayConceptQueryPlan features) { this.dateContexts = dateContexts; this.features = features; @@ -41,14 +37,14 @@ public FormQueryPlan(List dateContexts, ArrayConceptQueryPlan featu } // Either all date contexts have an relative event date or none has one - boolean withRelativeEventdate = dateContexts.get(0).getEventDate() != null; + boolean withRelativeEventDate = dateContexts.get(0).getEventDate() != null; for (DateContext dateContext : dateContexts) { - if ((dateContext.getEventDate() == null) == withRelativeEventdate) { - throw new IllegalStateException("Queryplan has absolute AND relative date contexts. Only one kind is allowed."); + if ((dateContext.getEventDate() == null) == withRelativeEventDate) { + throw new IllegalStateException("QueryPlan has absolute AND relative date contexts. Only one kind is allowed."); } } - constantCount = withRelativeEventdate ? 4 : 3; // resolution indicator, index value, (event date,) date range + constantCount = withRelativeEventDate ? 4 : 3; // resolution indicator, index value, (event date,) date range } @Override @@ -98,10 +94,15 @@ public void init(QueryExecutionContext ctx, Entity entity) { } private MultilineEntityResult createResultForNotContained(Entity entity, DateContext dateContext) { + List result = new ArrayList<>(); result.add(new Object[features.getAggregatorSize()]); - return ResultModifier.modify(new MultilineEntityResult(entity.getId(), result), ResultModifier.existAggValuesSetterFor(getAggregators(), OptionalInt.of(0)) - .unaryAndThen(v -> addConstants(v, dateContext))); + + return ResultModifier.modify( + new MultilineEntityResult(entity.getId(), result), + ResultModifier.existAggValuesSetterFor(getAggregators(), OptionalInt.of(0)) + .unaryAndThen(v -> addConstants(v, dateContext)) + ); } public List> getAggregators() { @@ -141,12 +142,6 @@ public boolean isOfInterest(Entity entity) { @Override public Optional> getValidityDateAggregator() { - DateAggregator agg = new DateAggregator(DateAggregationAction.MERGE); - - subPlans.forEach( - p -> p.getValidityDateAggregator().ifPresent(agg::register) - ); - return Optional.empty(); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregator.java index 55fa03bc45..8c888beda8 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregator.java @@ -22,13 +22,13 @@ public class DateAggregator implements Aggregator { private final DateAggregationAction action; - private Set> children = new HashSet<>(); + private final Set> children = new HashSet<>(); /** * Register {@Aggregator}s from lower levels for the final result generation. */ public void register(Aggregator child) { - this.children.add(child); + children.add(child); } From b0ae94e07b724363014768452ddbf3242148b280 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Tue, 31 Aug 2021 17:18:57 +0200 Subject: [PATCH 08/16] undo change for reuse --- .../conquery/models/config/ThreadPoolDefinition.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ThreadPoolDefinition.java b/backend/src/main/java/com/bakdata/conquery/models/config/ThreadPoolDefinition.java index 681820867f..661bc775a3 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ThreadPoolDefinition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ThreadPoolDefinition.java @@ -11,16 +11,18 @@ import lombok.Setter; import lombok.ToString; -@Getter @Setter @ToString +@Getter +@Setter +@ToString public class ThreadPoolDefinition { @Min(0) - private int minThreads = 1; + private int minThreads = Runtime.getRuntime().availableProcessors(); @Min(1) - private int maxThreads = 2; + private int maxThreads = Runtime.getRuntime().availableProcessors(); private boolean allowCoreThreadTimeOut = false; private Duration keepAliveTime = Duration.seconds(60); private Duration shutdownTime = Duration.hours(1); - + public ThreadPoolExecutor createService(String nameFormat) { final ThreadPoolExecutor executor = new ThreadPoolExecutor( minThreads, From fd9c49add4d6449a790756cf54ffcc0baba146d7 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Tue, 31 Aug 2021 17:44:55 +0200 Subject: [PATCH 09/16] ImmutableSet retains the prior order --- .../aggregators/specific/value/AllValuesAggregator.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/AllValuesAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/AllValuesAggregator.java index bbfec5ada1..1b1f2921e7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/AllValuesAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/AllValuesAggregator.java @@ -9,6 +9,7 @@ import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.SingleColumnAggregator; +import com.google.common.collect.ImmutableSet; /** * Aggregator gathering all unique values in a column, into a Set. @@ -37,7 +38,7 @@ public void acceptEvent(Bucket bucket, int event) { @Override public Set getAggregationResult() { - return entries.isEmpty() ? null : Set.copyOf(entries); + return entries.isEmpty() ? null : ImmutableSet.copyOf(entries); } @Override From 25c7b6843706a16553d5c51676d070856a5bf5a7 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Tue, 31 Aug 2021 18:07:59 +0200 Subject: [PATCH 10/16] I knew i forgot one --- .../queryplan/aggregators/specific/PrefixTextAggregator.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/PrefixTextAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/PrefixTextAggregator.java index ad6cacdad3..d63482871b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/PrefixTextAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/PrefixTextAggregator.java @@ -9,6 +9,7 @@ import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.aggregators.SingleColumnAggregator; +import com.google.common.collect.ImmutableSet; /** * Aggregator, returning all values of a column, beginning with a specified value. @@ -46,7 +47,7 @@ public void acceptEvent(Bucket bucket, int event) { @Override public Set getAggregationResult() { - return entries.isEmpty() ? null : Set.copyOf(entries); + return entries.isEmpty() ? null : ImmutableSet.copyOf(entries); } @Override From 5b258a98ff17c00e02557af2b76894701b706b3e Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Wed, 1 Sep 2021 14:47:39 +0200 Subject: [PATCH 11/16] pull init into separate phase before execute --- .../forms/managed/AbsoluteFormQueryPlan.java | 3 +- .../forms/managed/EntityDateQueryPlan.java | 3 +- .../forms/managed/RelativeFormQueryPlan.java | 108 +++++++++--------- .../conquery/models/query/QueryJob.java | 9 +- .../queryplan/ArrayConceptQueryPlan.java | 3 - .../query/queryplan/ConceptQueryPlan.java | 2 - .../query/queryplan/SecondaryIdQueryPlan.java | 22 +++- .../query/queryplan/TableExportQueryPlan.java | 2 +- 8 files changed, 85 insertions(+), 67 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/AbsoluteFormQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/AbsoluteFormQueryPlan.java index a0301d81d6..af51219d58 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/AbsoluteFormQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/AbsoluteFormQueryPlan.java @@ -20,7 +20,8 @@ public class AbsoluteFormQueryPlan implements QueryPlan { @Override public void init(QueryExecutionContext ctxt, Entity entity) { - + query.init(ctxt, entity); + subPlan.init(ctxt, entity); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/EntityDateQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/EntityDateQueryPlan.java index 14c6433fc5..4769e87f7d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/EntityDateQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/EntityDateQueryPlan.java @@ -33,7 +33,8 @@ public class EntityDateQueryPlan implements QueryPlan { @Override public void init(QueryExecutionContext ctxt, Entity entity) { - + query.init(ctxt, entity); + features.init(ctxt, entity); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQueryPlan.java index 7673f4befb..85fc7af267 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQueryPlan.java @@ -29,7 +29,8 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -@Getter @RequiredArgsConstructor +@Getter +@RequiredArgsConstructor public class RelativeFormQueryPlan implements QueryPlan { // Position of fixed columns in the result. (This is without identifier column[s], they are added upon result rendering) @@ -42,19 +43,26 @@ public class RelativeFormQueryPlan implements QueryPlan { private final QueryPlan query; private final ArrayConceptQueryPlan featurePlan; private final ArrayConceptQueryPlan outcomePlan; + private final TemporalSampler indexSelector; private final IndexPlacement indexPlacement; + private final int timeCountBefore; private final int timeCountAfter; private final DateContext.CalendarUnit timeUnit; private final List resolutionsAndAlignmentMap; - private final transient List featureSubqueries = new ArrayList<>(); - private final transient List outcomeSubqueries = new ArrayList<>(); + private FormQueryPlan featureSubquery = null; + private FormQueryPlan outcomeSubquery = null; @Override public void init(QueryExecutionContext ctxt, Entity entity) { + query.init(ctxt, entity); + featurePlan.init(ctxt, entity); + outcomePlan.init(ctxt, entity); + featureSubquery = null; + outcomeSubquery = null; } @Override @@ -81,33 +89,32 @@ public Optional execute(QueryExecutionContext ctx, Entity List results = new ArrayList<>(); results.add(new Object[size]); return Optional.of( - ResultModifier.modify(new MultilineEntityResult(entity.getId(), results), ResultModifier.existAggValuesSetterFor(getAggregators(), OptionalInt.of(getFirstAggregatorPosition()))) + ResultModifier.modify(new MultilineEntityResult(entity.getId(), results), ResultModifier.existAggValuesSetterFor(getAggregators(), OptionalInt + .of(getFirstAggregatorPosition()))) ); } int sample = sampled.getAsInt(); - List contexts = DateContext - .generateRelativeContexts(sample, indexPlacement, timeCountBefore, timeCountAfter, timeUnit, resolutionsAndAlignmentMap); - - // create feature and outcome plans - FormQueryPlan featureSubquery = createSubQuery(featurePlan, contexts, FeatureGroup.FEATURE); - FormQueryPlan outcomeSubquery = createSubQuery(outcomePlan, contexts, FeatureGroup.OUTCOME); + List contexts = + DateContext.generateRelativeContexts(sample, indexPlacement, timeCountBefore, timeCountAfter, timeUnit, resolutionsAndAlignmentMap); - featureSubqueries.add(featureSubquery); - outcomeSubqueries.add(outcomeSubquery); + // create feature and outcome plans + featureSubquery = createSubQuery(featurePlan, contexts, FeatureGroup.FEATURE); + outcomeSubquery = createSubQuery(outcomePlan, contexts, FeatureGroup.OUTCOME); // determine result length and check against aggregators in query int featureLength = featureSubquery.columnCount(); int outcomeLength = outcomeSubquery.columnCount(); - + featureSubquery.init(ctx, entity); + outcomeSubquery.init(ctx, entity); Optional featureResult = featureSubquery.execute(ctx, entity); Optional outcomeResult = outcomeSubquery.execute(ctx, entity); // determine result length and check against aggregators in query - checkResultWidth(featureResult.get(), featureLength); - checkResultWidth(outcomeResult.get(), outcomeLength); + assertResultWidth(featureResult.get(), featureLength); + assertResultWidth(outcomeResult.get(), outcomeLength); List featureResultValues = featureResult.get().getValues(); List outcomeResultValues = outcomeResult.get().getValues(); @@ -155,23 +162,23 @@ public Optional execute(QueryExecutionContext ctx, Entity } - private int getFeatureDateRangePosition(){ - return featurePlan.getAggregatorSize() > 0? 3 : -1; + private int getFeatureDateRangePosition() { + return featurePlan.getAggregatorSize() > 0 ? 3 : -1; } - private int getOutcomeDateRangePosition(){ - if (outcomePlan.getAggregatorSize() > 0){ - return featurePlan.getAggregatorSize() > 0? 4 : 3; + private int getOutcomeDateRangePosition() { + if (outcomePlan.getAggregatorSize() > 0) { + return featurePlan.getAggregatorSize() > 0 ? 4 : 3; } return -1; } - private int getFirstAggregatorPosition(){ + private int getFirstAggregatorPosition() { if (outcomePlan.getAggregatorSize() <= 0 && featurePlan.getAggregatorSize() <= 0) { throw new ConqueryError.ExecutionProcessingError(); } if (outcomePlan.getAggregatorSize() > 0 && featurePlan.getAggregatorSize() <= 0 - || featurePlan.getAggregatorSize() > 0 && outcomePlan.getAggregatorSize() <= 0){ + || featurePlan.getAggregatorSize() > 0 && outcomePlan.getAggregatorSize() <= 0) { return 4; } return 5; @@ -189,34 +196,37 @@ private int calculateCompleteLength() { //return featureLength + outcomeLength - 3/* ^= [RESOLUTION], [INDEX], [EVENTDATE] */; } - private int checkResultWidth(EntityResult subResult, int resultWidth) { + private void assertResultWidth(EntityResult subResult, int resultWidth) { int resultColumnCount = subResult.columnCount(); - if(resultColumnCount != resultWidth) { - throw new IllegalStateException(String - .format("Aggregator number (%d) and result number (%d) are not the same", resultWidth, resultColumnCount)); + if (resultColumnCount == resultWidth) { + return; } - return resultWidth; + + throw new IllegalStateException(String.format("Aggregator number (%d) and result number (%d) are not the same", resultWidth, resultColumnCount)); } private boolean hasCompleteDateContexts(List contexts) { - if(contexts.isEmpty()){ + if (contexts.isEmpty()) { return false; } - if (featurePlan.getAggregatorSize() > 0 && outcomePlan.getAggregatorSize() > 0) { - // We have features and outcomes check if both have complete date ranges (they should be at the beginning of the list) - return contexts.size()>=2 - && contexts.get(0).getSubdivisionMode().equals(DateContext.Resolution.COMPLETE) - && contexts.get(1).getSubdivisionMode().equals(DateContext.Resolution.COMPLETE) - && !contexts.get(0).getFeatureGroup().equals(contexts.get(1).getFeatureGroup()); + + if (featurePlan.getAggregatorSize() <= 0 || outcomePlan.getAggregatorSize() <= 0) { + // Otherwise, if only features or outcomes are given check the first date context. The empty feature/outcome query + // will still return an empty result which will be merged with to a complete result. + return DateContext.Resolution.COMPLETE.equals(contexts.get(0).getSubdivisionMode()); } - // Otherwise, if only features or outcomes are given check the first date context. The empty feature/outcome query - // will still return an empty result which will be merged with to a complete result. - return contexts.get(0).getSubdivisionMode().equals(DateContext.Resolution.COMPLETE); + + // We have features and outcomes check if both have complete date ranges (they should be at the beginning of the list) + return contexts.size() >= 2 + && DateContext.Resolution.COMPLETE.equals(contexts.get(0).getSubdivisionMode()) + && DateContext.Resolution.COMPLETE.equals(contexts.get(1).getSubdivisionMode()) + && !contexts.get(0).getFeatureGroup().equals(contexts.get(1).getFeatureGroup()); + } - private FormQueryPlan createSubQuery(ArrayConceptQueryPlan subPlan, List contexts, FeatureGroup featureGroup) { + private static FormQueryPlan createSubQuery(ArrayConceptQueryPlan subPlan, List contexts, FeatureGroup featureGroup) { List list = new ArrayList<>(contexts); list.removeIf(dctx -> dctx.getFeatureGroup() != featureGroup); @@ -225,18 +235,16 @@ private FormQueryPlan createSubQuery(ArrayConceptQueryPlan subPlan, List> getAggregators() { - return ImmutableList.copyOf(Iterables.concat(featurePlan.getAggregators(),outcomePlan.getAggregators())); + return ImmutableList.copyOf(Iterables.concat(featurePlan.getAggregators(), outcomePlan.getAggregators())); } @Override @@ -260,13 +269,8 @@ public boolean isOfInterest(Entity entity) { public Optional> getValidityDateAggregator() { DateAggregator agg = new DateAggregator(DateAggregationAction.MERGE); - for (FormQueryPlan featureSubquery : featureSubqueries) { - featureSubquery.getValidityDateAggregator().ifPresent(agg::register); - } - - for (FormQueryPlan outcomeSubquery : outcomeSubqueries) { - outcomeSubquery.getValidityDateAggregator().ifPresent(agg::register); - } + featureSubquery.getValidityDateAggregator().ifPresent(agg::register); + outcomeSubquery.getValidityDateAggregator().ifPresent(agg::register); return agg.hasChildren() ? Optional.of(agg) : Optional.empty(); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/QueryJob.java b/backend/src/main/java/com/bakdata/conquery/models/query/QueryJob.java index c251a6446a..f721288516 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/QueryJob.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/QueryJob.java @@ -15,25 +15,26 @@ public class QueryJob implements Supplier> { private final QueryExecutionContext ctx; private final ThreadLocal> plan; private final Entity entity; - + @Override public Optional get() { - if(ctx.isQueryCancelled()){ + if (ctx.isQueryCancelled()) { return Optional.empty(); } try { QueryPlan queryPlan = plan.get(); + queryPlan.init(ctx, entity); return queryPlan.execute(ctx, entity); } catch (ConqueryError e) { // Catch errors, propagate them with their id. - throw new ConqueryError.ExecutionJobErrorWrapper(entity,e); + throw new ConqueryError.ExecutionJobErrorWrapper(entity, e); } catch (Exception e) { - throw new ConqueryError.ExecutionJobErrorWrapper(entity,new ConqueryError.UnknownError(e)); + throw new ConqueryError.ExecutionJobErrorWrapper(entity, new ConqueryError.UnknownError(e)); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ArrayConceptQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ArrayConceptQueryPlan.java index 4073ae3d36..adcbe1b875 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ArrayConceptQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ArrayConceptQueryPlan.java @@ -87,8 +87,6 @@ public Optional execute(QueryExecutionContext ctx, Entit // Only override if none has been set from a higher level ctx = QueryUtils.determineDateAggregatorForContext(ctx, this::getValidityDateAggregator); - init(ctx, entity); - if (!isOfInterest(entity)) { return Optional.empty(); } @@ -208,7 +206,6 @@ private int calculateCopyLength(SinglelineEntityResult singleLineResult) { return length; } - //TODO unused? public void nextTable(QueryExecutionContext ctx, Table currentTable) { childPlans.forEach(plan -> plan.nextTable(ctx, currentTable)); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ConceptQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ConceptQueryPlan.java index 3b9c5b44d3..9331338391 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ConceptQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ConceptQueryPlan.java @@ -69,8 +69,6 @@ public Optional execute(QueryExecutionContext ctx, Entit // Only override if none has been set from a higher level ctx = QueryUtils.determineDateAggregatorForContext(ctx, this::getValidityDateAggregator); - init(ctx, entity); - if (!isOfInterest(entity)) { return Optional.empty(); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java index ac8cfacd37..855f17f869 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java @@ -2,9 +2,11 @@ import java.util.ArrayList; import java.util.HashMap; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Queue; import java.util.Set; import com.bakdata.conquery.apiv1.query.ConceptQuery; @@ -21,6 +23,7 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.results.MultilineEntityResult; import com.bakdata.conquery.util.QueryUtils; +import lombok.AccessLevel; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.Setter; @@ -63,8 +66,6 @@ public class SecondaryIdQueryPlan implements QueryPlan { @Override public Optional execute(QueryExecutionContext ctx, Entity entity) { - init(ctx, entity); - if (!queryPlan.isOfInterest(entity)) { return Optional.empty(); } @@ -104,11 +105,21 @@ private Optional createResult(Entity entity) { return Optional.of(new MultilineEntityResult(entity.getId(), result)); } + /** + * This helps us avoid allocations, instead allowing us to reuse the queries. + */ + @Getter(AccessLevel.NONE) + private final Queue childPlanReusePool = new LinkedList<>(); + @Override public void init(QueryExecutionContext ctx, Entity entity) { queryPlan = query.createQueryPlan(queryPlanContext.withSelectedSecondaryId(secondaryId)); queryPlan.init(ctx, entity); + + // Dump the created children into reuse-pool + childPlanReusePool.addAll(childPerKey.values()); + childPerKey.clear(); } @@ -201,7 +212,12 @@ private boolean isOfInterest(Bucket bucket) { */ private ConceptQueryPlan createChild(QueryExecutionContext currentContext, Bucket currentBucket) { - final ConceptQueryPlan plan = query.createQueryPlan(queryPlanContext.withSelectedSecondaryId(secondaryId)); + ConceptQueryPlan plan; + + // Try to reuse old child plan first before allocating new ones + if((plan = childPlanReusePool.poll()) == null) { + plan = query.createQueryPlan(queryPlanContext.withSelectedSecondaryId(secondaryId)); + } final QueryExecutionContext context = QueryUtils.determineDateAggregatorForContext(currentContext, plan::getValidityDateAggregator); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/TableExportQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/TableExportQueryPlan.java index b8791beee1..83b3ba0521 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/TableExportQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/TableExportQueryPlan.java @@ -45,7 +45,7 @@ public Optional> getValidityDateAggregator() { @Override public void init(QueryExecutionContext ctxt, Entity entity) { - // Does nothing + subPlan.init(ctxt, entity); } @Override From 710b1959055f209104efe76d1a08cf554a463462 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Wed, 1 Sep 2021 17:07:18 +0200 Subject: [PATCH 12/16] fix too much deletion of init --- .../models/forms/managed/FormQueryPlan.java | 3 +- .../forms/managed/RelativeFormQueryPlan.java | 12 +- .../ABS_EXPORT/ABS_EXPORT_Query.test.json | 228 +++++++++--------- 3 files changed, 121 insertions(+), 122 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/FormQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/FormQueryPlan.java index fb533702b2..c7956459c4 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/FormQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/FormQueryPlan.java @@ -50,8 +50,6 @@ public FormQueryPlan(List dateContexts, ArrayConceptQueryPlan featu @Override public Optional execute(QueryExecutionContext ctx, Entity entity) { - init(ctx, entity); - if (!isOfInterest(entity)) { // If the entity is not covered by the query generate a basic result line with constants but without features return Optional.of(createResultForNotContained(entity, null)); @@ -67,6 +65,7 @@ public Optional execute(QueryExecutionContext ctx, Entity QueryExecutionContext innerContext = QueryUtils.determineDateAggregatorForContext(ctx, features::getValidityDateAggregator) .withDateRestriction(dateRestriction); + features.init(ctx, entity); Optional subResult = features.execute(innerContext, entity); diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQueryPlan.java index 85fc7af267..ec640ee8b0 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQueryPlan.java @@ -184,13 +184,13 @@ private int getFirstAggregatorPosition() { return 5; } + /** + * Whole result is the concatenation of the subresults. The final output format + * combines resolution info, index and eventdate of both sub queries. The + * feature/outcome sub queries are of in form of: [RESOLUTION], [INDEX], [EVENTDATE], [FEATURE/OUTCOME_DR], [FEATURE/OUTCOME_SELECTS]... + * The wanted format is: [RESOLUTION], [INDEX], [EVENTDATE], [FEATURE_DR], [OUTCOME_DR], [FEATURE_SELECTS]... , [OUTCOME_SELECTS] + */ private int calculateCompleteLength() { - /* - * Whole result is the concatenation of the subresults. The final output format - * combines resolution info, index and eventdate of both sub queries. The - * feature/outcome sub queries are of in form of: [RESOLUTION], [INDEX], [EVENTDATE], [FEATURE/OUTCOME_DR], [FEATURE/OUTCOME_SELECTS]... - * The wanted format is: [RESOLUTION], [INDEX], [EVENTDATE], [FEATURE_DR], [OUTCOME_DR], [FEATURE_SELECTS]... , [OUTCOME_SELECTS] - */ return getFirstAggregatorPosition() + featurePlan.getAggregatorSize() + outcomePlan.getAggregatorSize(); //return featureLength + outcomeLength - 3/* ^= [RESOLUTION], [INDEX], [EVENTDATE] */; diff --git a/backend/src/test/resources/tests/query/ABS_EXPORT/ABS_EXPORT_Query.test.json b/backend/src/test/resources/tests/query/ABS_EXPORT/ABS_EXPORT_Query.test.json index 5f54932284..9646f65b66 100644 --- a/backend/src/test/resources/tests/query/ABS_EXPORT/ABS_EXPORT_Query.test.json +++ b/backend/src/test/resources/tests/query/ABS_EXPORT/ABS_EXPORT_Query.test.json @@ -1,124 +1,124 @@ { - "type": "QUERY_TEST", - "label": "ABS_EXPORT Test", - "expectedCsv": "tests/query/ABS_EXPORT/expected.csv", + "type": "QUERY_TEST", + "label": "ABS_EXPORT Test", + "expectedCsv": "tests/query/ABS_EXPORT/expected.csv", + "query": { + "type": "ABSOLUTE_FORM_QUERY", + "dateRange": { + "min": "2000-01-01", + "max": "2002-12-31" + }, + "resolutionsAndAlignmentMap": [ + { + "resolution": "COMPLETE", + "alignment": "NO_ALIGN" + }, + { + "resolution": "YEARS", + "alignment": "QUARTER" + } + ], "query": { - "type": "ABSOLUTE_FORM_QUERY", - "dateRange": { - "min": "2000-01-01", - "max": "2002-12-31" - }, - "resolutionsAndAlignmentMap": [ - { - "resolution": "COMPLETE", - "alignment": "NO_ALIGN" - }, - { - "resolution": "YEARS", - "alignment": "QUARTER" - } + "type": "CONCEPT_QUERY", + "root": { + "type": "CONCEPT", + "ids": [ + "test_tree.test_child1" ], - "query": { - "type": "CONCEPT_QUERY", - "root": { - "type": "CONCEPT", - "ids": [ - "test_tree.test_child1" - ], - "tables": [ - { - "id": "test_tree.test_column", - "filters": [] - } - ] - } - }, - "features": { - "type": "ARRAY_CONCEPT_QUERY", - "childQueries": { - "type": "CONCEPT_QUERY", - "root": { - "type": "OR", - "children": { - "type": "CONCEPT", - "ids": [ - "test_tree.test_child1" - ], - "selects": [ - "test_tree.exists" - ], - "tables": [ - { - "id": "test_tree.test_column", - "filters": [] - } - ] - } - } - } - } + "tables": [ + { + "id": "test_tree.test_column", + "filters": [] + } + ] + } }, - "concepts": [ - { - "label": "test_tree", - "type": "TREE", - "connectors": { - "label": "tree_label", - "name": "test_column", - "column": "test_table.test_column", - "validityDates": { - "label": "datum", - "column": "test_table.datum" - } - }, + "features": { + "type": "ARRAY_CONCEPT_QUERY", + "childQueries": { + "type": "CONCEPT_QUERY", + "root": { + "type": "OR", + "children": { + "type": "CONCEPT", + "ids": [ + "test_tree.test_child1" + ], "selects": [ - { - "type": "EXISTS", - "name": "exists" - } + "test_tree.exists" ], - "children": [ - { - "label": "test_child1", - "description": " ", - "condition": { - "type": "PREFIX_LIST", - "prefixes": "A1" - }, - "children": [] - }, - { - "label": "test_child2", - "description": " ", - "condition": { - "type": "PREFIX_LIST", - "prefixes": "B2" - }, - "children": [] - } + "tables": [ + { + "id": "test_tree.test_column", + "filters": [] + } ] + } } - ], - "content": { - "tables": [ - { - "csv": "tests/query/ABS_EXPORT/content.csv", - "name": "test_table", - "primaryColumn": { - "name": "pid", - "type": "STRING" - }, - "columns": [ - { - "name": "datum", - "type": "DATE" - }, - { - "name": "test_column", - "type": "STRING" - } - ] - } - ] + } } + }, + "concepts": [ + { + "label": "test_tree", + "type": "TREE", + "connectors": { + "label": "tree_label", + "name": "test_column", + "column": "test_table.test_column", + "validityDates": { + "label": "datum", + "column": "test_table.datum" + } + }, + "selects": [ + { + "type": "EXISTS", + "name": "exists" + } + ], + "children": [ + { + "label": "test_child1", + "description": " ", + "condition": { + "type": "PREFIX_LIST", + "prefixes": "A1" + }, + "children": [] + }, + { + "label": "test_child2", + "description": " ", + "condition": { + "type": "PREFIX_LIST", + "prefixes": "B2" + }, + "children": [] + } + ] + } + ], + "content": { + "tables": [ + { + "csv": "tests/query/ABS_EXPORT/content.csv", + "name": "test_table", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "datum", + "type": "DATE" + }, + { + "name": "test_column", + "type": "STRING" + } + ] + } + ] + } } \ No newline at end of file From 04c968751d4350e72d35a3a955871c1c1233f731 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Fri, 3 Sep 2021 14:39:46 +0200 Subject: [PATCH 13/16] adds more documentation, rename getAggregationResult to createAggregationResult to make computation more obvious in API --- .../forms/managed/EntityDateQueryPlan.java | 2 +- .../forms/managed/RelativeFormQueryPlan.java | 2 +- .../models/forms/util/ResultModifier.java | 2 +- .../namespaces/specific/ExecuteQuery.java | 4 +- .../filter/CollectionNotEmptyFilterNode.java | 2 +- .../models/query/filter/RangeFilterNode.java | 2 +- .../queryplan/ArrayConceptQueryPlan.java | 2 +- .../query/queryplan/ConceptQueryPlan.java | 2 +- .../query/queryplan/DateAggregator.java | 6 +-- .../query/queryplan/EventIterating.java | 54 +++++++++++++------ .../models/query/queryplan/QPNode.java | 4 +- .../queryplan/aggregators/Aggregator.java | 29 ++++++---- .../aggregators/ColumnAggregator.java | 2 +- .../DistinctValuesWrapperAggregator.java | 4 +- .../MultiDistinctValuesWrapperAggregator.java | 4 +- .../specific/ConstantValueAggregator.java | 4 +- .../aggregators/specific/CountAggregator.java | 2 +- .../CountQuartersOfDateRangeAggregator.java | 2 +- .../CountQuartersOfDatesAggregator.java | 2 +- .../specific/DateDistanceAggregator.java | 2 +- .../specific/DateUnionAggregator.java | 2 +- .../specific/DurationSumAggregator.java | 2 +- .../specific/EventDateUnionAggregator.java | 6 +-- .../specific/EventDurationSumAggregator.java | 6 +-- .../specific/ExistsAggregator.java | 4 +- .../specific/MultiSelectAggregator.java | 2 +- .../specific/PrefixTextAggregator.java | 2 +- .../specific/QuarterAggregator.java | 4 +- .../specific/QuartersInYearAggregator.java | 2 +- .../specific/SelectAggregator.java | 2 +- .../specific/SpecialDateUnion.java | 4 +- .../diffsum/DecimalDiffSumAggregator.java | 2 +- .../diffsum/IntegerDiffSumAggregator.java | 2 +- .../diffsum/MoneyDiffSumAggregator.java | 2 +- .../diffsum/RealDiffSumAggregator.java | 2 +- .../specific/sum/DecimalSumAggregator.java | 2 +- .../specific/sum/IntegerSumAggregator.java | 2 +- .../specific/sum/MoneySumAggregator.java | 2 +- .../specific/sum/RealSumAggregator.java | 2 +- .../specific/value/AllValuesAggregator.java | 2 +- .../specific/value/FirstValueAggregator.java | 2 +- .../specific/value/LastValueAggregator.java | 2 +- .../specific/value/RandomValueAggregator.java | 2 +- .../query/queryplan/filter/FilterNode.java | 2 +- .../specific/temporal/TemporalQueryNode.java | 4 +- .../conquery/io/result/ResultTestUtil.java | 2 +- 46 files changed, 115 insertions(+), 86 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/EntityDateQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/EntityDateQueryPlan.java index 4769e87f7d..2389852673 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/EntityDateQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/EntityDateQueryPlan.java @@ -53,7 +53,7 @@ public Optional execute(QueryExecutionContext ctx, Entity return Optional.empty(); } - final CDateSet aggregationResult = validityDateAggregator.get().getAggregationResult(); + final CDateSet aggregationResult = validityDateAggregator.get().createAggregationResult(); aggregationResult.retainAll(dateRestriction); // Generate DateContexts in the provided resolutions diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQueryPlan.java index ec640ee8b0..337a8b45f7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQueryPlan.java @@ -79,7 +79,7 @@ public Optional execute(QueryExecutionContext ctx, Entity int size = calculateCompleteLength(); EntityResult contained = preResult.get(); // Gather all validity dates from prerequisite - CDateSet dateSet = query.getValidityDateAggregator().map(Aggregator::getAggregationResult).orElseGet(CDateSet::create); + CDateSet dateSet = query.getValidityDateAggregator().map(Aggregator::createAggregationResult).orElseGet(CDateSet::create); final OptionalInt sampled = indexSelector.sample(dateSet); diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/util/ResultModifier.java b/backend/src/main/java/com/bakdata/conquery/models/forms/util/ResultModifier.java index b0a68a799f..acd47df56d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/util/ResultModifier.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/util/ResultModifier.java @@ -42,7 +42,7 @@ private static Object[] setExistAggValues(List> aggregators, Objec Aggregator agg = aggregators.get(i); // Fill EXIST aggregators with false which evaluated to 'null' if (agg instanceof ExistsAggregator && Objects.isNull(result[i + aggIdx])) { - result[i + aggIdx] = agg.getAggregationResult(); + result[i + aggIdx] = agg.createAggregationResult(); } } return result; diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ExecuteQuery.java b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ExecuteQuery.java index cfb634b498..de67ed70e9 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ExecuteQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ExecuteQuery.java @@ -54,10 +54,8 @@ public void react(Worker worker) throws Exception { final ShardResult result = createShardResult(worker); - // Generate query plans for this execution. For ManagedQueries this is only one plan. - // The results are send directly to these ManagesQueries + // Before we start the query, we create it once to test if it will succeed before creating it multiple times for evaluation per core. try { - // Assert that we can create the queryPlan query.createQueryPlan(new QueryPlanContext(worker)); } catch (Exception e) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/filter/CollectionNotEmptyFilterNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/filter/CollectionNotEmptyFilterNode.java index 53c62461b4..c48cda21d2 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/filter/CollectionNotEmptyFilterNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/filter/CollectionNotEmptyFilterNode.java @@ -16,6 +16,6 @@ public CollectionNotEmptyFilterNode(Aggregator> aggregator) { @Override public boolean isContained() { - return !getAggregator().getAggregationResult().isEmpty(); + return !getAggregator().createAggregationResult().isEmpty(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/filter/RangeFilterNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/filter/RangeFilterNode.java index 1c54cd1a62..7fed724866 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/filter/RangeFilterNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/filter/RangeFilterNode.java @@ -15,6 +15,6 @@ public RangeFilterNode(IRange filterValue, Aggregator aggregator) @Override public boolean isContained() { - return filterValue.contains(getAggregator().getAggregationResult()); + return filterValue.contains(getAggregator().createAggregationResult()); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ArrayConceptQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ArrayConceptQueryPlan.java index adcbe1b875..33890d6877 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ArrayConceptQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ArrayConceptQueryPlan.java @@ -134,7 +134,7 @@ public Optional execute(QueryExecutionContext ctx, Entit if (generateDateAggregation) { // Dateset was needed, add it to the front. - resultValues[VALIDITY_DATE_POSITION] = validityDateAggregator.getAggregationResult(); + resultValues[VALIDITY_DATE_POSITION] = validityDateAggregator.createAggregationResult(); } return Optional.of(new SinglelineEntityResult(entity.getId(), resultValues)); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ConceptQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ConceptQueryPlan.java index 9331338391..aaae5d0e51 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ConceptQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/ConceptQueryPlan.java @@ -57,7 +57,7 @@ protected SinglelineEntityResult createResult() { Object[] values = new Object[aggregators.size()]; for (int i = 0; i < values.length; i++) { - values[i] = aggregators.get(i).getAggregationResult(); + values[i] = aggregators.get(i).createAggregationResult(); } return new SinglelineEntityResult(entity.getId(), values); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregator.java index 8c888beda8..39abb0ab00 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/DateAggregator.java @@ -18,7 +18,7 @@ * flexibility through different {@link DateAggregationAction}s. */ @RequiredArgsConstructor -public class DateAggregator implements Aggregator { +public class DateAggregator extends Aggregator { private final DateAggregationAction action; @@ -51,10 +51,10 @@ public void acceptEvent(Bucket bucket, int event) { } @Override - public CDateSet getAggregationResult() { + public CDateSet createAggregationResult() { final Set all = new HashSet<>(); children.forEach(s -> { - CDateSet result = s.getAggregationResult(); + CDateSet result = s.createAggregationResult(); if(result != null) { all.add(result); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/EventIterating.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/EventIterating.java index ca680ea42c..8e6687c6ef 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/EventIterating.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/EventIterating.java @@ -8,26 +8,48 @@ import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.entity.Entity; -public interface EventIterating { - - default void collectRequiredTables(Set
requiredTables) {} - - default Set
collectRequiredTables() { - HashSet
out = new HashSet<>(); - this.collectRequiredTables(out); +/** + * Implementing classes will be called in QueryEngine for evaluation. + *

+ * Order of execution should be + * 1) {@link EventIterating#init(Entity, QueryExecutionContext)} (once per Entity): resetting the {@link QueryPlan} for evaluation. + * 2) {@link EventIterating#isOfInterest(Entity)} (once per Entity): Checking if the Entity has interesting data for the {@link EventIterating} + * 3) {@link EventIterating#nextTable(QueryExecutionContext, Table)} (once per Table): Initializing the Event Iterating for the Table + * 4) {@link EventIterating#isOfInterest(Bucket)} (once per Bucket): Check if Bucket contains relevant information for this {@link EventIterating} + * 5) {@link EventIterating#nextBlock(Bucket)} (once per Bucket): Initialize the {@link EventIterating} for evaluation of the {@link Bucket}, for example prefetching {@link com.bakdata.conquery.models.events.CBlock}s + * 6) {@link EventIterating#acceptEvent(Bucket, int)} (per Event): Evaluation of the {@link QueryPlan} for this {@link EventIterating} + *

+ */ +public abstract class EventIterating { + + public void collectRequiredTables(Set

requiredTables) { + } + + public Set
collectRequiredTables() { + Set
out = new HashSet<>(); + collectRequiredTables(out); return out; } - public void init(Entity entity, QueryExecutionContext context); - - default void nextTable(QueryExecutionContext ctx, Table currentTable) {} - - default void nextBlock(Bucket bucket) {} + /** + * Completely reset this object for reuse. + */ + public abstract void init(Entity entity, QueryExecutionContext context); + + public void nextTable(QueryExecutionContext ctx, Table currentTable) { + } + + public void nextBlock(Bucket bucket) { + } - void acceptEvent(Bucket bucket, int event); + public abstract void acceptEvent(Bucket bucket, int event); - default boolean isOfInterest(Bucket bucket){ return true; } - - default boolean isOfInterest(Entity entity){ return true; } + public boolean isOfInterest(Bucket bucket) { + return true; + } + + public boolean isOfInterest(Entity entity) { + return true; + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QPNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QPNode.java index d092a2b70d..8f5519dd2e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QPNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/QPNode.java @@ -15,7 +15,7 @@ import lombok.Setter; @Getter(AccessLevel.PROTECTED) @Setter(AccessLevel.PROTECTED) -public abstract class QPNode implements EventIterating { +public abstract class QPNode extends EventIterating { protected QueryExecutionContext context; protected Entity entity; @@ -44,7 +44,7 @@ public List getChildren() { @Override public String toString() { - return this.getClass().getSimpleName(); + return getClass().getSimpleName(); } /** diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/Aggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/Aggregator.java index 2723046719..c1c45ee5aa 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/Aggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/Aggregator.java @@ -1,27 +1,36 @@ package com.bakdata.conquery.models.query.queryplan.aggregators; +import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.query.queryplan.EventIterating; import com.fasterxml.jackson.annotation.JsonIgnore; /** - * An aggregator iterates over events, computing a value alongside. Values are fed through {@code aggregateEvent}, and the result can be queried at {@code getAggregationResult}. - * - * Every Aggregator has an associtaed {@code ResultType} that is used for rendering purposes. - * - * See Also {@code EventIterating} and {@code CtxCloneable}. - * + * An aggregator iterates over events, computing a value alongside. Values are fed through {@link Aggregator#acceptEvent(Bucket, int)}, and the result can be queried at {@link Aggregator#createAggregationResult()}. + *

+ * Every Aggregator has an associated {@code ResultType} that is used for rendering purposes. + *

+ * See Also {@code EventIterating}. + *

* Aggregators are used to produce additional values for query results, but are also used in filters, to restrict the result set. - * + *

* An Aggregator is usually created by an associated {@code Select}, which is their API Layer counterpart. * * @param Java result type after aggregation. */ -public interface Aggregator extends EventIterating { +public abstract class Aggregator extends EventIterating { - T getAggregationResult(); + /** + * Compute aggregation result. + * + * @implSpec Returned objects may not be reused in later Query evaluation, therefore createAggregationResult must copy collections when they are returned. + */ + public abstract T createAggregationResult(); + /** + * Specific type of the result used for rendering. + */ @JsonIgnore - ResultType getResultType(); + public abstract ResultType getResultType(); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/ColumnAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/ColumnAggregator.java index 15ae09c6fc..e7eaf62734 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/ColumnAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/ColumnAggregator.java @@ -9,7 +9,7 @@ /** * Base class for aggregators acting on columns. */ -public abstract class ColumnAggregator implements Aggregator { +public abstract class ColumnAggregator extends Aggregator { @Override public void collectRequiredTables(Set

out) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/DistinctValuesWrapperAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/DistinctValuesWrapperAggregator.java index 129cdc6ab8..835293d7e4 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/DistinctValuesWrapperAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/DistinctValuesWrapperAggregator.java @@ -29,8 +29,8 @@ public DistinctValuesWrapperAggregator(ColumnAggregator aggregator, Colum } @Override - public VALUE getAggregationResult() { - return aggregator.getAggregationResult(); + public VALUE createAggregationResult() { + return aggregator.createAggregationResult(); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/MultiDistinctValuesWrapperAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/MultiDistinctValuesWrapperAggregator.java index e00a013594..e107bdbf92 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/MultiDistinctValuesWrapperAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/MultiDistinctValuesWrapperAggregator.java @@ -30,8 +30,8 @@ public MultiDistinctValuesWrapperAggregator(ColumnAggregator aggregator, } @Override - public VALUE getAggregationResult() { - return aggregator.getAggregationResult(); + public VALUE createAggregationResult() { + return aggregator.createAggregationResult(); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/ConstantValueAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/ConstantValueAggregator.java index 8de392a88c..2df1b61521 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/ConstantValueAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/ConstantValueAggregator.java @@ -15,13 +15,13 @@ @Getter @RequiredArgsConstructor @ToString -public class ConstantValueAggregator implements Aggregator { +public class ConstantValueAggregator extends Aggregator { private final Object value; private final ResultType type; @Override - public Object getAggregationResult() { + public Object createAggregationResult() { return value; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountAggregator.java index 04b0fda1d4..d088dc7de2 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountAggregator.java @@ -31,7 +31,7 @@ public void acceptEvent(Bucket bucket, int event) { } @Override - public Long getAggregationResult() { + public Long createAggregationResult() { return count > 0 ? count : null; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountQuartersOfDateRangeAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountQuartersOfDateRangeAggregator.java index bdfa52b4df..19fef97380 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountQuartersOfDateRangeAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountQuartersOfDateRangeAggregator.java @@ -81,7 +81,7 @@ public void addDateRange(CDateRange dateRange) { } @Override - public Long getAggregationResult() { + public Long createAggregationResult() { return quarters.isEmpty() ? null : (long) quarters.size(); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountQuartersOfDatesAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountQuartersOfDatesAggregator.java index eb46f95e18..a647861538 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountQuartersOfDatesAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/CountQuartersOfDatesAggregator.java @@ -40,7 +40,7 @@ public void acceptEvent(Bucket bucket, int event) { } @Override - public Long getAggregationResult() { + public Long createAggregationResult() { return quarters.isEmpty() ? null : (long) quarters.size(); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DateDistanceAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DateDistanceAggregator.java index bce9aeed6a..1d47b8d48e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DateDistanceAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DateDistanceAggregator.java @@ -45,7 +45,7 @@ public void nextTable(QueryExecutionContext ctx, Table currentTable) { } @Override - public Long getAggregationResult() { + public Long createAggregationResult() { return result != Long.MAX_VALUE || hit ? result : null; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DateUnionAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DateUnionAggregator.java index a42c7d4dc5..8f7aa9ae04 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DateUnionAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DateUnionAggregator.java @@ -48,7 +48,7 @@ public void acceptEvent(Bucket bucket, int event) { } @Override - public CDateSet getAggregationResult() { + public CDateSet createAggregationResult() { return CDateSet.create(set.asRanges()); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DurationSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DurationSumAggregator.java index 285bb01b29..96804787df 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DurationSumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/DurationSumAggregator.java @@ -49,7 +49,7 @@ public void acceptEvent(Bucket bucket, int event) { } @Override - public Long getAggregationResult() { + public Long createAggregationResult() { return set.isEmpty() ? null : set.countDays(); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDateUnionAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDateUnionAggregator.java index 85c3522b8f..e8d570431a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDateUnionAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDateUnionAggregator.java @@ -19,7 +19,7 @@ * */ @RequiredArgsConstructor -public class EventDateUnionAggregator implements Aggregator{ +public class EventDateUnionAggregator extends Aggregator { private final Set
requiredTables; private Column validityDateColumn; @@ -44,11 +44,11 @@ public void nextTable(QueryExecutionContext ctx, Table currentTable) { } dateRestriction = ctx.getDateRestriction(); - Aggregator.super.nextTable(ctx, currentTable); + super.nextTable(ctx, currentTable); } @Override - public CDateSet getAggregationResult() { + public CDateSet createAggregationResult() { return CDateSet.create(set.asRanges()); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDurationSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDurationSumAggregator.java index e3491e7406..51be41e127 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDurationSumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDurationSumAggregator.java @@ -17,7 +17,7 @@ /** * Aggregator, counting the number of days present. */ -public class EventDurationSumAggregator implements Aggregator { +public class EventDurationSumAggregator extends Aggregator { private Optional> queryDateAggregator = Optional.empty(); private final CDateSet set = CDateSet.create(); @@ -60,10 +60,10 @@ public void acceptEvent(Bucket bucket, int event) { } @Override - public Long getAggregationResult() { + public Long createAggregationResult() { queryDateAggregator - .map(Aggregator::getAggregationResult) + .map(Aggregator::createAggregationResult) .ifPresent( set::retainAll ); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/ExistsAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/ExistsAggregator.java index 2014e36c1f..ef66d3b1e7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/ExistsAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/ExistsAggregator.java @@ -17,7 +17,7 @@ */ @RequiredArgsConstructor @ToString(of = "requiredTables") -public class ExistsAggregator implements Aggregator { +public class ExistsAggregator extends Aggregator { private final Set
requiredTables; @@ -32,7 +32,7 @@ public void setReference(QPNode ref) { public void acceptEvent(Bucket bucket, int event) { } @Override - public Boolean getAggregationResult() { + public Boolean createAggregationResult() { return reference.isContained(); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/MultiSelectAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/MultiSelectAggregator.java index 70061c2ebb..63f180c0f6 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/MultiSelectAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/MultiSelectAggregator.java @@ -59,7 +59,7 @@ public void acceptEvent(Bucket bucket, int event) { } @Override - public Map getAggregationResult() { + public Map createAggregationResult() { Map out = new HashMap<>(); for (int i = 0; i < hits.length; i++) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/PrefixTextAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/PrefixTextAggregator.java index d63482871b..301be86233 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/PrefixTextAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/PrefixTextAggregator.java @@ -46,7 +46,7 @@ public void acceptEvent(Bucket bucket, int event) { } @Override - public Set getAggregationResult() { + public Set createAggregationResult() { return entries.isEmpty() ? null : ImmutableSet.copyOf(entries); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuarterAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuarterAggregator.java index bf88299551..a98b12590c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuarterAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuarterAggregator.java @@ -21,7 +21,7 @@ * Samples the incoming dates outputting the year-quarter of the sample. */ @Data -public class QuarterAggregator implements Aggregator { +public class QuarterAggregator extends Aggregator { private final TemporalSampler sampler; @@ -61,7 +61,7 @@ public void acceptEvent(Bucket bucket, int event) { } @Override - public String getAggregationResult() { + public String createAggregationResult() { if (set.isEmpty()) { return null; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuartersInYearAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuartersInYearAggregator.java index 883b37dfb5..97e28fee62 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuartersInYearAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuartersInYearAggregator.java @@ -53,7 +53,7 @@ public void acceptEvent(Bucket bucket, int event) { } @Override - public Long getAggregationResult() { + public Long createAggregationResult() { if(quartersInYear.isEmpty()) { return null; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SelectAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SelectAggregator.java index 898202e48a..c24515e809 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SelectAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SelectAggregator.java @@ -51,7 +51,7 @@ public void acceptEvent(Bucket bucket, int event) { } @Override - public Long getAggregationResult() { + public Long createAggregationResult() { return hits > 0 ? hits : null; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SpecialDateUnion.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SpecialDateUnion.java index ef842a8087..c0deda0d9e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SpecialDateUnion.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SpecialDateUnion.java @@ -14,7 +14,7 @@ * Special Aggregator, used to calculate the times an entity has events after filtering. */ @RequiredArgsConstructor -public class SpecialDateUnion implements Aggregator { +public class SpecialDateUnion extends Aggregator { private CDateSet set = CDateSet.create(); @@ -54,7 +54,7 @@ public void merge(CDateSet other){ } @Override - public CDateSet getAggregationResult() { + public CDateSet createAggregationResult() { return CDateSet.create(set.asRanges()); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/DecimalDiffSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/DecimalDiffSumAggregator.java index 58a875357f..e12b3bb029 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/DecimalDiffSumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/DecimalDiffSumAggregator.java @@ -56,7 +56,7 @@ public void acceptEvent(Bucket bucket, int event) { } @Override - public BigDecimal getAggregationResult() { + public BigDecimal createAggregationResult() { return hit ? sum : null; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/IntegerDiffSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/IntegerDiffSumAggregator.java index 5826b78f4d..b63321ada1 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/IntegerDiffSumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/IntegerDiffSumAggregator.java @@ -54,7 +54,7 @@ public void acceptEvent(Bucket bucket, int event) { } @Override - public Long getAggregationResult() { + public Long createAggregationResult() { return hit ? sum : null; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/MoneyDiffSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/MoneyDiffSumAggregator.java index 7bc80856c2..cf10e4ada4 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/MoneyDiffSumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/MoneyDiffSumAggregator.java @@ -54,7 +54,7 @@ public void acceptEvent(Bucket bucket, int event) { } @Override - public Long getAggregationResult() { + public Long createAggregationResult() { return hit ? sum : null; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/RealDiffSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/RealDiffSumAggregator.java index 3def598b59..fc25608e19 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/RealDiffSumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/diffsum/RealDiffSumAggregator.java @@ -59,7 +59,7 @@ public void acceptEvent(Bucket bucket, int event) { } @Override - public Double getAggregationResult() { + public Double createAggregationResult() { return hit ? sum : null; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/DecimalSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/DecimalSumAggregator.java index e83a7e446d..dc2f1f6910 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/DecimalSumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/DecimalSumAggregator.java @@ -41,7 +41,7 @@ public void acceptEvent(Bucket bucket, int event) { } @Override - public BigDecimal getAggregationResult() { + public BigDecimal createAggregationResult() { return hit ? sum : null; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/IntegerSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/IntegerSumAggregator.java index 7fa39d3973..50d31ac2f5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/IntegerSumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/IntegerSumAggregator.java @@ -39,7 +39,7 @@ public void acceptEvent(Bucket bucket, int event) { } @Override - public Long getAggregationResult() { + public Long createAggregationResult() { return hit ? sum : null; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/MoneySumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/MoneySumAggregator.java index 2180dad86c..86f1b3e8df 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/MoneySumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/MoneySumAggregator.java @@ -40,7 +40,7 @@ public void acceptEvent(Bucket bucket, int event) { } @Override - public Long getAggregationResult() { + public Long createAggregationResult() { return hit ? sum : null; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/RealSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/RealSumAggregator.java index 0c8cd1a9cb..ad4ce8c122 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/RealSumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/sum/RealSumAggregator.java @@ -40,7 +40,7 @@ public void acceptEvent(Bucket bucket, int event) { } @Override - public Double getAggregationResult() { + public Double createAggregationResult() { return hit ? sum : null; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/AllValuesAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/AllValuesAggregator.java index 1b1f2921e7..b4cef8309b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/AllValuesAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/AllValuesAggregator.java @@ -37,7 +37,7 @@ public void acceptEvent(Bucket bucket, int event) { } @Override - public Set getAggregationResult() { + public Set createAggregationResult() { return entries.isEmpty() ? null : ImmutableSet.copyOf(entries); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/FirstValueAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/FirstValueAggregator.java index 1d3fae3870..586cb8286e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/FirstValueAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/FirstValueAggregator.java @@ -76,7 +76,7 @@ else if (next == date) { } @Override - public VALUE getAggregationResult() { + public VALUE createAggregationResult() { if (selectedBucket == null && selectedEvent.isEmpty()) { return null; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/LastValueAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/LastValueAggregator.java index fb2ab7672c..ebe54a85bc 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/LastValueAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/LastValueAggregator.java @@ -77,7 +77,7 @@ else if (next == date) { } @Override - public VALUE getAggregationResult() { + public VALUE createAggregationResult() { if (selectedBucket == null && selectedEvent.isEmpty()) { return null; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/RandomValueAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/RandomValueAggregator.java index 26b6f39be0..42d7c3ac2a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/RandomValueAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/RandomValueAggregator.java @@ -59,7 +59,7 @@ public void acceptEvent(Bucket bucket, int event) { } @Override - public VALUE getAggregationResult() { + public VALUE createAggregationResult() { if (bucket == null) { return null; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/filter/FilterNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/filter/FilterNode.java index bb19a4b516..26fa228cdb 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/filter/FilterNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/filter/FilterNode.java @@ -6,7 +6,7 @@ import lombok.Setter; @AllArgsConstructor -public abstract class FilterNode implements EventIterating { +public abstract class FilterNode extends EventIterating { @Setter @Getter protected FILTER_VALUE filterValue; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/temporal/TemporalQueryNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/temporal/TemporalQueryNode.java index 190ca4a9b7..f53f1ee891 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/temporal/TemporalQueryNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/temporal/TemporalQueryNode.java @@ -113,9 +113,9 @@ public final boolean isContained() { return false; } - CDateSet referenceDurations = CDateSet.create(getReference().getChild().getDateAggregator().getAggregationResult()); + CDateSet referenceDurations = CDateSet.create(getReference().getChild().getDateAggregator().createAggregationResult()); // Create copy as we are mutating the set - CDateSet precedingDurations = CDateSet.create(getPreceding().getChild().getDateAggregator().getAggregationResult()); + CDateSet precedingDurations = CDateSet.create(getPreceding().getChild().getDateAggregator().createAggregationResult()); OptionalInt sampledReference = getReference().getSampler().sample(referenceDurations); diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java b/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java index 4284aafe84..86bfb588b7 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java @@ -75,7 +75,7 @@ public void acceptEvent(Bucket bucket, int event) { } @Override - public String getAggregationResult() { + public String createAggregationResult() { throw new UnsupportedOperationException(); } From 1ec4d908abd463ee9c783f46f6e4fb74e333194f Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Fri, 3 Sep 2021 14:45:32 +0200 Subject: [PATCH 14/16] also check if we can run forms --- .../namespaces/specific/ExecuteForm.java | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ExecuteForm.java b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ExecuteForm.java index 791fcd749c..79bfcdf5c2 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ExecuteForm.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ExecuteForm.java @@ -1,15 +1,19 @@ package com.bakdata.conquery.models.messages.namespaces.specific; +import static com.bakdata.conquery.models.error.ConqueryError.asConqueryError; + import java.util.Map; import java.util.Map.Entry; import com.bakdata.conquery.apiv1.query.Query; import com.bakdata.conquery.io.cps.CPSType; +import com.bakdata.conquery.models.error.ConqueryError; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.messages.namespaces.NamespacedMessage; import com.bakdata.conquery.models.messages.namespaces.WorkerMessage; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.QueryExecutor; +import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.results.FormShardResult; import com.bakdata.conquery.models.query.results.ShardResult; import com.bakdata.conquery.models.worker.Worker; @@ -52,13 +56,26 @@ public void react(Worker worker) throws Exception { queryExecutor.unsetQueryCancelled(formId); + // Execute all plans. for (Entry entry : queries.entrySet()) { + final Query query = entry.getValue(); ShardResult result = createResult(worker, entry.getKey()); + // Before we start the query, we create it once to test if it will succeed before creating it multiple times for evaluation per core. + try { + query.createQueryPlan(new QueryPlanContext(worker)); + } + catch (Exception e) { + ConqueryError err = asConqueryError(e); + log.warn("Failed to create query plans for {}.", formId, err); + queryExecutor.sendFailureToManagerNode(result, err); + return; + } + final QueryExecutionContext subQueryContext = new QueryExecutionContext(formId, queryExecutor, worker.getStorage(), worker.getBucketManager()); - if (!queryExecutor.execute(entry.getValue(), subQueryContext, result)) { + if (!queryExecutor.execute(query, subQueryContext, result)) { return; } } From fae01dc83d31cf8b20427dcba4d49e4b94cfdae9 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Mon, 20 Sep 2021 13:16:11 +0200 Subject: [PATCH 15/16] fix merge problems --- .../models/forms/managed/RelativeFormQueryPlan.java | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQueryPlan.java index 14c49918ca..3a99e8fbbd 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQueryPlan.java @@ -217,18 +217,14 @@ private boolean hasCompleteDateContexts(List contexts) { if (featurePlan.getAggregatorSize() <= 0 || outcomePlan.getAggregatorSize() <= 0) { // Otherwise, if only features or outcomes are given check the first date context. The empty feature/outcome query // will still return an empty result which will be merged with to a complete result. - return DateContext.Resolution.COMPLETE.equals(contexts.get(0).getSubdivisionMode()); + return Resolution.COMPLETE.equals(contexts.get(0).getSubdivisionMode()); } // We have features and outcomes check if both have complete date ranges (they should be at the beginning of the list) return contexts.size() >= 2 - && contexts.get(0).getSubdivisionMode().equals(Resolution.COMPLETE) - && contexts.get(1).getSubdivisionMode().equals(Resolution.COMPLETE) + && Resolution.COMPLETE.equals(contexts.get(0).getSubdivisionMode()) + && Resolution.COMPLETE.equals(contexts.get(1).getSubdivisionMode()) && !contexts.get(0).getFeatureGroup().equals(contexts.get(1).getFeatureGroup()); - } - // Otherwise, if only features or outcomes are given check the first date context. The empty feature/outcome query - // will still return an empty result which will be merged with to a complete result. - return contexts.get(0).getSubdivisionMode().equals(Resolution.COMPLETE); } private static FormQueryPlan createSubQuery(ArrayConceptQueryPlan subPlan, List contexts, FeatureGroup featureGroup) { From badd453e657ba12f4d955fdb2572725541fafef8 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Mon, 20 Sep 2021 14:22:08 +0200 Subject: [PATCH 16/16] create query in createQueryPlan --- .../com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java | 2 +- .../models/query/queryplan/SecondaryIdQueryPlan.java | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java index 72cae95541..03184e4ba0 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java @@ -67,7 +67,7 @@ public class SecondaryIdQuery extends Query { @Override public SecondaryIdQueryPlan createQueryPlan(QueryPlanContext context) { - return new SecondaryIdQueryPlan(query, context, secondaryId, withSecondaryId, withoutSecondaryId); + return new SecondaryIdQueryPlan(query, context, secondaryId, withSecondaryId, withoutSecondaryId, query.createQueryPlan(context.withSelectedSecondaryId(secondaryId))); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java index 855f17f869..37f6a71e7b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java @@ -51,9 +51,11 @@ public class SecondaryIdQueryPlan implements QueryPlan { private final Set tablesWithSecondaryId; private final Set
tablesWithoutSecondaryId; + private final ConceptQueryPlan queryPlan; + + private final Map childPerKey = new HashMap<>(); - private ConceptQueryPlan queryPlan; /** @@ -113,7 +115,6 @@ private Optional createResult(Entity entity) { @Override public void init(QueryExecutionContext ctx, Entity entity) { - queryPlan = query.createQueryPlan(queryPlanContext.withSelectedSecondaryId(secondaryId)); queryPlan.init(ctx, entity);