From 189beb578d648f4446bef26181210d75a2cd84b1 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Thu, 28 Oct 2021 21:06:37 +0200 Subject: [PATCH 1/3] adds more documentation --- .../apiv1/query/SecondaryIdQuery.java | 9 +- .../concept/specific/external/CQExternal.java | 2 +- .../query/queryplan/SecondaryIdQueryPlan.java | 8 +- .../specific/ConstantValueAggregator.java | 6 +- .../queryplan/specific/ExternalNode.java | 34 +++--- .../SECONDARY_IDS.test.json | 114 ++++++++++++++++++ .../SECONDARY_ID_EXTERNAL_LOGICAL/content.csv | 7 ++ .../expected.csv | 3 + 8 files changed, 155 insertions(+), 28 deletions(-) create mode 100644 backend/src/test/resources/tests/query/SECONDARY_ID_EXTERNAL_LOGICAL/SECONDARY_IDS.test.json create mode 100644 backend/src/test/resources/tests/query/SECONDARY_ID_EXTERNAL_LOGICAL/content.csv create mode 100644 backend/src/test/resources/tests/query/SECONDARY_ID_EXTERNAL_LOGICAL/expected.csv diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java index 03184e4ba0..969c0943fe 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java @@ -11,6 +11,7 @@ import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; +import com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.jackson.InternalOnly; import com.bakdata.conquery.io.jackson.serializer.NsIdRef; @@ -77,17 +78,17 @@ public void collectRequiredQueries(Set> requiredQueries) { } @Override - public void resolve(QueryResolveContext context) { + public void resolve(final QueryResolveContext context) { DateAggregationMode resolvedDateAggregationMode = dateAggregationMode; if (context.getDateAggregationMode() != null) { log.trace("Overriding date aggregation mode ({}) with mode from context ({})", dateAggregationMode, context.getDateAggregationMode()); resolvedDateAggregationMode = context.getDateAggregationMode(); } - context = context.withDateAggregationMode(resolvedDateAggregationMode); + final QueryResolveContext resolvedContext = context.withDateAggregationMode(resolvedDateAggregationMode); this.query = new ConceptQuery(root); - query.resolve(context); + query.resolve(resolvedContext); withSecondaryId = new HashSet<>(); withoutSecondaryId = new HashSet<>(); @@ -98,6 +99,8 @@ public void resolve(QueryResolveContext context) { // partition tables by their holding of the requested SecondaryId. // This assumes that from the root, only ConceptNodes hold TableIds we are interested in. query.visit(queryElement -> { + // We cannot check for CQExternal here and add the ALL_IDS Table because it is not serializable at the moment + if (!(queryElement instanceof CQConcept)) { return; } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java index d6143938f6..98f53465a2 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java @@ -94,7 +94,7 @@ public QPNode createQueryPlan(QueryPlanContext context, ConceptQueryPlan plan) { } // Allocate at once, the maximum possible size - final Map extraAggregators = new HashMap<>(format.size()); + final Map>> extraAggregators = new HashMap<>(format.size()); if (extra != null) { for (int col = 0; col < format.size(); col++) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java index 37f6a71e7b..61b08e88a6 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/SecondaryIdQueryPlan.java @@ -72,15 +72,18 @@ public Optional execute(QueryExecutionContext ctx, Entity return Optional.empty(); } - //first execute only tables with secondaryIds, creating all sub-queries + // First execute only tables with secondaryIds, creating all sub-queries for (Column entry : tablesWithSecondaryId) { executeQueriesWithSecondaryId(ctx, entity, entry); } - //afterwards the remaining tables, since we now spawned all children + // Afterwards the remaining tables, since we now spawned all children for (Table currentTable : tablesWithoutSecondaryId) { executeQueriesWithoutSecondaryId(ctx, entity, currentTable); } + // Do a last run with the ALL_IDS Table to trigger ExternalNodes + executeQueriesWithoutSecondaryId(ctx, entity, ctx.getStorage().getDataset().getAllIdsTable()); + return createResult(entity); } @@ -117,7 +120,6 @@ private Optional createResult(Entity entity) { public void init(QueryExecutionContext ctx, Entity entity) { queryPlan.init(ctx, entity); - // Dump the created children into reuse-pool childPlanReusePool.addAll(childPerKey.values()); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/ConstantValueAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/ConstantValueAggregator.java index a98bdde52d..9f26eb33a0 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/ConstantValueAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/ConstantValueAggregator.java @@ -16,14 +16,14 @@ @Getter @AllArgsConstructor @ToString -public class ConstantValueAggregator extends Aggregator { +public class ConstantValueAggregator extends Aggregator { @Setter - private Object value; + private T value; private final ResultType type; @Override - public Object createAggregationResult() { + public T createAggregationResult() { return value; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ExternalNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ExternalNode.java index e1cf1bbbea..2c00d3f512 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ExternalNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ExternalNode.java @@ -8,9 +8,11 @@ import javax.validation.constraints.NotEmpty; +import com.bakdata.conquery.io.jackson.serializer.CDateSetDeserializer; import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.datasets.Table; import com.bakdata.conquery.models.events.Bucket; +import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.models.query.queryplan.QPNode; @@ -24,18 +26,18 @@ public class ExternalNode extends QPNode { private final Table table; - private SpecialDateUnion dateUnion = new SpecialDateUnion(); + private CDateSet dateUnion = CDateSet.create(); @NotEmpty @NonNull private final Map includedEntities; private final Map>> extraData; - private final Map extraAggregators; + private final Map>> extraAggregators; private CDateSet contained; - public ExternalNode(Table table, Map includedEntities, Map>> extraData, Map extraAggregators) { + public ExternalNode(Table table, Map includedEntities, Map>> extraData, Map>> extraAggregators) { this.includedEntities = includedEntities; this.table = table; this.extraData = extraData; @@ -46,15 +48,15 @@ public ExternalNode(Table table, Map includedEntities, Map aggregator : extraAggregators.values()) { aggregator.setValue(null); } - for (Map.Entry colAndAgg : extraAggregators.entrySet()) { + for (Map.Entry>> colAndAgg : extraAggregators.entrySet()) { final String col = colAndAgg.getKey(); - final ConstantValueAggregator agg = colAndAgg.getValue(); + final ConstantValueAggregator> agg = colAndAgg.getValue(); // Clear if entity has no value for the column if (!extraData.getOrDefault(entity.getId(), Collections.emptyMap()).containsKey(col)) { @@ -67,21 +69,17 @@ public void init(Entity entity, QueryExecutionContext context) { @Override public void nextTable(QueryExecutionContext ctx, Table currentTable) { - if (contained != null) { - CDateSet newSet = CDateSet.create(ctx.getDateRestriction()); - newSet.retainAll(contained); - ctx = ctx.withDateRestriction(newSet); - } - super.nextTable(ctx, currentTable); - dateUnion.nextTable(getContext(), currentTable); + + if (table.equals(currentTable) && contained != null){ + dateUnion.addAll(contained); + dateUnion.retainAll(ctx.getDateRestriction()); + } } @Override public void acceptEvent(Bucket bucket, int event) { - if (contained != null) { - dateUnion.acceptEvent(bucket, event); - } + // Nothing to do } @Override @@ -91,7 +89,7 @@ public boolean isContained() { @Override public Collection> getDateAggregators() { - return Set.of(dateUnion); + return Set.of(new ConstantValueAggregator<>(dateUnion, new ResultType.ListT(ResultType.DateRangeT.INSTANCE))); } @Override diff --git a/backend/src/test/resources/tests/query/SECONDARY_ID_EXTERNAL_LOGICAL/SECONDARY_IDS.test.json b/backend/src/test/resources/tests/query/SECONDARY_ID_EXTERNAL_LOGICAL/SECONDARY_IDS.test.json new file mode 100644 index 0000000000..92a8b35525 --- /dev/null +++ b/backend/src/test/resources/tests/query/SECONDARY_ID_EXTERNAL_LOGICAL/SECONDARY_IDS.test.json @@ -0,0 +1,114 @@ +{ + "type": "QUERY_TEST", + "label": "SECONDARY_ID Test", + "expectedCsv": "tests/query/SECONDARY_ID_EXTERNAL_LOGICAL/expected.csv", + "query": { + "type": "SECONDARY_ID_QUERY", + "root": { + "type": "AND", + "children": [ + { + "type": "EXTERNAL", + "format": [ + "ID", + "START_DATE", + "END_DATE" + ], + "values": [ + [ + "pid", + "START_DATUM", + "END_DATUM" + ], + [ + "a", + "01.01.2016", + "30.06.2016" + ] + ] + }, + { + "ids": [ + "number" + ], + "type": "CONCEPT", + "excludeFromSecondaryIdQuery": false, + "tables": [ + { + "id": "number.number_connector", + "filters": [ + { + "filter": "number.number_connector.value", + "type": "REAL_RANGE", + "value": { + "min": 1, + "max": 1 + } + } + ] + } + ] + } + ] + }, + "dateAggregationMode": "LOGICAL", + "secondaryId": "secondary" + }, + "concepts": [ + { + "label": "number", + "type": "TREE", + "connectors": [ + { + "label": "number_connector", + "table": "table1", + "validityDates": { + "label": "datum", + "column": "table1.datum" + }, + "filters": { + "label": "value", + "description": "xy", + "column": "table1.value", + "type": "NUMBER" + } + } + ] + } + ], + "content": { + "secondaryIds": [ + { + "name": "secondary" + }, + { + "name": "ignored" + } + ], + "tables": [ + { + "csv": "tests/query/SECONDARY_ID_EXTERNAL_LOGICAL/content.csv", + "name": "table1", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "sid", + "type": "STRING", + "secondaryId": "secondary" + }, + { + "name": "value", + "type": "REAL" + }, + { + "name": "datum", + "type": "DATE_RANGE" + } + ] + } + ] + } +} \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/SECONDARY_ID_EXTERNAL_LOGICAL/content.csv b/backend/src/test/resources/tests/query/SECONDARY_ID_EXTERNAL_LOGICAL/content.csv new file mode 100644 index 0000000000..04f724603a --- /dev/null +++ b/backend/src/test/resources/tests/query/SECONDARY_ID_EXTERNAL_LOGICAL/content.csv @@ -0,0 +1,7 @@ +pid,sid,value,datum +a,f_a1,1,"2016-06-30/2016-06-30" +a,f_a2,1,"2014-06-30/2015-06-30" +a,,1,"2010-06-30/2010-06-30" +a,f_a3,1.01,"2014-06-30/2015-06-30" +b,f_b1,1,"2015-02-03/2015-06-30" +a,f_a1,1,"2014-06-30/2015-06-30" \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/SECONDARY_ID_EXTERNAL_LOGICAL/expected.csv b/backend/src/test/resources/tests/query/SECONDARY_ID_EXTERNAL_LOGICAL/expected.csv new file mode 100644 index 0000000000..09eef28e58 --- /dev/null +++ b/backend/src/test/resources/tests/query/SECONDARY_ID_EXTERNAL_LOGICAL/expected.csv @@ -0,0 +1,3 @@ +result,secondary,dates +a,f_a1,{2016-06-30/2016-06-30} +a,f_a2,{} \ No newline at end of file From 319f1eed0fcda16348544914383d62a1403033bf Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 1 Nov 2021 10:36:35 +0000 Subject: [PATCH 2/3] Update AutoDoc --- docs/Concept JSONs.md | 7 ++++--- docs/REST API JSONs.md | 14 +++++++------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/docs/Concept JSONs.md b/docs/Concept JSONs.md index 3939bdaeb0..eb98a40de9 100644 --- a/docs/Concept JSONs.md +++ b/docs/Concept JSONs.md @@ -674,19 +674,19 @@ Supported Fields: | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/identifiable/NamedImpl.java#L17) | name | `String` | ? | | |

-### QUARTER [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/QuarterSelect.java#L14-L16) +### QUARTER [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/QuarterSelect.java#L15-L17) Output first, last or random Year-Quarter in time
Details

-Java Type: `com.bakdata.conquery.models.datasets.concepts.select.connector.specific.QuarterSelect` +Java Type: `com.bakdata.conquery.models.datasets.concepts.select.concept.specific.QuarterSelect` Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java#L35) | description | `String` | ? | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/QuarterSelect.java#L22) | sample | one of EARLIEST, LATEST, RANDOM | ? | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/QuarterSelect.java#L23) | sample | one of EARLIEST, LATEST, RANDOM | ? | | | | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/identifiable/Labeled.java#L25-L29) | label | `String` | ? | "someLabel" | shown in the frontend | | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/identifiable/NamedImpl.java#L17) | name | `String` | ? | | |

@@ -852,5 +852,6 @@ A Marker UniversalSelect is any of: * [EVENT_DATE_UNION](#EVENT_DATE_UNION) * [EVENT_DURATION_SUM](#EVENT_DURATION_SUM) * [EXISTS](#EXISTS) +* [QUARTER](#QUARTER)

diff --git a/docs/REST API JSONs.md b/docs/REST API JSONs.md index 4d9bf073ec..6a13a723f6 100644 --- a/docs/REST API JSONs.md +++ b/docs/REST API JSONs.md @@ -354,7 +354,7 @@ Supported Fields: | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/forms/managed/RelativeFormQuery.java#L49) | timeUnit | one of DAYS, QUARTERS, YEARS | ? | | |

-### SECONDARY_ID_QUERY [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java#L36) +### SECONDARY_ID_QUERY [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java#L37)
Details

@@ -365,12 +365,12 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java#L49) | dateAggregationMode | one of NONE, MERGE, INTERSECT, LOGICAL | `"MERGE"` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java#L53-L55) | query | [CONCEPT_QUERY](#CONCEPT_QUERY) | ␀ | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java#L42) | root | [@NotNull CQElement](#Base-CQElement) | `null` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java#L45) | secondaryId | ID of `@NsIdRef @NotNull SecondaryIdDescription` | `null` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java#L59) | withSecondaryId | list of ID of `@NsIdRefCollection Set` | ␀ | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java#L63) | withoutSecondaryId | list of ID of `@NsIdRefCollection Set` | ␀ | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java#L50) | dateAggregationMode | one of NONE, MERGE, INTERSECT, LOGICAL | `"MERGE"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java#L54-L56) | query | [CONCEPT_QUERY](#CONCEPT_QUERY) | ␀ | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java#L43) | root | [@NotNull CQElement](#Base-CQElement) | `null` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java#L46) | secondaryId | ID of `@NsIdRef @NotNull SecondaryIdDescription` | `null` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java#L60) | withSecondaryId | list of ID of `@NsIdRefCollection Set` | ␀ | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java#L64) | withoutSecondaryId | list of ID of `@NsIdRefCollection Set
` | ␀ | | |

### TABLE_EXPORT [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java#L49-L51) From 1e6ca15656330a99ea54617b19a4f977d005dec8 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Mon, 1 Nov 2021 12:56:44 +0100 Subject: [PATCH 3/3] simplify test --- .../SECONDARY_IDS.test.json | 18 +----------------- .../SECONDARY_ID_EXTERNAL_LOGICAL/content.csv | 1 - 2 files changed, 1 insertion(+), 18 deletions(-) diff --git a/backend/src/test/resources/tests/query/SECONDARY_ID_EXTERNAL_LOGICAL/SECONDARY_IDS.test.json b/backend/src/test/resources/tests/query/SECONDARY_ID_EXTERNAL_LOGICAL/SECONDARY_IDS.test.json index 92a8b35525..68ce0689b8 100644 --- a/backend/src/test/resources/tests/query/SECONDARY_ID_EXTERNAL_LOGICAL/SECONDARY_IDS.test.json +++ b/backend/src/test/resources/tests/query/SECONDARY_ID_EXTERNAL_LOGICAL/SECONDARY_IDS.test.json @@ -35,17 +35,7 @@ "excludeFromSecondaryIdQuery": false, "tables": [ { - "id": "number.number_connector", - "filters": [ - { - "filter": "number.number_connector.value", - "type": "REAL_RANGE", - "value": { - "min": 1, - "max": 1 - } - } - ] + "id": "number.number_connector" } ] } @@ -65,12 +55,6 @@ "validityDates": { "label": "datum", "column": "table1.datum" - }, - "filters": { - "label": "value", - "description": "xy", - "column": "table1.value", - "type": "NUMBER" } } ] diff --git a/backend/src/test/resources/tests/query/SECONDARY_ID_EXTERNAL_LOGICAL/content.csv b/backend/src/test/resources/tests/query/SECONDARY_ID_EXTERNAL_LOGICAL/content.csv index 04f724603a..a2f9c24861 100644 --- a/backend/src/test/resources/tests/query/SECONDARY_ID_EXTERNAL_LOGICAL/content.csv +++ b/backend/src/test/resources/tests/query/SECONDARY_ID_EXTERNAL_LOGICAL/content.csv @@ -2,6 +2,5 @@ pid,sid,value,datum a,f_a1,1,"2016-06-30/2016-06-30" a,f_a2,1,"2014-06-30/2015-06-30" a,,1,"2010-06-30/2010-06-30" -a,f_a3,1.01,"2014-06-30/2015-06-30" b,f_b1,1,"2015-02-03/2015-06-30" a,f_a1,1,"2014-06-30/2015-06-30" \ No newline at end of file