From 324cc58708dc856c506a8b847781e9ed5551cf0b Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Tue, 6 Jul 2021 16:33:40 +0200 Subject: [PATCH 01/82] initial rework of IdMapping including CQExternal, toward explicitly stated Ids instead of guessing the used Ids Also refactors reading of IdMapping to use Headers and not indices, we don't need insane performance here anyway --- .../conquery/apiv1/QueryProcessor.java | 2 +- .../apiv1/query/QueryDescription.java | 2 +- .../query/concept/specific/CQExternal.java | 195 ------------------ .../concept/specific/external/CQExternal.java | 194 +++++++++++++++++ .../concept/specific/external/DateColumn.java | 50 +++++ .../concept/specific/external/DateFormat.java | 78 +++++++ .../specific/external/DateSetColumn.java | 6 + .../specific/external/FormatColumn.java | 18 ++ .../concept/specific/external/IdColumn.java | 13 ++ .../PersistentIdMapDeserializer.java | 18 +- .../serializer/PersistentIdMapSerializer.java | 20 +- .../conquery/io/result/ResultUtil.java | 30 +-- .../conquery/io/result/csv/CsvRenderer.java | 4 +- .../conquery/io/storage/NamespaceStorage.java | 9 +- .../conquery/io/storage/StoreInfo.java | 4 +- .../conquery/models/config/NoIdMapping.java | 62 +----- .../models/config/SimpleIdMapping.java | 41 ++-- .../conquery/models/config/StoreFactory.java | 4 +- .../models/config/XodusStoreFactory.java | 6 +- .../conquery/models/datasets/Table.java | 8 + .../models/exceptions/ParsingException.java | 2 +- .../exceptions/validators/ValidCSVFormat.java | 90 -------- .../identifiable/mapping/CsvEntityId.java | 2 +- .../mapping/DefaultIdAccessorImpl.java | 18 -- .../mapping/DefaultIdMappingAccessor.java | 38 ---- .../models/identifiable/mapping/EntityId.java | 5 - .../identifiable/mapping/EntityIdMap.java | 72 +++++++ ...ternalEntityId.java => EntityPrintId.java} | 10 +- .../identifiable/mapping/IdAccessor.java | 14 -- .../identifiable/mapping/IdAccessorImpl.java | 79 ------- .../models/identifiable/mapping/IdMapper.java | 27 --- .../mapping/IdMappingAccessor.java | 67 ------ .../identifiable/mapping/IdMappingConfig.java | 75 +++---- .../identifiable/mapping/PersistentIdMap.java | 78 ------- .../identifiable/mapping/PrintIdMapper.java | 2 +- ...lEntityId.java => UnresolvedEntityId.java} | 4 +- .../parser/specific/DateRangeParser.java | 4 +- .../models/query/ExecutionManager.java | 10 +- .../conquery/models/query/ManagedQuery.java | 2 +- .../admin/rest/AdminDatasetProcessor.java | 61 ++---- .../admin/ui/DatasetsUIResource.java | 4 +- .../com/bakdata/conquery/util/QueryUtils.java | 2 +- .../api/StoredQueriesProcessorTest.java | 2 +- .../integration/common/IntegrationUtils.java | 2 + .../integration/common/LoadingUtil.java | 18 +- .../integration/common/RequiredTable.java | 3 + .../json/AbstractQueryEngineTest.java | 29 ++- .../conquery/integration/json/FormTest.java | 2 +- .../conquery/integration/json/QueryTest.java | 3 +- .../integration/json/filter/FilterTest.java | 9 +- .../tests/ConceptPermissionTest.java | 2 +- .../integration/tests/RestartTest.java | 12 +- .../integration/tests/ReusedQueryTest.java | 2 +- .../ConceptUpdateAndDeletionTest.java | 2 +- .../tests/deletion/DatasetDeletionTest.java | 6 +- .../tests/deletion/ImportDeletionTest.java | 2 +- .../tests/deletion/TableDeletionTest.java | 9 +- .../io/result/CsvLineStreamRenderer.java | 6 +- .../arrow/ArrowResultGenerationTest.java | 4 +- .../result/csv/CsvResultGenerationTest.java | 4 +- .../result/excel/ExcelResultRenderTest.java | 4 +- .../conquery/models/SerializationTests.java | 4 +- .../models/execution/DefaultLabelTest.java | 2 +- .../identifiable/IdMapSerialisationTest.java | 22 +- .../util/NonPersistentStoreFactory.java | 4 +- docs/REST API JSONs.md | 2 +- 66 files changed, 663 insertions(+), 922 deletions(-) delete mode 100644 backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQExternal.java create mode 100644 backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java create mode 100644 backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateColumn.java create mode 100644 backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java create mode 100644 backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateSetColumn.java create mode 100644 backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/FormatColumn.java create mode 100644 backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/IdColumn.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/models/exceptions/validators/ValidCSVFormat.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/DefaultIdAccessorImpl.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/DefaultIdMappingAccessor.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityId.java create mode 100644 backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java rename backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/{ExternalEntityId.java => EntityPrintId.java} (63%) delete mode 100644 backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdAccessor.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdAccessorImpl.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMapper.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingAccessor.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/PersistentIdMap.java rename backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/{SufficientExternalEntityId.java => UnresolvedEntityId.java} (69%) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java index c8665e2246..69d79fc520 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java @@ -2,7 +2,7 @@ import com.bakdata.conquery.apiv1.query.*; import com.bakdata.conquery.apiv1.query.concept.specific.CQAnd; -import com.bakdata.conquery.apiv1.query.concept.specific.CQExternal; +import com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal; import com.bakdata.conquery.io.result.ResultRender.ResultRendererProvider; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.metrics.ExecutionMetrics; diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/QueryDescription.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/QueryDescription.java index 965000dcbd..44a419493f 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/QueryDescription.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/QueryDescription.java @@ -16,7 +16,7 @@ import com.bakdata.conquery.models.identifiable.ids.NamespacedIdentifiable; import com.bakdata.conquery.models.query.QueryResolveContext; import com.bakdata.conquery.models.query.Visitable; -import com.bakdata.conquery.apiv1.query.concept.specific.CQExternal; +import com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal; import com.bakdata.conquery.models.query.visitor.QueryVisitor; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.util.QueryUtils; diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQExternal.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQExternal.java deleted file mode 100644 index feb8337408..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQExternal.java +++ /dev/null @@ -1,195 +0,0 @@ -package com.bakdata.conquery.apiv1.query.concept.specific; - -import java.time.LocalDate; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; - -import javax.validation.constraints.NotEmpty; - -import com.bakdata.conquery.apiv1.query.CQElement; -import com.bakdata.conquery.io.cps.CPSType; -import com.bakdata.conquery.io.jackson.InternalOnly; -import com.bakdata.conquery.models.common.CDateSet; -import com.bakdata.conquery.models.common.daterange.CDateRange; -import com.bakdata.conquery.models.dictionary.EncodedDictionary; -import com.bakdata.conquery.models.error.ConqueryError; -import com.bakdata.conquery.models.exceptions.ParsingException; -import com.bakdata.conquery.models.exceptions.validators.ValidCSVFormat; -import com.bakdata.conquery.models.identifiable.mapping.CsvEntityId; -import com.bakdata.conquery.models.identifiable.mapping.IdAccessor; -import com.bakdata.conquery.models.identifiable.mapping.IdAccessorImpl; -import com.bakdata.conquery.models.identifiable.mapping.IdMappingConfig; -import com.bakdata.conquery.models.preproc.parser.specific.DateRangeParser; -import com.bakdata.conquery.models.query.QueryPlanContext; -import com.bakdata.conquery.models.query.QueryResolveContext; -import com.bakdata.conquery.models.query.queryplan.ConceptQueryPlan; -import com.bakdata.conquery.models.query.queryplan.QPNode; -import com.bakdata.conquery.models.query.queryplan.specific.ExternalNode; -import com.bakdata.conquery.models.query.resultinfo.ResultInfoCollector; -import com.bakdata.conquery.util.DateFormats; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.google.common.collect.MoreCollectors; -import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; -import lombok.Getter; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@CPSType(id = "EXTERNAL", base = CQElement.class) -@RequiredArgsConstructor(onConstructor_ = @JsonCreator) -public class CQExternal extends CQElement { - - @Getter - @NotEmpty - @ValidCSVFormat - private final List format; - @Getter - @NotEmpty - private final String[][] values; - - @Getter @InternalOnly - private Map valuesResolved; - - @Override - public QPNode createQueryPlan(QueryPlanContext context, ConceptQueryPlan plan) { - if (valuesResolved == null) { - throw new IllegalStateException("CQExternal needs to be resolved before creating a plan"); - } - return new ExternalNode(context.getStorage().getDataset().getAllIdsTable(), valuesResolved); - } - - - @Override - public void resolve(QueryResolveContext context) { - EncodedDictionary primary = context.getNamespace().getStorage().getPrimaryDictionary(); - Optional dateFormat = format.stream() - .map(FormatColumn::getDateFormat) - .filter(Objects::nonNull) - .distinct() - .collect(MoreCollectors.toOptional()); - int[] dateIndices = format.stream() - .filter(fc -> fc.getDateFormat() != null) - .mapToInt(format::indexOf) - .toArray(); - - valuesResolved = new Int2ObjectOpenHashMap<>(); - - IdMappingConfig mapping = context.getConfig().getIdMapping(); - - IdAccessor idAccessor = mapping.mappingFromCsvHeader( - IdAccessorImpl.selectIdFields(values[0], format), - context.getNamespace().getStorage().getIdMapping() - ); - List> nonResolved = new ArrayList<>(); - - if (values[0].length != format.size()) { - throw new ConqueryError.ExternalResolveFormatError(format.size(), values[0].length); - } - - - // ignore the first row, because this is the header - for (int i = 1; i < values.length; i++) { - String[] row = values[i]; - - //read the dates from the row - try { - CDateSet dates = dateFormat.map(df -> { - try { - return df.readDates(dateIndices, row, context.getConfig().getPreprocessor().getParsers().getDateFormats()); - } - catch (Exception e) { - throw new RuntimeException(e); - } - }).orElseGet(CDateSet::createFull); - // remove all fields from the data line that are not id fields, in case the mapping is not possible we avoid the data columns to be joined - CsvEntityId id = idAccessor.getCsvEntityId(IdAccessorImpl.selectIdFields(row, format)); - - int resolvedId; - if (id != null && (resolvedId = primary.getId(id.getCsvId())) != -1) { - valuesResolved.put(resolvedId, dates); - } - else { - nonResolved.add(Arrays.asList(row)); - } - } - catch (Exception e) { - log.warn("Failed to parse id from " + Arrays.toString(row), e); - } - } - if (!nonResolved.isEmpty()) { - log.warn( - "Could not resolve {} of the {} rows. Not resolved: {}", - nonResolved.size(), - values.length - 1, - nonResolved.subList(0, Math.min(nonResolved.size(), 10)) - ); - } - - if (valuesResolved.isEmpty()) { - throw new ConqueryError.ExternalResolveEmptyError(); - } - } - - @Override - public void collectResultInfos(ResultInfoCollector collector) { - } - - public enum DateFormat { - EVENT_DATE { - @Override - public CDateSet readDates(int[] dateIndices, String[] row, DateFormats dateFormats) throws ParsingException { - return CDateSet.create(Collections.singleton(CDateRange.exactly(dateFormats.parseToLocalDate(row[dateIndices[0]])))); - } - }, - START_END_DATE { - @Override - public CDateSet readDates(int[] dateIndices, String[] row, DateFormats dateFormats) throws ParsingException { - LocalDate start = row[dateIndices[0]] == null ? null : dateFormats.parseToLocalDate(row[dateIndices[0]]); - - LocalDate end = (dateIndices.length < 2 || row[dateIndices[1]] == null) ? - null : - dateFormats.parseToLocalDate(row[dateIndices[1]]); - - if (start == null && end == null) { - return null; - } - - return CDateSet.create(Collections.singleton(CDateRange.of(start, end))); - } - }, - DATE_RANGE { - @Override - public CDateSet readDates(int[] dateIndices, String[] row, DateFormats dateFormats) throws ParsingException { - return CDateSet.create(Collections.singleton(DateRangeParser.parseISORange(row[dateIndices[0]], dateFormats))); - } - }, - DATE_SET { - @Override - public CDateSet readDates(int[] dateIndices, String[] row, DateFormats dateFormats) throws ParsingException { - return CDateSet.parse(row[dateIndices[0]], dateFormats); - } - }; - - public abstract CDateSet readDates(int[] dateIndices, String[] row, DateFormats dateFormats) throws ParsingException; - } - - @RequiredArgsConstructor - @Getter - public enum FormatColumn { - ID(true, null), - EVENT_DATE(false, DateFormat.EVENT_DATE), - START_DATE(false, DateFormat.START_END_DATE), - END_DATE(false, DateFormat.START_END_DATE), - DATE_RANGE(false, DateFormat.DATE_RANGE), - DATE_SET(false, DateFormat.DATE_SET), - IGNORE(true, null); - - private final boolean duplicatesAllowed; - private final DateFormat dateFormat; - } -} diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java new file mode 100644 index 0000000000..094f6f026f --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java @@ -0,0 +1,194 @@ +package com.bakdata.conquery.apiv1.query.concept.specific.external; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import javax.validation.constraints.NotEmpty; + +import com.bakdata.conquery.apiv1.query.CQElement; +import com.bakdata.conquery.io.cps.CPSType; +import com.bakdata.conquery.io.jackson.InternalOnly; +import com.bakdata.conquery.models.common.CDateSet; +import com.bakdata.conquery.models.dictionary.EncodedDictionary; +import com.bakdata.conquery.models.error.ConqueryError; +import com.bakdata.conquery.models.identifiable.mapping.CsvEntityId; +import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; +import com.bakdata.conquery.models.identifiable.mapping.UnresolvedEntityId; +import com.bakdata.conquery.models.query.QueryPlanContext; +import com.bakdata.conquery.models.query.QueryResolveContext; +import com.bakdata.conquery.models.query.queryplan.ConceptQueryPlan; +import com.bakdata.conquery.models.query.queryplan.QPNode; +import com.bakdata.conquery.models.query.queryplan.specific.ExternalNode; +import com.bakdata.conquery.models.query.resultinfo.ResultInfoCollector; +import com.bakdata.conquery.util.DateFormats; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.google.common.collect.MoreCollectors; +import io.dropwizard.validation.ValidationMethod; +import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@CPSType(id = "EXTERNAL", base = CQElement.class) +public class CQExternal extends CQElement { + + @Getter + @NotEmpty + private final List format; + @Getter + @NotEmpty + private final String[][] values; + + @Getter + @InternalOnly + private Map valuesResolved; + + public CQExternal(@NotEmpty List format, @NotEmpty String[][] values) { + this.format = format; + this.values = values; + + for (int index = 0; index < format.size(); index++) { + format.get(index).setPosition(index); + } + } + + @Override + public QPNode createQueryPlan(QueryPlanContext context, ConceptQueryPlan plan) { + if (valuesResolved == null) { + throw new IllegalStateException("CQExternal needs to be resolved before creating a plan"); + } + return new ExternalNode(context.getStorage().getDataset().getAllIdsTable(), valuesResolved); + } + + + @Override + public void resolve(QueryResolveContext context) { + + valuesResolved = new Int2ObjectOpenHashMap<>(); + + + //TODO probably easy to simplify. + + final DateColumn[] dateColumns = format.stream() + .filter(DateColumn.class::isInstance) + .map(DateColumn.class::cast) + .toArray(DateColumn[]::new); + + //TODO verify dateColumns match + + final DateFormat dateFormat = dateColumns.length > 0 ? dateColumns[0].getFormat() : DateFormat.ALL; + + + final IdColumn idColumn = this.format.stream() + .filter(IdColumn.class::isInstance) + .map(IdColumn.class::cast) + .collect(MoreCollectors.onlyElement()); + + + final EncodedDictionary primary = context.getNamespace().getStorage().getPrimaryDictionary(); + final EntityIdMap mapping = context.getNamespace().getStorage().getIdMapping(); + final DateFormats dateFormats = context.getConfig().getPreprocessor().getParsers().getDateFormats(); + + + List nonResolved = new ArrayList<>(); + + + // ignore the first row, because this is the header + for (int i = 1; i < values.length; i++) { + final String[] row = values[i]; + final UnresolvedEntityId externalId = idColumn.read(row); + + + //read the dates from the row + try { + + CDateSet dates = dateFormat.readDates(dateColumns, row, dateFormats); + + Optional id = mapping.toInternal(externalId); + + int resolvedId; + + if (id.isPresent() && (resolvedId = primary.getId(id.get().getCsvId())) != -1) { + valuesResolved.put(resolvedId, dates); + } + else { + nonResolved.add(row); + } + } + catch (Exception e) { + log.warn("Failed to parse id from {}", row, e); + } + } + if (!nonResolved.isEmpty()) { + log.warn( + "Could not resolve {} of the {} rows. Not resolved: {}", + nonResolved.size(), + values.length - 1, + nonResolved.subList(0, Math.min(nonResolved.size(), 10)) + ); + } + + if (valuesResolved.isEmpty()) { + throw new ConqueryError.ExternalResolveEmptyError(); + } + } + + @Override + public void collectResultInfos(ResultInfoCollector collector) { + } + + // @RequiredArgsConstructor + // @Getter + // public enum FormatColumn { + // ID(true, null), + // EVENT_DATE(false, DateFormat.EVENT_DATE), + // START_DATE(false, DateFormat.START_END_DATE), + // END_DATE(false, DateFormat.START_END_DATE), + // DATE_RANGE(false, DateFormat.DATE_RANGE), + // DATE_SET(false, DateFormat.DATE_SET), + // IGNORE(true, null); + // + // private final boolean duplicatesAllowed; + // private final DateFormat dateFormat; + // } + + + @JsonIgnore + @ValidationMethod(message = "Values and Format are not of same width.") + public boolean isAllSameLength() { + final int expected = format.size(); + return Arrays.stream(values).mapToInt(a -> a.length).allMatch(v -> expected == v); + } + + @JsonIgnore + @ValidationMethod(message = "Must use one Id.") + public boolean isOnlyOneId() { + return format.stream().filter(IdColumn.class::isInstance).count() == 1; + } + + //TODO implement this + + // @ValidationMethod(message = "Wrong usage of Dates.") + // public boolean isDatesAccessorExclusive() { + // if (format.contains(FormatColumn.DATE_RANGE)) { + // return !(format.contains(FormatColumn.DATE_SET) || format.contains(FormatColumn.START_DATE) || format.contains(FormatColumn.END_DATE)); + // } + // + // if (format.contains(FormatColumn.DATE_SET)) { + // return !(format.contains(FormatColumn.START_DATE) || format.contains(FormatColumn.END_DATE)); + // } + // + // if (format.stream().filter(FormatColumn.START_DATE::equals).count() > 1) { + // return false; + // } + // + // if (format.stream().filter(FormatColumn.END_DATE::equals).count() > 1) { + // return false; + // } + // + // return true; + // } +} diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateColumn.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateColumn.java new file mode 100644 index 0000000000..ec752f10d0 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateColumn.java @@ -0,0 +1,50 @@ +package com.bakdata.conquery.apiv1.query.concept.specific.external; + +import com.bakdata.conquery.io.cps.CPSType; +import lombok.Getter; + +@Getter +public abstract class DateColumn extends FormatColumn { + + private final DateFormat format; + + public DateColumn(DateFormat format) { + this.format = format; + } + + @CPSType(id = "DATE_SET", base = FormatColumn.class) + public static class DateSet extends DateColumn { + public DateSet() { + super(DateFormat.DATE_SET); + } + } + + @CPSType(id = "DATE_RANGE", base = FormatColumn.class) + public static class DateRange extends DateColumn { + public DateRange() { + super(DateFormat.DATE_RANGE); + } + } + + @CPSType(id = "EVENT_DATE", base = FormatColumn.class) + public static class EventDate extends DateColumn { + public EventDate() { + super(DateFormat.EVENT_DATE); + } + } + + + @CPSType(id = "START_DATE", base = FormatColumn.class) + public static class DateStart extends DateColumn { + public DateStart() { + super(DateFormat.START_END_DATE); + } + } + + @CPSType(id = "END_DATE", base = FormatColumn.class) + public static class DateEnd extends DateColumn { + public DateEnd() { + super(DateFormat.START_END_DATE); + } + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java new file mode 100644 index 0000000000..cd4c15dbe7 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java @@ -0,0 +1,78 @@ +package com.bakdata.conquery.apiv1.query.concept.specific.external; + +import java.time.LocalDate; +import java.util.Arrays; +import java.util.Collections; + +import com.bakdata.conquery.models.common.CDateSet; +import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.bakdata.conquery.models.preproc.parser.specific.DateRangeParser; +import com.bakdata.conquery.util.DateFormats; +import com.google.common.base.Preconditions; +import com.google.common.collect.MoreCollectors; + +public enum DateFormat { + EVENT_DATE { + @Override + public CDateSet readDates(DateColumn[] dateIndices, String[] row, DateFormats dateFormats) { + Preconditions.checkArgument(dateIndices.length == 1); + + return CDateSet.create(Collections.singleton(CDateRange.exactly(dateFormats.parseToLocalDate(row[dateIndices[0].getPosition()])))); + } + }, + START_END_DATE { + @Override + public CDateSet readDates(DateColumn[] dateIndices, String[] row, DateFormats dateFormats) { + Preconditions.checkArgument(dateIndices.length == 1 || dateIndices.length == 2); + + + LocalDate start = Arrays.stream(dateIndices) + .filter(DateColumn.DateStart.class::isInstance) + .map(DateColumn::getPosition) + .map(idx -> row[idx]) + .map(dateFormats::parseToLocalDate) + .collect(MoreCollectors.toOptional()) + .orElse(null); + + LocalDate end = Arrays.stream(dateIndices) + .filter(DateColumn.DateEnd.class::isInstance) + .map(DateColumn::getPosition) + .map(idx -> row[idx]) + .map(dateFormats::parseToLocalDate) + .collect(MoreCollectors.toOptional()) + .orElse(null); + + if (start == null && end == null) { + return null; + } + + return CDateSet.create(Collections.singleton(CDateRange.of(start, end))); + } + }, + DATE_RANGE { + @Override + public CDateSet readDates(DateColumn[] dateIndices, String[] row, DateFormats dateFormats) { + Preconditions.checkArgument(dateIndices.length == 1); + + return CDateSet.create(Collections.singleton(DateRangeParser.parseISORange(row[dateIndices[0].getPosition()], dateFormats))); + } + }, + DATE_SET { + @Override + public CDateSet readDates(DateColumn[] dateIndices, String[] row, DateFormats dateFormats) { + Preconditions.checkArgument(dateIndices.length == 1); + + return CDateSet.parse(row[dateIndices[0].getPosition()], dateFormats); + } + }, + ALL { + @Override + public CDateSet readDates(DateColumn[] dateIndices, String[] row, DateFormats dateFormats) { + Preconditions.checkArgument(dateIndices.length == 0); + + return CDateSet.createFull(); + } + }; + + public abstract CDateSet readDates(DateColumn[] dateIndices, String[] row, DateFormats dateFormats); +} diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateSetColumn.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateSetColumn.java new file mode 100644 index 0000000000..80efa72de5 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateSetColumn.java @@ -0,0 +1,6 @@ +package com.bakdata.conquery.apiv1.query.concept.specific.external; + +import com.bakdata.conquery.apiv1.forms.Form; +import com.bakdata.conquery.io.cps.CPSType; + + diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/FormatColumn.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/FormatColumn.java new file mode 100644 index 0000000000..62f304a28d --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/FormatColumn.java @@ -0,0 +1,18 @@ +package com.bakdata.conquery.apiv1.query.concept.specific.external; + +import com.bakdata.conquery.io.cps.CPSBase; +import com.bakdata.conquery.io.jackson.InternalOnly; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import lombok.Data; +import lombok.Getter; +import lombok.Setter; + +@CPSBase +@JsonTypeInfo(use = JsonTypeInfo.Id.CUSTOM, property = "type") +@Data +public abstract class FormatColumn { + + @InternalOnly @Setter @Getter + private int position; + +} diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/IdColumn.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/IdColumn.java new file mode 100644 index 0000000000..5531120f7e --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/IdColumn.java @@ -0,0 +1,13 @@ +package com.bakdata.conquery.apiv1.query.concept.specific.external; + +import com.bakdata.conquery.io.cps.CPSType; +import com.bakdata.conquery.models.identifiable.mapping.UnresolvedEntityId; + +@CPSType(id = "ID", base = FormatColumn.class) +public class IdColumn extends FormatColumn { + + + public UnresolvedEntityId read(String[] row) { + return new UnresolvedEntityId(row[getPosition()]); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/PersistentIdMapDeserializer.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/PersistentIdMapDeserializer.java index 265fd27ddc..69cee6fe60 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/PersistentIdMapDeserializer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/PersistentIdMapDeserializer.java @@ -7,30 +7,30 @@ import java.util.Map; import com.bakdata.conquery.models.identifiable.mapping.CsvEntityId; -import com.bakdata.conquery.models.identifiable.mapping.ExternalEntityId; -import com.bakdata.conquery.models.identifiable.mapping.PersistentIdMap; -import com.bakdata.conquery.models.identifiable.mapping.SufficientExternalEntityId; +import com.bakdata.conquery.models.identifiable.mapping.EntityPrintId; +import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; +import com.bakdata.conquery.models.identifiable.mapping.UnresolvedEntityId; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.JsonDeserializer; -public class PersistentIdMapDeserializer extends JsonDeserializer { +public class PersistentIdMapDeserializer extends JsonDeserializer { private static final TypeReference arrayOfMapEntryType = new TypeReference>() {}; - private static final TypeReference mapOfCsvToExternalIdType = new TypeReference>() {}; + private static final TypeReference mapOfCsvToExternalIdType = new TypeReference>() {}; @Override - public PersistentIdMap deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { + public EntityIdMap deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { - Map csvIdToExternalIdMap = p.readValueAs(mapOfCsvToExternalIdType); - Map externalIdPartCsvIdMap = new HashMap<>(); + Map csvIdToExternalIdMap = p.readValueAs(mapOfCsvToExternalIdType); + Map externalIdPartCsvIdMap = new HashMap<>(); List mapAsList = p.readValueAs(arrayOfMapEntryType); mapAsList.forEach(externalIdMapEntry -> { externalIdPartCsvIdMap.put(externalIdMapEntry.getSufficientExternalEntityId(), externalIdMapEntry.getCsvEntityId()); }); - PersistentIdMap map = new PersistentIdMap(); + EntityIdMap map = new EntityIdMap(); map.getCsvIdToExternalIdMap().putAll(csvIdToExternalIdMap); map.getExternalIdPartCsvIdMap().putAll(externalIdPartCsvIdMap); return map; diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/PersistentIdMapSerializer.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/PersistentIdMapSerializer.java index 4250c3a58d..7af582ed73 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/PersistentIdMapSerializer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/PersistentIdMapSerializer.java @@ -3,31 +3,35 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Map; import com.bakdata.conquery.models.identifiable.mapping.CsvEntityId; -import com.bakdata.conquery.models.identifiable.mapping.PersistentIdMap; -import com.bakdata.conquery.models.identifiable.mapping.SufficientExternalEntityId; +import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; +import com.bakdata.conquery.models.identifiable.mapping.UnresolvedEntityId; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.JsonSerializer; import com.fasterxml.jackson.databind.SerializerProvider; - import lombok.Data; -public class PersistentIdMapSerializer extends JsonSerializer { +public class PersistentIdMapSerializer extends JsonSerializer { @Override - public void serialize(PersistentIdMap value, JsonGenerator gen, SerializerProvider provider) throws IOException { + public void serialize(EntityIdMap value, JsonGenerator gen, SerializerProvider provider) throws IOException { gen.writeObject(value.getCsvIdToExternalIdMap()); List mapAsList = new ArrayList<>(); - value.getExternalIdPartCsvIdMap().forEach((key, val) -> { + + for (Map.Entry entry : value.getExternalIdPartCsvIdMap().entrySet()) { + UnresolvedEntityId key = entry.getKey(); + CsvEntityId val = entry.getValue(); mapAsList.add(new ExternalIdMapEntry(key, val)); - }); + } + gen.writeObject(mapAsList); } @Data public static class ExternalIdMapEntry { - private final SufficientExternalEntityId sufficientExternalEntityId; + private final UnresolvedEntityId sufficientExternalEntityId; private final CsvEntityId csvEntityId; } } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/ResultUtil.java b/backend/src/main/java/com/bakdata/conquery/io/result/ResultUtil.java index d0ca96473d..2c46d37e8e 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/ResultUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/ResultUtil.java @@ -1,11 +1,19 @@ package com.bakdata.conquery.io.result; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; + +import javax.ws.rs.BadRequestException; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.StreamingOutput; + import com.bakdata.conquery.models.dictionary.EncodedDictionary; import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.identifiable.mapping.CsvEntityId; -import com.bakdata.conquery.models.identifiable.mapping.ExternalEntityId; +import com.bakdata.conquery.models.identifiable.mapping.EntityPrintId; import com.bakdata.conquery.models.identifiable.mapping.IdMappingConfig; import com.bakdata.conquery.models.identifiable.mapping.IdMappingState; +import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; import com.bakdata.conquery.models.query.SingleTableResult; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.worker.Namespace; @@ -13,23 +21,19 @@ import com.google.common.base.Strings; import lombok.extern.slf4j.Slf4j; -import javax.ws.rs.BadRequestException; -import javax.ws.rs.core.Response; -import javax.ws.rs.core.StreamingOutput; -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; - @Slf4j public class ResultUtil { - public static ExternalEntityId createId(Namespace namespace, EntityResult cer, IdMappingConfig idMappingConfig, IdMappingState mappingState) { + public static EntityPrintId createId(Namespace namespace, EntityResult cer, IdMappingConfig idMappingConfig, IdMappingState mappingState) { EncodedDictionary dict = namespace.getStorage().getPrimaryDictionary(); - return idMappingConfig - .toExternal( - new CsvEntityId(dict.getElement(cer.getEntityId())), - namespace, - mappingState); + final EntityIdMap idMapping = namespace.getStorage().getIdMapping(); + + + + return idMappingConfig.toExternal(new CsvEntityId(dict.getElement(cer.getEntityId())), namespace, + mappingState, idMapping + ); } public static Response makeResponseWithFileName(StreamingOutput out, String label, String fileExtension) { diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/csv/CsvRenderer.java b/backend/src/main/java/com/bakdata/conquery/io/result/csv/CsvRenderer.java index d28686aa43..3fd71ae087 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/csv/CsvRenderer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/csv/CsvRenderer.java @@ -1,6 +1,6 @@ package com.bakdata.conquery.io.result.csv; -import com.bakdata.conquery.models.identifiable.mapping.ExternalEntityId; +import com.bakdata.conquery.models.identifiable.mapping.EntityPrintId; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.results.EntityResult; @@ -42,7 +42,7 @@ private void createCSVBody(PrintSettings cfg, List infos, Stream infos, Object[] value) { + public void printLine(PrintSettings cfg, EntityPrintId entity, List infos, Object[] value) { // Cast here to Object[] so it is clear to intellij that the varargs call is intended writer.addValues((Object[]) entity.getExternalId()); try { diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java index c86b0a0589..0c311b7492 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java @@ -8,7 +8,7 @@ import com.bakdata.conquery.io.storage.xodus.stores.SingletonStore; import com.bakdata.conquery.models.datasets.concepts.StructureNode; import com.bakdata.conquery.models.config.StoreFactory; -import com.bakdata.conquery.models.identifiable.mapping.PersistentIdMap; +import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; import com.bakdata.conquery.models.worker.SingletonNamespaceCollection; import com.bakdata.conquery.models.worker.WorkerToBucketsMap; import lombok.Getter; @@ -23,7 +23,7 @@ public class NamespaceStorage extends NamespacedStorage { @Setter @NonNull private MetaStorage metaStorage; - protected SingletonStore idMapping; + protected SingletonStore idMapping; protected SingletonStore structure; protected SingletonStore workerToBuckets; @@ -38,6 +38,7 @@ public NamespaceStorage(Validator validator, StoreFactory storageFactory, String workerToBuckets = storageFactory.createWorkerToBucketsStore(pathName); } + @Override public void loadData() { super.loadData(); @@ -71,12 +72,12 @@ public void close() throws IOException { workerToBuckets.close(); } - public PersistentIdMap getIdMapping() { + public EntityIdMap getIdMapping() { return idMapping.get(); } - public void updateIdMapping(PersistentIdMap idMapping) { + public void updateIdMapping(EntityIdMap idMapping) { this.idMapping.update(idMapping); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/StoreInfo.java b/backend/src/main/java/com/bakdata/conquery/io/storage/StoreInfo.java index cc7a241234..ea6440b9fa 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/StoreInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/StoreInfo.java @@ -32,7 +32,7 @@ import com.bakdata.conquery.models.identifiable.ids.specific.SecondaryIdDescriptionId; import com.bakdata.conquery.models.identifiable.ids.specific.TableId; import com.bakdata.conquery.models.identifiable.ids.specific.UserId; -import com.bakdata.conquery.models.identifiable.mapping.PersistentIdMap; +import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.ShardNodeInformation; import com.bakdata.conquery.models.worker.SingletonNamespaceCollection; @@ -52,7 +52,7 @@ @ToString(of={"name", "keyType", "valueType" }) public enum StoreInfo implements IStoreInfo { DATASET(Dataset.class, Boolean.class), - ID_MAPPING(PersistentIdMap.class, Boolean.class), + ID_MAPPING(EntityIdMap.class, Boolean.class), NAMESPACES(DatasetRegistry.class, Boolean.class), SLAVE(ShardNodeInformation.class, Boolean.class), DICTIONARIES(Dictionary.class, DictionaryId.class), diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/NoIdMapping.java b/backend/src/main/java/com/bakdata/conquery/models/config/NoIdMapping.java index d6ebfda137..8bfffccbd2 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/NoIdMapping.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/NoIdMapping.java @@ -4,69 +4,23 @@ import java.util.List; import com.bakdata.conquery.io.cps.CPSType; -import com.bakdata.conquery.models.identifiable.mapping.*; -import lombok.RequiredArgsConstructor; -import org.apache.commons.lang3.ArrayUtils; +import com.bakdata.conquery.models.identifiable.mapping.CsvEntityId; +import com.bakdata.conquery.models.identifiable.mapping.IdMappingConfig; +import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; +import com.univocity.parsers.common.record.Record; @CPSType(base = IdMappingConfig.class, id = "NO_ID_MAPPING") public class NoIdMapping extends IdMappingConfig { - private static final String[] HEADER = new String[] {"result"}; - private static final NoIdMappingAccessor[] ACCESSORS = new NoIdMappingAccessor[]{NoIdMappingAccessor.INSTANCE}; + private static final String[] HEADER = new String[]{"result"}; @Override - public IdMappingAccessor[] getIdAccessors() { - return ACCESSORS; + protected void processRecord(Record record, CsvEntityId id, EntityIdMap mapping) { + // Do nothing. } @Override - public List getPrintIdFields(){ + public List getPrintIdFields() { return Arrays.asList(HEADER); } - @Override - public String[] getHeader() { - return HEADER; - } - - private enum NoIdMappingAccessor implements IdMappingAccessor { - INSTANCE; - @Override - public String[] getHeader() { - return HEADER; - } - - @Override - public boolean canBeApplied(List csvHeader) { - return true; - } - - @Override - public IdAccessor getApplicationMapping(String[] csvHeader, final PersistentIdMap idMapping) { - return NoIdAccessor.INSTANCE; - } - - @Override - public String[] extract(String[] dataLine) { - return new String[]{dataLine[0]}; - } - - @Override - public CsvEntityId getFallbackCsvId(String[] reorderedCsvLine) { - return new CsvEntityId(reorderedCsvLine[0]); - } - - @Override - public int findIndexFromMappingHeader(String csvHeaderField) { - return ArrayUtils.indexOf(getHeader(), csvHeaderField); - } - } - - @RequiredArgsConstructor - private enum NoIdAccessor implements IdAccessor { - INSTANCE; - @Override - public CsvEntityId getCsvEntityId(String[] csvLine) { - return new CsvEntityId(csvLine[0]); - } - } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/SimpleIdMapping.java b/backend/src/main/java/com/bakdata/conquery/models/config/SimpleIdMapping.java index 1250a54f8a..0b29966dee 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/SimpleIdMapping.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/SimpleIdMapping.java @@ -1,32 +1,28 @@ package com.bakdata.conquery.models.config; +import java.util.List; + import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.identifiable.mapping.CsvEntityId; -import com.bakdata.conquery.models.identifiable.mapping.DefaultIdMappingAccessor; -import com.bakdata.conquery.models.identifiable.mapping.IdMappingAccessor; import com.bakdata.conquery.models.identifiable.mapping.IdMappingConfig; -import org.apache.commons.lang3.ArrayUtils; - -import java.util.List; - +import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; +import com.univocity.parsers.common.record.Record; + +/** + * Identity mapping. + * + * TODO should this actually map to a separate column? + */ @CPSType(base = IdMappingConfig.class, id = "SIMPLE") public class SimpleIdMapping extends IdMappingConfig { @Override - public IdMappingAccessor[] getIdAccessors() { - return new IdMappingAccessor[]{ - new DefaultIdMappingAccessor(new int[]{0}, new String[]{"result"}) { - @Override - public CsvEntityId getFallbackCsvId(String[] reorderedCsvLine) { - return new CsvEntityId(reorderedCsvLine[0]); - } - - @Override - public int findIndexFromMappingHeader(String csvHeaderField) { - return ArrayUtils.indexOf(getHeader(), csvHeaderField); - } - } - }; + protected void processRecord(Record record, CsvEntityId id, EntityIdMap mapping) { + + mapping.addOutputMapping(id, id.getCsvId()); + + mapping.addInputMapping(id, id.getCsvId()); + } @Override @@ -34,9 +30,4 @@ public List getPrintIdFields() { return List.of("result"); } - @Override - public String[] getHeader() { - return new String[]{"id", "result"}; - } - } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java index 4343829324..8170c77fdc 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java @@ -24,7 +24,7 @@ import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.forms.configs.FormConfig; import com.bakdata.conquery.models.identifiable.CentralRegistry; -import com.bakdata.conquery.models.identifiable.mapping.PersistentIdMap; +import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.SingletonNamespaceCollection; import com.bakdata.conquery.models.worker.WorkerInformation; @@ -56,7 +56,7 @@ public interface StoreFactory { SingletonStore createWorkerInformationStore(String pathName); // NamespaceStorage - SingletonStore createIdMappingStore(String pathName); + SingletonStore createIdMappingStore(String pathName); SingletonStore createWorkerToBucketsStore(String pathName); SingletonStore createStructureStore(String pathName, SingletonNamespaceCollection centralRegistry); diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java index c93107f92d..8d823fdc62 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java @@ -53,7 +53,7 @@ import com.bakdata.conquery.models.forms.configs.FormConfig; import com.bakdata.conquery.models.identifiable.CentralRegistry; import com.bakdata.conquery.models.identifiable.ids.IId; -import com.bakdata.conquery.models.identifiable.mapping.PersistentIdMap; +import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.SingletonNamespaceCollection; import com.bakdata.conquery.models.worker.WorkerInformation; @@ -292,11 +292,11 @@ public SingletonStore createWorkerInformationStore(String pat } @Override - public SingletonStore createIdMappingStore(String pathName) { + public SingletonStore createIdMappingStore(String pathName) { final Environment environment = findEnvironment(pathName); synchronized (openStoresInEnv) { - final BigStore bigStore = + final BigStore bigStore = new BigStore<>(this, validator, environment, ID_MAPPING, openStoresInEnv.get(environment), this::closeEnvironment, this::removeEnvironment, objectMapper); return new SingletonStore<>(new CachedStore<>(bigStore)); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/Table.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/Table.java index 2e93835bfb..4bb7a5c864 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/Table.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/Table.java @@ -4,6 +4,7 @@ import java.util.Set; import java.util.stream.Stream; +import javax.annotation.PostConstruct; import javax.validation.Valid; import javax.validation.constraints.NotNull; @@ -33,6 +34,13 @@ public class Table extends Labeled implements NamespacedIdentifiable[] groups() default { }; - - Class[] payload() default { }; - - @Target({ FIELD }) - @Retention(RetentionPolicy.RUNTIME) - @Documented - @interface ValidCSVFormatList { - ValidCSVFormat[] value(); - } - - public static class ValidCSVFormatValidator implements ConstraintValidator> { - - @Override - public void initialize(ValidCSVFormat anno) {} - - @Override - public boolean isValid(List value, ConstraintValidatorContext context) { - context.disableDefaultConstraintViolation(); - if(value!=null) { - boolean correct = true; - if(value.stream().anyMatch(Objects::isNull)) { - context - .buildConstraintViolationWithTemplate("Null is not allowed as part of the format") - .addConstraintViolation(); - correct = false; - } - for(FormatColumn fc : FormatColumn.values()) { - if(!fc.isDuplicatesAllowed() && value.stream().filter(v->fc.equals(v)).count() > 1) { - context - .buildConstraintViolationWithTemplate("The format column "+fc+" is may not appear more than once") - .addConstraintViolation(); - correct = false; - } - } - if(!value.contains(FormatColumn.ID)) { - context - .buildConstraintViolationWithTemplate("The format does not contain "+FormatColumn.ID) - .addConstraintViolation(); - correct = false; - } - long dateTypes = - value.stream().map(FormatColumn::getDateFormat).filter(Objects::nonNull).distinct().count(); - if(dateTypes > 1) { - context - .buildConstraintViolationWithTemplate("The format may not contain more than one of DATE_RANGE, EVENT_DATE and (START_DATE and/or END_DATE)") - .addConstraintViolation(); - correct = false; - } - - - return correct; - } - return true; - } - - } -} \ No newline at end of file diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/CsvEntityId.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/CsvEntityId.java index d1ebcd02ed..e3afd02a6a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/CsvEntityId.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/CsvEntityId.java @@ -11,7 +11,7 @@ * This class holds the csvId of an Entity. */ @Data @RequiredArgsConstructor(onConstructor_=@JsonCreator) -public class CsvEntityId implements EntityId { +public class CsvEntityId { /** * The csvId of an entity. diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/DefaultIdAccessorImpl.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/DefaultIdAccessorImpl.java deleted file mode 100644 index f943449a88..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/DefaultIdAccessorImpl.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.bakdata.conquery.models.identifiable.mapping; - -import lombok.RequiredArgsConstructor; - -/** - * This class is used as an IdAccessorImpl whenever we fail to get a proper configured one. - * * - */ -@RequiredArgsConstructor -public enum DefaultIdAccessorImpl implements IdAccessor { - - INSTANCE; - - @Override - public CsvEntityId getCsvEntityId(String[] csvLine) { - return new CsvEntityId(csvLine[0]); - } -} diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/DefaultIdMappingAccessor.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/DefaultIdMappingAccessor.java deleted file mode 100644 index d744de430a..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/DefaultIdMappingAccessor.java +++ /dev/null @@ -1,38 +0,0 @@ -package com.bakdata.conquery.models.identifiable.mapping; - -import lombok.Getter; -import lombok.ToString; - -@ToString -public abstract class DefaultIdMappingAccessor implements IdMappingAccessor { - - @Getter - protected final int[] idsUsed; - @Getter - protected final String[] header; - - /** - * Select values from array in order by index. - * @param values The input values to select from - * @param indices The indices of the values to select and the order to select them in. - */ - public static String[] select(String[] values, int[] indices){ - String[] out = new String[indices.length]; - - for (int index = 0; index < indices.length; index++) { - out[index] = values[indices[index]]; - } - - return out; - } - - public DefaultIdMappingAccessor(int[] idsUsed, String[] header) { - this.idsUsed = idsUsed; - this.header = header; - } - - @Override - public String[] extract(String[] dataLine) { - return select(dataLine, idsUsed); - } -} diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityId.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityId.java deleted file mode 100644 index b3af753f28..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityId.java +++ /dev/null @@ -1,5 +0,0 @@ -package com.bakdata.conquery.models.identifiable.mapping; - -public interface EntityId { - -} diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java new file mode 100644 index 0000000000..fdfd432801 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java @@ -0,0 +1,72 @@ +package com.bakdata.conquery.models.identifiable.mapping; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +import com.bakdata.conquery.io.jackson.serializer.PersistentIdMapDeserializer; +import com.bakdata.conquery.io.jackson.serializer.PersistentIdMapSerializer; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +/** + * Mapping from Csv Entity Id to External Entity Id and back from the + * combinations of Accessor + IDs to the Entity Id. + */ +@Getter +@EqualsAndHashCode +@RequiredArgsConstructor +@Slf4j +@JsonSerialize(using = PersistentIdMapSerializer.class) +@JsonDeserialize(using = PersistentIdMapDeserializer.class) +public class EntityIdMap { + + + /** + * The map from csv entity ids to external entity ids. + */ + private final Map csvIdToExternalIdMap = new HashMap<>(); + + /** + * The map from external entity ids to csv entity ids. + */ + private final Map externalIdPartCsvIdMap = new HashMap<>(); + + /** + * Map an internal id to an external. + */ + public EntityPrintId toExternal(CsvEntityId internal) { + return csvIdToExternalIdMap.get(internal); + } + + /** + * Map an external to an internal id. + */ + public Optional toInternal(UnresolvedEntityId external) { + return Optional.ofNullable(externalIdPartCsvIdMap.get(external)); + } + + public void addOutputMapping(CsvEntityId csvEntityId, String... parts) { + EntityPrintId externalEntityId = new EntityPrintId(parts); + final EntityPrintId prior = csvIdToExternalIdMap.put(csvEntityId, externalEntityId); + + if (prior != null && prior.equals(externalEntityId)) { + log.warn("Duplicate mapping for {} to {} and {}", csvEntityId, externalEntityId, prior); + } + } + + public void addInputMapping(CsvEntityId csvEntityId, String part) { + UnresolvedEntityId externalEntityId = new UnresolvedEntityId(part); + + final CsvEntityId prior = externalIdPartCsvIdMap.put(externalEntityId, csvEntityId); + + if (prior != null && prior.equals(csvEntityId)) { + log.warn("Duplicate mapping for {} to {} and {}", externalEntityId, csvEntityId, prior); + } + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/ExternalEntityId.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityPrintId.java similarity index 63% rename from backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/ExternalEntityId.java rename to backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityPrintId.java index e75b8f72d1..e24df607bd 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/ExternalEntityId.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityPrintId.java @@ -12,7 +12,7 @@ */ @Data @RequiredArgsConstructor(onConstructor_ = @JsonCreator) -public class ExternalEntityId implements EntityId, Comparable { +public class EntityPrintId implements Comparable { /** * The external Entity Id. @@ -25,12 +25,12 @@ public class ExternalEntityId implements EntityId, Comparable * @param csvEntityId the given csvEntityId. * @return the casted ExternalEntityId. */ - public static ExternalEntityId from(CsvEntityId csvEntityId) { - return new ExternalEntityId(new String[] { csvEntityId.getCsvId() }); + public static EntityPrintId from(CsvEntityId csvEntityId) { + return new EntityPrintId(new String[] {csvEntityId.getCsvId() }); } @Override - public int compareTo(ExternalEntityId o) { - return Arrays.compare(externalId, o.externalId); + public int compareTo(EntityPrintId o) { + return Arrays.compare(getExternalId(), o.getExternalId()); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdAccessor.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdAccessor.java deleted file mode 100644 index 1d5a4ca20f..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdAccessor.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.bakdata.conquery.models.identifiable.mapping; - -/** - * A combination of together unique csv identifiers. Used for mapping parts of external Ids to csvId - * - */ -public interface IdAccessor { - /** - * Maps a line from a Csv to a CsvEntityId. - * @param csvLine A filtered line from a given Csv. - * @return the corresponding CsvEntityId. - */ - CsvEntityId getCsvEntityId(String[] csvLine); -} diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdAccessorImpl.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdAccessorImpl.java deleted file mode 100644 index 891189d93e..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdAccessorImpl.java +++ /dev/null @@ -1,79 +0,0 @@ -package com.bakdata.conquery.models.identifiable.mapping; - -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; - -import com.bakdata.conquery.apiv1.query.concept.specific.CQExternal; -import lombok.Getter; -import lombok.RequiredArgsConstructor; -import org.apache.commons.lang3.ArrayUtils; - -/** - * The standard Implementation for an IdAccessor. - */ -@RequiredArgsConstructor -public class IdAccessorImpl implements IdAccessor { - - /** - * The corresponding accessor. - */ - @Getter - private final IdMappingAccessor accessor; - - /** - * The mapping from column indices in the given CSV to the original uploaded csv. - */ - @Getter - private final int[] applicationMapping; - - /** - * The used namespace storage. - */ - @Getter - private final PersistentIdMap idMapping; - - /** - * removes all non Id Fields from a given CSV Line respective to the given formal Columns. - * @param csvLine A line from the csv. - * @param formatColumns The format description for the given Csv.. - * @return The filtered csvLine. - */ - public static String[] selectIdFields(String[] csvLine, List formatColumns) { - List result = new ArrayList<>(); - for (int i = 0; i < csvLine.length; i++) { - if (formatColumns.get(i) == CQExternal.FormatColumn.ID) { - result.add(csvLine[i]); - } - } - - return result.toArray(ArrayUtils.EMPTY_STRING_ARRAY); - } - - @Override - public CsvEntityId getCsvEntityId(String[] csvLine) { - String[] reorderedCsvLine = reorder(csvLine); - - return Optional - .ofNullable(idMapping) - .map(m -> m.toInternal(new SufficientExternalEntityId(reorderedCsvLine))) - // fallback: we join everything relevant together - .orElseGet(()->accessor.getFallbackCsvId(reorderedCsvLine)); - } - - /** - * Reorder takes a line from an uploaded CSV an reorders the id columns, to match the specified format. - * @param csvLine The csv line. - * @return Outputs a csvLine like the one in the original uploaded id mapping CSV - */ - protected String[] reorder(String[] csvLine) { - String[] reorderedCsvLine = new String[accessor.getHeader().length]; - for (int i = 0; i < csvLine.length; i++) { - int indexInHeader = applicationMapping[i]; - if (indexInHeader != -1) { - reorderedCsvLine[indexInHeader] = csvLine[i]; - } - } - return reorderedCsvLine; - } -} diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMapper.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMapper.java deleted file mode 100644 index 98320e597a..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMapper.java +++ /dev/null @@ -1,27 +0,0 @@ -package com.bakdata.conquery.models.identifiable.mapping; - -/** - * Interface for classes that use to map the internal ids to readable, specific or even opaque (for pseudomization) ids and vice versa. - * How internal and external ids look like depends on the individual implementation. - * An important notice is, that multiple external ids can be mapped to a single internal id. - * This is provided through the {@link IdMappingAccessor}s which can filter out id columns which work as "primary keys" but have the same uniqueness - * to the entity as other "primary columns". - */ -public interface IdMapper { - - /** - * Map an internal id to an external. - */ - ExternalEntityId toExternal(CsvEntityId internal); - - /** - * Map an external to an internal id. - */ - CsvEntityId toInternal(SufficientExternalEntityId external); - - /** - * Add a mapping. The implementation looks after maintaining the mapping and inverse mapping. - */ - void addMapping(CsvEntityId internal, ExternalEntityId external, IdMappingAccessor[] idMappingAccessors); - -} diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingAccessor.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingAccessor.java deleted file mode 100644 index 156e0b1bb7..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingAccessor.java +++ /dev/null @@ -1,67 +0,0 @@ -package com.bakdata.conquery.models.identifiable.mapping; - -import java.util.Arrays; -import java.util.List; - -public interface IdMappingAccessor { - - String[] getHeader(); - - /** - * Check whether all Fields used for the mapping by this id mapping accessor are - * present. - * - * @param csvHeader - * List of the header Strings. - * @return whether the mapping can be applied to the header. - */ - default boolean canBeApplied(List csvHeader) { - for (String requiredHeader : getHeader()) { - if (!csvHeader.contains(requiredHeader)) { - return false; - } - } - return true; - } - - /** - * Retrieves an applicationMapping which maps from CsvHeader to IdMappingCsv - * Indices. Assumes that canBeApplied has been checked before and returned True. - * - * @param csvHeader - * Array of the header Strings. - * @param idMapping - * @return The IdAccessor. - */ - default IdAccessor getApplicationMapping(String[] csvHeader, final PersistentIdMap idMapping) { - int[] applicationMapping = new int[csvHeader.length]; - Arrays.fill(applicationMapping, -1); - for (int indexInHeader = 0; indexInHeader < csvHeader.length; indexInHeader++) { - String csvHeaderField = csvHeader[indexInHeader]; - int indexInCsvHeader = findIndexFromMappingHeader(csvHeaderField); - if (indexInCsvHeader != -1) { - applicationMapping[indexInHeader] = indexInCsvHeader; - } - } - return new IdAccessorImpl(this, applicationMapping, idMapping); - } - - /** - * Returns the index of the header in the setted mapping that maps best to the provided header in the CSV. - * @param csvHeaderField The header field in the CSV that is machted to the predefined required headers. - * @return The index of the predefined header that matched best. - */ - int findIndexFromMappingHeader(String csvHeaderField); - - /** - * Extracts the Id information from a CSV line using this accessor - * configuration. - * - * @param dataLine - * A Line from a CSV. - * @return the dataLine without the unused fields. - */ - String[] extract(String[] dataLine); - - CsvEntityId getFallbackCsvId(String[] reorderedCsvLine); -} diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingConfig.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingConfig.java index 6c9c25e871..9ec2055df8 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingConfig.java @@ -1,21 +1,17 @@ package com.bakdata.conquery.models.identifiable.mapping; import java.util.*; -import java.util.function.Function; import com.bakdata.conquery.io.cps.CPSBase; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.execution.ManagedExecution; -import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.worker.Namespace; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import lombok.NonNull; +import com.univocity.parsers.common.record.Record; +import com.univocity.parsers.csv.CsvParser; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.ListUtils; -import org.apache.commons.lang3.ArrayUtils; -import org.apache.commons.lang3.StringUtils; @Slf4j @@ -24,41 +20,31 @@ @JsonTypeInfo(use = JsonTypeInfo.Id.CUSTOM, property = "type") public abstract class IdMappingConfig { - public PersistentIdMap generateIdMapping(Iterator csvIterator) throws IllegalArgumentException { + public EntityIdMap generateIdMapping(CsvParser parser) throws IllegalArgumentException { - PersistentIdMap mapping = new PersistentIdMap(); + EntityIdMap mapping = new EntityIdMap(); - if (!Arrays.equals(this.getHeader(), csvIterator.next(), StringUtils::compareIgnoreCase)) { - throw new IllegalArgumentException("The uploaded CSVs Header does not match the expected"); - } + //TODO Check headers match parser.getContext().headers(); - // first column is the external key, the rest is part of the csv id - csvIterator.forEachRemaining( - (s) -> mapping.addMapping(new CsvEntityId(s[0]), new ExternalEntityId(Arrays.copyOfRange(s, 1, s.length)), getIdAccessors())); + Record record; - mapping.checkIntegrity(Arrays.asList(getIdAccessors())); + while((record = parser.parseNextRecord()) != null){ + final String id = record.getString("id"); - return mapping; - } + final CsvEntityId csvEntityId = new CsvEntityId(id); - @JsonIgnore - public int getHeaderSize() { - return getHeader().length; - } + processRecord(record, csvEntityId, mapping); + } - @JsonIgnore - public abstract IdMappingAccessor[] getIdAccessors(); + //TODO mapping.checkIntegrity(Arrays.asList(getIdAccessors())); - @JsonIgnore - public List getPrintIdFields() { - return List.of(ArrayUtils.subarray(getHeader(), 1, getHeaderSize())); + return mapping; } - /** - * Header of the Mapping-CSV file. - */ + protected abstract void processRecord(Record record, CsvEntityId id, EntityIdMap mapping); + @JsonIgnore - public abstract String[] getHeader(); + public abstract List getPrintIdFields(); /** * Is called once before a mapping is used before a query result is created to @@ -72,29 +58,20 @@ public IdMappingState initToExternal(User user, ManagedExecution execution) { /** * Converts the internal ID to the an external. */ - public ExternalEntityId toExternal(CsvEntityId csvEntityId, Namespace namespace, IdMappingState state) { + public EntityPrintId toExternal(CsvEntityId csvEntityId, Namespace namespace, IdMappingState state, EntityIdMap mapping) { // The state may be uses by implementations of this class - PersistentIdMap mapping = namespace.getStorage().getIdMapping(); - if (mapping != null) { - ExternalEntityId externalEntityId = mapping.toExternal(csvEntityId); - if (externalEntityId != null) { - return externalEntityId; - } + + if (mapping == null) { + return EntityPrintId.from(csvEntityId); } - return ExternalEntityId.from(csvEntityId); - } - @NonNull - public IdAccessor mappingFromCsvHeader(String[] csvHeader, PersistentIdMap idMapping) { - for (IdMappingAccessor accessor : getIdAccessors()) { - if (accessor.canBeApplied(Arrays.asList(csvHeader))) { - log.info("Using accessor (with required headers {}) to extract mapping from CSV with the header containing the ID columns: {}", accessor.getHeader(), csvHeader); - return accessor.getApplicationMapping(csvHeader, idMapping); - } + EntityPrintId externalEntityId = mapping.toExternal(csvEntityId); + + if (externalEntityId == null) { + return EntityPrintId.from(csvEntityId); } - log.info("Using the default accessor implementation."); - return DefaultIdAccessorImpl.INSTANCE; - } + return externalEntityId; + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/PersistentIdMap.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/PersistentIdMap.java deleted file mode 100644 index b3fd373843..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/PersistentIdMap.java +++ /dev/null @@ -1,78 +0,0 @@ -package com.bakdata.conquery.models.identifiable.mapping; - -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; - -import com.bakdata.conquery.io.jackson.serializer.PersistentIdMapDeserializer; -import com.bakdata.conquery.io.jackson.serializer.PersistentIdMapSerializer; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.RequiredArgsConstructor; - -/** - * Mapping from Csv Entity Id to External Entity Id and back from the - * combinations of Accessor + IDs to the Entity Id. - */ -@Getter -@EqualsAndHashCode -@RequiredArgsConstructor -@JsonSerialize(using = PersistentIdMapSerializer.class) -@JsonDeserialize(using = PersistentIdMapDeserializer.class) -public class PersistentIdMap implements IdMapper { - - /** - * The map from csv entity ids to external entity ids. - */ - private final Map csvIdToExternalIdMap = new HashMap<>(); - - /** - * The map from external entity ids to csv entity ids. - */ - private final Map externalIdPartCsvIdMap = new HashMap<>(); - - @Override - public ExternalEntityId toExternal(CsvEntityId internal) { - return csvIdToExternalIdMap.get(internal); - } - - @Override - public CsvEntityId toInternal(SufficientExternalEntityId external) { - return externalIdPartCsvIdMap.get(external); - } - - @Override - public void addMapping(CsvEntityId internal, ExternalEntityId external, IdMappingAccessor[] idMappingAccessors) { - // Map internal to external - csvIdToExternalIdMap.put(internal, external); - // Map the inverse with different unique shapes - for (IdMappingAccessor accessor : idMappingAccessors) { - externalIdPartCsvIdMap.put(new SufficientExternalEntityId(accessor.extract(external.getExternalId())), internal); - } - } - - /** - * Checks if the given CsvContent produces unique results in perspective to all - * IdMappingAccessors. - * - * @param data - * Map of CsvEntityId to External Ids as read from the given CSV. - * @throws IllegalArgumentException - * if the inserted Ids are not unique. - */ - public void checkIntegrity(Collection idAccessor) { - // check that each idMappingAccessor leads to at most one tuple - for (IdMappingAccessor idMappingAccessor : idAccessor) { - long distinctSize = csvIdToExternalIdMap.values().stream().map(p -> idMappingAccessor.extract(p.getExternalId())).distinct() - .count(); - // check if we still have the same size as before - if (distinctSize != csvIdToExternalIdMap.size()) { - throw new IllegalArgumentException( - "The inserted IDs are not unique respective to the idMapping Accessor " + idMappingAccessor); - } - } - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/PrintIdMapper.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/PrintIdMapper.java index 8ef5374f89..3f010e8b23 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/PrintIdMapper.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/PrintIdMapper.java @@ -7,5 +7,5 @@ * Maps an internalId to an external representation. */ public interface PrintIdMapper { - ExternalEntityId map(EntityResult entityResult); + EntityPrintId map(EntityResult entityResult); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/SufficientExternalEntityId.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/UnresolvedEntityId.java similarity index 69% rename from backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/SufficientExternalEntityId.java rename to backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/UnresolvedEntityId.java index 9827487da5..3af90118c4 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/SufficientExternalEntityId.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/UnresolvedEntityId.java @@ -6,7 +6,7 @@ @Data @RequiredArgsConstructor(onConstructor_ = @JsonCreator) -public class SufficientExternalEntityId implements EntityId { +public class UnresolvedEntityId { - private final String[] externalIdPart; + private final String externalId; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/DateRangeParser.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/DateRangeParser.java index f364674e51..b123947d37 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/DateRangeParser.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/DateRangeParser.java @@ -38,11 +38,11 @@ public DateRangeParser(ParserConfig config) { } @Override - protected CDateRange parseValue(@Nonnull String value) throws ParsingException { + protected CDateRange parseValue(@Nonnull String value) { return DateRangeParser.parseISORange(value, dateFormats); } - public static CDateRange parseISORange(String value, DateFormats dateFormats) throws ParsingException { + public static CDateRange parseISORange(String value, DateFormats dateFormats) { if (value == null) { return null; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/ExecutionManager.java b/backend/src/main/java/com/bakdata/conquery/models/query/ExecutionManager.java index 2a1d5689d5..a8de15b669 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/ExecutionManager.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/ExecutionManager.java @@ -68,7 +68,15 @@ public ManagedExecution runQuery(DatasetRegistry datasets, QueryDescription q public void execute(DatasetRegistry datasets, ManagedExecution execution, ConqueryConfig config) { // Initialize the query / create subqueries - execution.initExecutable(datasets, config); + try { + execution.initExecutable(datasets, config); + }catch (Exception e){ + log.error("Failed to initialize Query[{}]", execution.getId(), e); + + //TODO we don't want to store completely faulty queries but is that right like this? + datasets.getMetaStorage().removeExecution(execution.getId()); + throw e; + } log.info("Executing Query[{}] in Datasets[{}]", execution.getQueryId(), execution.getRequiredDatasets()); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java b/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java index 4bb0b16137..b9b4896bfc 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java @@ -35,7 +35,7 @@ import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.apiv1.query.SecondaryIdQuery; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; -import com.bakdata.conquery.apiv1.query.concept.specific.CQExternal; +import com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal; import com.bakdata.conquery.apiv1.query.concept.specific.CQReusedQuery; import com.bakdata.conquery.models.query.queryplan.QueryPlan; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java index e5ec8c90dd..1c075c5ad2 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java @@ -15,27 +15,17 @@ import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.Response; -import java.io.IOException; -import java.io.InputStream; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import java.util.Set; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -import javax.annotation.Nullable; -import javax.validation.Validator; -import javax.ws.rs.ForbiddenException; -import javax.ws.rs.WebApplicationException; -import javax.ws.rs.core.Response; - import com.bakdata.conquery.apiv1.FilterSearch; import com.bakdata.conquery.io.jackson.InternalOnly; import com.bakdata.conquery.io.jackson.Jackson; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.io.storage.NamespaceStorage; +import com.bakdata.conquery.models.config.ConqueryConfig; +import com.bakdata.conquery.models.datasets.Column; +import com.bakdata.conquery.models.datasets.Dataset; +import com.bakdata.conquery.models.datasets.Import; +import com.bakdata.conquery.models.datasets.SecondaryIdDescription; +import com.bakdata.conquery.models.datasets.Table; import com.bakdata.conquery.models.datasets.concepts.Concept; import com.bakdata.conquery.models.datasets.concepts.Connector; import com.bakdata.conquery.models.datasets.concepts.StructureNode; @@ -43,29 +33,14 @@ import com.bakdata.conquery.models.datasets.concepts.select.concept.specific.EventDurationSumSelect; import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeConnector; import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; -import com.bakdata.conquery.models.config.ConqueryConfig; -import com.bakdata.conquery.models.datasets.Column; -import com.bakdata.conquery.models.datasets.Dataset; -import com.bakdata.conquery.models.datasets.Import; -import com.bakdata.conquery.models.datasets.SecondaryIdDescription; -import com.bakdata.conquery.models.datasets.Table; -import com.bakdata.conquery.models.exceptions.JSONException; -import com.bakdata.conquery.models.datasets.Column; -import com.bakdata.conquery.models.datasets.Dataset; -import com.bakdata.conquery.models.datasets.Import; -import com.bakdata.conquery.models.datasets.SecondaryIdDescription; -import com.bakdata.conquery.models.datasets.Table; import com.bakdata.conquery.models.exceptions.ValidatorHelper; import com.bakdata.conquery.models.identifiable.IdMutex; import com.bakdata.conquery.models.identifiable.Identifiable; import com.bakdata.conquery.models.identifiable.ids.specific.ConceptId; -import com.bakdata.conquery.models.identifiable.ids.specific.DictionaryId; -import com.bakdata.conquery.models.identifiable.ids.specific.TableId; -import com.bakdata.conquery.models.identifiable.ids.specific.ConceptId; import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; import com.bakdata.conquery.models.identifiable.ids.specific.DictionaryId; import com.bakdata.conquery.models.identifiable.ids.specific.TableId; -import com.bakdata.conquery.models.identifiable.mapping.PersistentIdMap; +import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; import com.bakdata.conquery.models.jobs.ImportJob; import com.bakdata.conquery.models.jobs.JobManager; import com.bakdata.conquery.models.jobs.SimpleJob; @@ -226,10 +201,6 @@ else if (!table.getDataset().equals(dataset)) { ValidatorHelper.failOnError(log, validator.validate(table)); - for (int p = 0; p < table.getColumns().length; p++) { - table.getColumns()[p].setPosition(p); - } - namespace.getStorage().addTable(table); namespace.sendToAll(new UpdateTable(table)); } @@ -281,13 +252,21 @@ public void setIdMapping(InputStream data, Namespace namespace) { log.info("Received IdMapping for Dataset[{}]", namespace.getDataset().getId()); CsvParser parser = config.getCsv() - .withSkipHeader(false) - .withParseHeaders(false) - .createParser(); + .withSkipHeader(false) + .withParseHeaders(true) + .createParser(); + + try { + + parser.beginParsing(data); - PersistentIdMap mapping = config.getIdMapping().generateIdMapping(parser.iterate(data).iterator()); + EntityIdMap mapping = config.getIdMapping().generateIdMapping(parser); + namespace.getStorage().updateIdMapping(mapping); - namespace.getStorage().updateIdMapping(mapping); + } + finally { + parser.stopParsing(); + } } /** diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/DatasetsUIResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/DatasetsUIResource.java index 6ccea17ddb..e0d12a0613 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/DatasetsUIResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/DatasetsUIResource.java @@ -22,7 +22,7 @@ import com.bakdata.conquery.models.datasets.SecondaryIdDescription; import com.bakdata.conquery.models.dictionary.Dictionary; import com.bakdata.conquery.models.identifiable.ids.specific.TableId; -import com.bakdata.conquery.models.identifiable.mapping.PersistentIdMap; +import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.resources.admin.rest.AdminDatasetProcessor; import com.bakdata.conquery.resources.admin.rest.UIProcessor; @@ -113,7 +113,7 @@ imp, getNamespace().getStorage().getAllConcepts() @GET @Path("mapping") public View getIdMapping() { - PersistentIdMap mapping = namespace.getStorage().getIdMapping(); + EntityIdMap mapping = namespace.getStorage().getIdMapping(); if (mapping != null && mapping.getCsvIdToExternalIdMap() != null) { return new UIView<>("idmapping.html.ftl", uiProcessor.getUIContext(), mapping.getCsvIdToExternalIdMap()); } diff --git a/backend/src/main/java/com/bakdata/conquery/util/QueryUtils.java b/backend/src/main/java/com/bakdata/conquery/util/QueryUtils.java index e5f633f4ac..3e8c5366e1 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/QueryUtils.java +++ b/backend/src/main/java/com/bakdata/conquery/util/QueryUtils.java @@ -29,7 +29,7 @@ import com.bakdata.conquery.models.query.NamespacedIdentifiableHolding; import com.bakdata.conquery.apiv1.query.concept.specific.CQAnd; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; -import com.bakdata.conquery.apiv1.query.concept.specific.CQExternal; +import com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal; import com.bakdata.conquery.apiv1.query.concept.specific.CQOr; import com.bakdata.conquery.apiv1.query.concept.specific.CQReusedQuery; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; diff --git a/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java b/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java index cb127e1571..29a663c29e 100644 --- a/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java +++ b/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java @@ -39,7 +39,7 @@ import com.bakdata.conquery.apiv1.query.SecondaryIdQuery; import com.bakdata.conquery.apiv1.query.concept.specific.CQAnd; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; -import com.bakdata.conquery.apiv1.query.concept.specific.CQExternal; +import com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.resources.api.ResultArrowFileResource; import com.bakdata.conquery.resources.api.ResultArrowStreamResource; diff --git a/backend/src/test/java/com/bakdata/conquery/integration/common/IntegrationUtils.java b/backend/src/test/java/com/bakdata/conquery/integration/common/IntegrationUtils.java index cbe2f62991..36cac857eb 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/common/IntegrationUtils.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/common/IntegrationUtils.java @@ -92,6 +92,7 @@ private static JsonNode getRawExecutionStatus(String id, StandaloneSupport conqu // We try at most 5 times, queryStatus waits for 10s, we therefore don't need to timeout here. // Query getQueryStatus until it is no longer running. for (int trial = 0; trial < 5; trial++) { + log.debug("Trying to get Query result"); JsonNode execStatusRaw = conquery.getClient() @@ -129,6 +130,7 @@ public static ManagedExecutionId assertQueryResult(StandaloneSupport conquery, I .getConqueryTokenRealm() .createTokenForUser(user.getId()); + // Submit Query Response response = conquery.getClient() .target(postQueryURI) .request(MediaType.APPLICATION_JSON_TYPE) diff --git a/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java b/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java index f2dd19bb29..bad4ab8d7e 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java @@ -16,16 +16,22 @@ import java.util.Map; import java.util.UUID; +import javax.swing.text.Utilities; +import javax.validation.Valid; +import javax.validation.constraints.NotEmpty; import javax.ws.rs.client.Entity; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import com.bakdata.conquery.ConqueryConstants; +import com.bakdata.conquery.apiv1.query.concept.specific.external.DateColumn; +import com.bakdata.conquery.apiv1.query.concept.specific.external.IdColumn; import com.bakdata.conquery.integration.json.ConqueryTestSpec; import com.bakdata.conquery.io.jackson.Jackson; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.auth.permissions.AbilitySets; import com.bakdata.conquery.models.auth.permissions.ExecutionPermission; +import com.bakdata.conquery.models.datasets.Table; import com.bakdata.conquery.models.datasets.concepts.Concept; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.datasets.SecondaryIdDescription; @@ -37,8 +43,7 @@ import com.bakdata.conquery.models.preproc.outputs.OutputDescription; import com.bakdata.conquery.apiv1.query.IQuery; import com.bakdata.conquery.apiv1.query.ConceptQuery; -import com.bakdata.conquery.apiv1.query.concept.specific.CQExternal; -import com.bakdata.conquery.apiv1.query.concept.specific.CQExternal.FormatColumn; +import com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal; import com.bakdata.conquery.models.worker.SingletonNamespaceCollection; import com.bakdata.conquery.resources.ResourceConstants; import com.bakdata.conquery.resources.admin.rest.AdminDatasetResource; @@ -70,7 +75,7 @@ public static void importPreviousQueries(StandaloneSupport support, RequiredData final CsvParser parser = support.getConfig().getCsv().withParseHeaders(false).withSkipHeader(false).createParser(); String[][] data = parser.parseAll(queryResults.stream()).toArray(new String[0][]); - ConceptQuery q = new ConceptQuery(new CQExternal(Arrays.asList(FormatColumn.ID, FormatColumn.DATE_SET), data)); + ConceptQuery q = new ConceptQuery(new CQExternal(Arrays.asList(new IdColumn(), new DateColumn.DateSet()), data)); ManagedExecution managed = support.getNamespace().getExecutionManager().createQuery(support.getNamespace().getNamespaces(),q, queryId, user, support.getNamespace().getDataset()); user.addPermission(support.getMetaStorage(), ExecutionPermission.onInstance(AbilitySets.QUERY_CREATOR, managed.getId())); @@ -100,10 +105,11 @@ public static void importPreviousQueries(StandaloneSupport support, RequiredData } } - public static void importTables(StandaloneSupport support, RequiredData content) throws JSONException { + public static void importTables(StandaloneSupport support, List tables) throws JSONException { - for (RequiredTable rTable : content.getTables()) { - support.getDatasetsProcessor().addTable(rTable.toTable(support.getDataset(), support.getNamespace().getStorage().getCentralRegistry()), support.getNamespace()); + for (RequiredTable rTable : tables) { + final Table table = rTable.toTable(support.getDataset(), support.getNamespace().getStorage().getCentralRegistry()); + support.getDatasetsProcessor().addTable(table, support.getNamespace()); } } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/common/RequiredTable.java b/backend/src/test/java/com/bakdata/conquery/integration/common/RequiredTable.java index 36fd4db7f1..6a75635f91 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/common/RequiredTable.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/common/RequiredTable.java @@ -42,6 +42,9 @@ public Table toTable(Dataset dataset, CentralRegistry centralRegistry) { table.setName(name); table.setColumns(Arrays.stream(columns) .map(col -> col.toColumn(table, centralRegistry)).toArray(Column[]::new)); + + table.init(); + return table; } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java b/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java index 16e12f6815..4e9e780ae5 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java @@ -9,13 +9,18 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import com.bakdata.conquery.integration.common.IntegrationUtils; +import com.bakdata.conquery.integration.common.LoadingUtil; import com.bakdata.conquery.integration.common.ResourceFile; import com.bakdata.conquery.io.result.CsvLineStreamRenderer; +import com.bakdata.conquery.io.result.ResultTestUtil; import com.bakdata.conquery.io.result.ResultUtil; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.execution.ExecutionState; +import com.bakdata.conquery.models.execution.ManagedExecution; +import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.identifiable.mapping.IdMappingState; import com.bakdata.conquery.apiv1.query.IQuery; import com.bakdata.conquery.models.query.ManagedQuery; @@ -54,30 +59,22 @@ public void executeTest(StandaloneSupport standaloneSupport) throws IOException final ConqueryConfig config = standaloneSupport.getConfig(); final User testUser = standaloneSupport.getTestUser(); - ManagedQuery managed = (ManagedQuery) standaloneSupport.getNamespace().getExecutionManager().runQuery(namespaces, query, testUser, dataset, config); - managed.awaitDone(10, TimeUnit.SECONDS); - while (managed.getState() != ExecutionState.DONE && managed.getState() != ExecutionState.FAILED) { - log.warn("waiting for more than 10 seconds on " + getLabel()); - managed.awaitDone(1, TimeUnit.DAYS); - } + final ManagedExecutionId executionId = IntegrationUtils.assertQueryResult(standaloneSupport, query, -1, ExecutionState.DONE, testUser, 201); - if (managed.getState() == ExecutionState.FAILED) { - log.error("Failure in Query[{}]. The error was: {}", managed.getId(), managed.getError()); - fail("Query failed (see above)"); - } + final ManagedQuery execution = (ManagedQuery) standaloneSupport.getMetaStorage().getExecution(executionId); //check result info size - List resultInfos = managed.getResultInfo(); + List resultInfos = execution.getResultInfo(); assertThat( - managed.streamResults() + execution.streamResults() .flatMap(EntityResult::streamValues) ) .as("Should have same size as result infos") .allSatisfy(v -> assertThat(v).hasSameSizeAs(resultInfos)); - IdMappingState mappingState = config.getIdMapping().initToExternal(testUser, managed); + IdMappingState mappingState = config.getIdMapping().initToExternal(testUser, execution); PrintSettings PRINT_SETTINGS = new PrintSettings( @@ -94,7 +91,7 @@ public void executeTest(StandaloneSupport standaloneSupport) throws IOException List actual = renderer.toStream( config.getIdMapping().getPrintIdFields(), resultInfos, - managed.streamResults() + execution.streamResults() ).collect(Collectors.toList()); ResourceFile expectedCsv = getExpectedCsv(); @@ -103,8 +100,8 @@ public void executeTest(StandaloneSupport standaloneSupport) throws IOException assertThat(actual).as("Results for %s are not as expected.", this).containsExactlyInAnyOrderElementsOf(expected); // check that getLastResultCount returns the correct size - if (managed.streamResults().noneMatch(MultilineEntityResult.class::isInstance)) { - assertThat(managed.getLastResultCount()).as("Result count for %s is not as expected.", this).isEqualTo(expected.size() - 1); + if (execution.streamResults().noneMatch(MultilineEntityResult.class::isInstance)) { + assertThat(execution.getLastResultCount()).as("Result count for %s is not as expected.", this).isEqualTo(expected.size() - 1); } log.info("INTEGRATION TEST SUCCESSFUL {} {} on {} rows", getClass().getSimpleName(), this, expected.size()); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java b/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java index d544db4deb..9abeee7fee 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java @@ -87,7 +87,7 @@ public void importRequiredData(StandaloneSupport support) throws Exception { importSecondaryIds(support, content.getSecondaryIds()); support.waitUntilWorkDone(); - LoadingUtil.importTables(support, content); + LoadingUtil.importTables(support, content.getTables()); support.waitUntilWorkDone(); log.info("{} IMPORT TABLES", getLabel()); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/QueryTest.java b/backend/src/test/java/com/bakdata/conquery/integration/json/QueryTest.java index d559b7e841..359e5253ae 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/QueryTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/QueryTest.java @@ -18,6 +18,7 @@ import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; +import org.jvnet.hk2.internal.Utilities; @Slf4j @Getter @@ -50,7 +51,7 @@ public void importRequiredData(StandaloneSupport support) throws Exception { importSecondaryIds(support, content.getSecondaryIds()); support.waitUntilWorkDone(); - importTables(support, content); + importTables(support, content.getTables()); support.waitUntilWorkDone(); importConcepts(support, rawConcepts); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/filter/FilterTest.java b/backend/src/test/java/com/bakdata/conquery/integration/json/filter/FilterTest.java index 88144f7c17..de5bfbf5e3 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/filter/FilterTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/filter/FilterTest.java @@ -8,7 +8,6 @@ import com.bakdata.conquery.integration.common.LoadingUtil; import com.bakdata.conquery.integration.common.RequiredData; -import com.bakdata.conquery.integration.common.RequiredTable; import com.bakdata.conquery.integration.common.ResourceFile; import com.bakdata.conquery.integration.json.AbstractQueryEngineTest; import com.bakdata.conquery.integration.json.ConqueryTestSpec; @@ -74,7 +73,7 @@ public void importRequiredData(StandaloneSupport support) throws Exception { content = parseSubTree(support, rawContent, RequiredData.class); - importTables(support); + LoadingUtil.importTables(support, content.getTables()); support.waitUntilWorkDone(); @@ -145,10 +144,6 @@ public IQuery getQuery() { } private void importTables(StandaloneSupport support) throws JSONException { - Dataset dataset = support.getDataset(); - - for (RequiredTable rTable : content.getTables()) { - support.getDatasetsProcessor().addTable(rTable.toTable(support.getDataset(), support.getNamespace().getStorage().getCentralRegistry()), support.getNamespace()); - } + LoadingUtil.importTables(support, content.getTables()); } } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/ConceptPermissionTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/ConceptPermissionTest.java index 8bfe254998..03804982cf 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/ConceptPermissionTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/ConceptPermissionTest.java @@ -42,7 +42,7 @@ public void execute(StandaloneSupport conquery) throws Exception { importSecondaryIds(conquery, test.getContent().getSecondaryIds()); conquery.waitUntilWorkDone(); - LoadingUtil.importTables(conquery, test.getContent()); + LoadingUtil.importTables(conquery, test.getContent().getTables()); conquery.waitUntilWorkDone(); LoadingUtil.importConcepts(conquery, test.getRawConcepts()); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/RestartTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/RestartTest.java index 124aed4e24..150393cfba 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/RestartTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/RestartTest.java @@ -17,7 +17,7 @@ import com.bakdata.conquery.models.exceptions.ValidatorHelper; import com.bakdata.conquery.models.identifiable.IdMapSerialisationTest; import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; -import com.bakdata.conquery.models.identifiable.mapping.PersistentIdMap; +import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; import com.bakdata.conquery.resources.admin.rest.AdminDatasetProcessor; import com.bakdata.conquery.resources.admin.rest.AdminProcessor; import com.bakdata.conquery.util.support.StandaloneSupport; @@ -49,7 +49,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { String testJson = In.resource("/tests/query/RESTART_TEST_DATA/SIMPLE_TREECONCEPT_Query.json").withUTF8().readAll(); Validator validator = Validators.newValidator(); - PersistentIdMap persistentIdMap = IdMapSerialisationTest.createTestPersistentMap(); + EntityIdMap entityIdMap = IdMapSerialisationTest.createTestPersistentMap(); ManagerNode manager = testConquery.getStandaloneCommand().getManager(); AdminDatasetProcessor adminDatasetProcessor = manager.getAdmin().getAdminDatasetProcessor(); @@ -69,7 +69,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { // IDMapping Testing NamespaceStorage namespaceStorage = conquery.getNamespaceStorage(); - namespaceStorage.updateIdMapping(persistentIdMap); + namespaceStorage.updateIdMapping(entityIdMap); final Dataset dataset1 = adminDatasetProcessor.addDataset(TEST_DATASET_1); @@ -166,9 +166,9 @@ public void execute(String name, TestConquery testConquery) throws Exception { } - PersistentIdMap persistentIdMapAfterRestart = conquery.getNamespaceStorage() - .getIdMapping(); - assertThat(persistentIdMapAfterRestart).isEqualTo(persistentIdMap); + EntityIdMap entityIdMapAfterRestart = conquery.getNamespaceStorage() + .getIdMapping(); + assertThat(entityIdMapAfterRestart).isEqualTo(entityIdMap); // Cleanup adminDatasetProcessor.deleteDataset(dataset1); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/ReusedQueryTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/ReusedQueryTest.java index 4a0b886735..ccffcbc2a3 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/ReusedQueryTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/ReusedQueryTest.java @@ -69,7 +69,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { importSecondaryIds(conquery, test.getContent().getSecondaryIds()); conquery.waitUntilWorkDone(); - LoadingUtil.importTables(conquery, test.getContent()); + LoadingUtil.importTables(conquery, test.getContent().getTables()); conquery.waitUntilWorkDone(); LoadingUtil.importConcepts(conquery, test.getRawConcepts()); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ConceptUpdateAndDeletionTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ConceptUpdateAndDeletionTest.java index e31a836275..06e000238d 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ConceptUpdateAndDeletionTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ConceptUpdateAndDeletionTest.java @@ -61,7 +61,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { importSecondaryIds(conquery, test.getContent().getSecondaryIds()); conquery.waitUntilWorkDone(); - LoadingUtil.importTables(conquery, test.getContent()); + LoadingUtil.importTables(conquery, test.getContent().getTables()); conquery.waitUntilWorkDone(); LoadingUtil.importConcepts(conquery, test.getRawConcepts()); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/DatasetDeletionTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/DatasetDeletionTest.java index 7e97acb318..7b9c08bf3f 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/DatasetDeletionTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/DatasetDeletionTest.java @@ -50,7 +50,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { importSecondaryIds(conquery, test.getContent().getSecondaryIds()); conquery.waitUntilWorkDone(); - LoadingUtil.importTables(conquery, test.getContent()); + LoadingUtil.importTables(conquery, test.getContent().getTables()); conquery.waitUntilWorkDone(); LoadingUtil.importConcepts(conquery, test.getRawConcepts()); @@ -184,9 +184,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { namespace = storage.getDatasetRegistry().get(dataset.getId()); // only import the deleted import/table - for (RequiredTable table : test.getContent().getTables()) { - conquery2.getDatasetsProcessor().addTable(table.toTable(conquery.getDataset(), conquery2.getNamespace().getStorage().getCentralRegistry()), conquery2.getNamespace()); - } + LoadingUtil.importTables(conquery2,test.getContent().getTables()); assertThat(conquery2.getNamespace().getStorage().getTables()).isNotEmpty(); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ImportDeletionTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ImportDeletionTest.java index 8446a1660f..c61a887be5 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ImportDeletionTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ImportDeletionTest.java @@ -72,7 +72,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { importSecondaryIds(conquery, test.getContent().getSecondaryIds()); conquery.waitUntilWorkDone(); - LoadingUtil.importTables(conquery, test.getContent()); + LoadingUtil.importTables(conquery, test.getContent().getTables()); conquery.waitUntilWorkDone(); LoadingUtil.importConcepts(conquery, test.getRawConcepts()); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/TableDeletionTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/TableDeletionTest.java index 6fcb9f2ba2..9a373120e7 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/TableDeletionTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/TableDeletionTest.java @@ -61,7 +61,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { importSecondaryIds(conquery, test.getContent().getSecondaryIds()); conquery.waitUntilWorkDone(); - LoadingUtil.importTables(conquery, test.getContent()); + LoadingUtil.importTables(conquery, test.getContent().getTables()); conquery.waitUntilWorkDone(); LoadingUtil.importConcepts(conquery, test.getRawConcepts()); @@ -186,9 +186,10 @@ public void execute(String name, TestConquery testConquery) throws Exception { // Load the same import into the same table, with only the deleted import/table { // only import the deleted import/table - conquery.getDatasetsProcessor().addTable(test.getContent().getTables().stream() - .filter(table -> table.getName().equalsIgnoreCase(tableId.getTable())) - .map(requiredTable -> requiredTable.toTable(conquery.getDataset(), conquery.getNamespace().getStorage().getCentralRegistry())).findFirst().get(), conquery.getNamespace()); + LoadingUtil.importTables(conquery,test.getContent().getTables().stream() + .filter(table -> table.getName().equalsIgnoreCase(tableId.getTable())) + .collect(Collectors.toList())); + conquery.waitUntilWorkDone(); LoadingUtil.importTableContents(conquery, test.getContent().getTables().stream() diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/CsvLineStreamRenderer.java b/backend/src/test/java/com/bakdata/conquery/io/result/CsvLineStreamRenderer.java index a02cf27075..20e44448e4 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/CsvLineStreamRenderer.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/CsvLineStreamRenderer.java @@ -6,7 +6,7 @@ import java.util.List; import java.util.stream.Stream; -import com.bakdata.conquery.models.identifiable.mapping.ExternalEntityId; +import com.bakdata.conquery.models.identifiable.mapping.EntityPrintId; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.results.EntityResult; @@ -45,14 +45,14 @@ private Stream createCSVBody(PrintSettings cfg, List infos, } - private Stream createCSVLine(PrintSettings cfg, List infos, Pair idResult) { + private Stream createCSVLine(PrintSettings cfg, List infos, Pair idResult) { return idResult .getValue() .streamValues() .map(result -> print(cfg, idResult.getKey(), infos, result)); } - private String print(PrintSettings cfg, ExternalEntityId entity, List infos, Object[] value) { + private String print(PrintSettings cfg, EntityPrintId entity, List infos, Object[] value) { List result = new ArrayList<>(entity.getExternalId().length + value.length); result.addAll(Arrays.asList(entity.getExternalId())); try { diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java index 688746e39c..c7fee059e5 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java @@ -23,7 +23,7 @@ import com.bakdata.conquery.io.result.ResultTestUtil; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.externalservice.ResultType; -import com.bakdata.conquery.models.identifiable.mapping.ExternalEntityId; +import com.bakdata.conquery.models.identifiable.mapping.EntityPrintId; import com.bakdata.conquery.models.query.ManagedQuery; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; @@ -111,7 +111,7 @@ void writeAndRead() throws IOException { Locale.ROOT, null, CONFIG, - (cer) -> new ExternalEntityId(new String[]{Integer.toString(cer.getEntityId()), Integer.toString(cer.getEntityId())}), + (cer) -> new EntityPrintId(new String[]{Integer.toString(cer.getEntityId()), Integer.toString(cer.getEntityId())}), (selectInfo) -> selectInfo.getSelect().getLabel()); // The Shard nodes send Object[] but since Jackson is used for deserialization, nested collections are always a list because they are not further specialized List results = getTestEntityResults(); diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/csv/CsvResultGenerationTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/csv/CsvResultGenerationTest.java index cc3677a974..8f37f71405 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/csv/CsvResultGenerationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/csv/CsvResultGenerationTest.java @@ -17,7 +17,7 @@ import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.i18n.I18n; -import com.bakdata.conquery.models.identifiable.mapping.ExternalEntityId; +import com.bakdata.conquery.models.identifiable.mapping.EntityPrintId; import com.bakdata.conquery.models.query.ManagedQuery; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; @@ -51,7 +51,7 @@ void writeAndRead() throws IOException { Locale.GERMAN, null, CONFIG, - (cer) -> new ExternalEntityId(new String[]{Integer.toString(cer.getEntityId()), Integer.toString(cer.getEntityId())}), + (cer) -> new EntityPrintId(new String[]{Integer.toString(cer.getEntityId()), Integer.toString(cer.getEntityId())}), (selectInfo) -> selectInfo.getSelect().getLabel()); // The Shard nodes send Object[] but since Jackson is used for deserialization, nested collections are always a list because they are not further specialized List results = getTestEntityResults(); diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java index a308bb5f29..737cad8747 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java @@ -20,7 +20,7 @@ import com.bakdata.conquery.models.config.ExcelConfig; import com.bakdata.conquery.models.externalservice.ResultType; import com.bakdata.conquery.models.i18n.I18n; -import com.bakdata.conquery.models.identifiable.mapping.ExternalEntityId; +import com.bakdata.conquery.models.identifiable.mapping.EntityPrintId; import com.bakdata.conquery.models.query.ManagedQuery; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; @@ -60,7 +60,7 @@ void writeAndRead() throws IOException { Locale.GERMAN, null, CONFIG, - (cer) -> new ExternalEntityId(new String[]{Integer.toString(cer.getEntityId()), Integer.toString(cer.getEntityId())}), + (cer) -> new EntityPrintId(new String[]{Integer.toString(cer.getEntityId()), Integer.toString(cer.getEntityId())}), (selectInfo) -> selectInfo.getSelect().getLabel()); // The Shard nodes send Object[] but since Jackson is used for deserialization, nested collections are always a list because they are not further specialized List results = getTestEntityResults(); diff --git a/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java b/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java index 3c6b1f047a..4582c4f968 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java +++ b/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java @@ -36,7 +36,7 @@ import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; import com.bakdata.conquery.models.identifiable.ids.specific.GroupId; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; -import com.bakdata.conquery.models.identifiable.mapping.PersistentIdMap; +import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; import com.bakdata.conquery.models.query.ManagedQuery; import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; @@ -193,7 +193,7 @@ public void treeConcept() throws IOException, JSONException{ @Test public void persistentIdMap() throws JSONException, IOException { - SerializationTestUtil.forType(PersistentIdMap.class) + SerializationTestUtil.forType(EntityIdMap.class) .test(IdMapSerialisationTest.createTestPersistentMap()); } diff --git a/backend/src/test/java/com/bakdata/conquery/models/execution/DefaultLabelTest.java b/backend/src/test/java/com/bakdata/conquery/models/execution/DefaultLabelTest.java index baaed4afdc..29c35b8775 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/execution/DefaultLabelTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/execution/DefaultLabelTest.java @@ -21,7 +21,7 @@ import com.bakdata.conquery.apiv1.query.ConceptQuery; import com.bakdata.conquery.apiv1.query.concept.specific.CQAnd; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; -import com.bakdata.conquery.apiv1.query.concept.specific.CQExternal; +import com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal; import com.bakdata.conquery.apiv1.query.concept.specific.CQReusedQuery; import com.bakdata.conquery.models.worker.DatasetRegistry; import org.jetbrains.annotations.NotNull; diff --git a/backend/src/test/java/com/bakdata/conquery/models/identifiable/IdMapSerialisationTest.java b/backend/src/test/java/com/bakdata/conquery/models/identifiable/IdMapSerialisationTest.java index d4028aacaa..a07e9cd65d 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/identifiable/IdMapSerialisationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/identifiable/IdMapSerialisationTest.java @@ -1,25 +1,17 @@ package com.bakdata.conquery.models.identifiable; import com.bakdata.conquery.models.identifiable.mapping.CsvEntityId; -import com.bakdata.conquery.models.identifiable.mapping.ExternalEntityId; -import com.bakdata.conquery.models.identifiable.mapping.IdMappingAccessor; -import com.bakdata.conquery.models.identifiable.mapping.IdMappingConfig; -import com.bakdata.conquery.models.identifiable.mapping.PersistentIdMap; -import com.bakdata.conquery.models.config.SimpleIdMapping; +import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; public class IdMapSerialisationTest { - public static PersistentIdMap createTestPersistentMap() { - PersistentIdMap persistentIdMap = new PersistentIdMap(); - IdMappingConfig mapping = new SimpleIdMapping(); - IdMappingAccessor[] accessors = mapping.getIdAccessors(); - - persistentIdMap.addMapping(new CsvEntityId("test1"), new ExternalEntityId(new String[] { "a", "b" }), accessors); - persistentIdMap.addMapping(new CsvEntityId("test2"), new ExternalEntityId(new String[] { "c", "d" }), accessors); - persistentIdMap.addMapping(new CsvEntityId("test3"), new ExternalEntityId(new String[] { "e", "f" }), accessors); - persistentIdMap.addMapping(new CsvEntityId("test4"), new ExternalEntityId(new String[] { "g", "h" }), accessors); + public static EntityIdMap createTestPersistentMap() { + EntityIdMap entityIdMap = new EntityIdMap(); - return persistentIdMap; + entityIdMap.addInputMapping(new CsvEntityId("test1"), "a"); + entityIdMap.addOutputMapping(new CsvEntityId("test2"), "c"); + + return entityIdMap; } } diff --git a/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java b/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java index 52868c9452..4af353f08e 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java +++ b/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java @@ -27,7 +27,7 @@ import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.forms.configs.FormConfig; import com.bakdata.conquery.models.identifiable.CentralRegistry; -import com.bakdata.conquery.models.identifiable.mapping.PersistentIdMap; +import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.SingletonNamespaceCollection; import com.bakdata.conquery.models.worker.WorkerInformation; @@ -94,7 +94,7 @@ public SingletonStore createWorkerInformationStore(String pat } @Override - public SingletonStore createIdMappingStore(String pathName) { + public SingletonStore createIdMappingStore(String pathName) { return ID_MAPPING.singleton(new NonPersistentStore()); } diff --git a/docs/REST API JSONs.md b/docs/REST API JSONs.md index 5c74f8a628..d338604f9c 100644 --- a/docs/REST API JSONs.md +++ b/docs/REST API JSONs.md @@ -514,7 +514,7 @@ Supported Fields:
Details

-Java Type: `com.bakdata.conquery.apiv1.query.concept.specific.CQExternal` +Java Type: `com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal` Supported Fields: From 5c67f6feaf179914a1c9d5a530923bd316aa0aa9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 6 Jul 2021 14:45:19 +0000 Subject: [PATCH 02/82] Update AutoDoc --- docs/Config JSON.md | 6 +++--- docs/REST API JSONs.md | 4 +--- docs/Table JSONs.md | 6 +++--- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/docs/Config JSON.md b/docs/Config JSON.md index bf349c0544..1e37d50489 100644 --- a/docs/Config JSON.md +++ b/docs/Config JSON.md @@ -141,7 +141,7 @@ An `IdMappingConfig` is used to define how multi column entity IDs are printed a Different types of IdMappingConfig can be used by setting `type` to one of the following values: -### NO_ID_MAPPING [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/NoIdMapping.java#L11) +### NO_ID_MAPPING [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/NoIdMapping.java#L12)

Details

@@ -152,8 +152,8 @@ No fields can be set for this type.

-### SIMPLE [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/SimpleIdMapping.java#L12) - +### SIMPLE [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/SimpleIdMapping.java#L11-L15) +Identity mapping. TODO should this actually map to a separate column?
Details

diff --git a/docs/REST API JSONs.md b/docs/REST API JSONs.md index d338604f9c..a5b0f33f4d 100644 --- a/docs/REST API JSONs.md +++ b/docs/REST API JSONs.md @@ -509,7 +509,7 @@ Supported Fields: | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQElement.java#L31-L33) | label | `String` | ? | | Allows the user to define labels. |

-### EXTERNAL [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQExternal.java#L42) +### EXTERNAL [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java#L34)
Details

@@ -521,8 +521,6 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQElement.java#L31-L33) | label | `String` | ? | | Allows the user to define labels. | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQExternal.java#L47) | format | list of one of ID, EVENT_DATE, START_DATE, END_DATE, DATE_RANGE, DATE_SET, IGNORE | ? | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQExternal.java#L51) | values | list of `String` | ? | | |

### NEGATION [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQNegation.java#L23) diff --git a/docs/Table JSONs.md b/docs/Table JSONs.md index 4ebcc5e134..070b05bba8 100644 --- a/docs/Table JSONs.md +++ b/docs/Table JSONs.md @@ -30,7 +30,7 @@ Supported Fields: | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/identifiable/NamedImpl.java#L17) | name | `String` | `null` | | |

-### Type Table [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/datasets/Table.java#L23) +### Type Table [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/datasets/Table.java#L24)
Details

@@ -41,8 +41,8 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/datasets/Table.java#L33) | columns | list of [Column](#Type-Column) | `[]` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/datasets/Table.java#L30) | dataset | ID of `@NonNull Dataset` | `null` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/datasets/Table.java#L34) | columns | list of [Column](#Type-Column) | `[]` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/datasets/Table.java#L31) | dataset | ID of `@NonNull Dataset` | `null` | | | | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/identifiable/Labeled.java#L25-L29) | label | `String` | `null` | "someLabel" | shown in the frontend | | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/identifiable/NamedImpl.java#L17) | name | `String` | `null` | | |

From d3bac28b2d1b9486bfdade0fa3c062ea08a64de1 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Wed, 7 Jul 2021 11:14:47 +0200 Subject: [PATCH 03/82] remove unnecessary helper classes --- .../concept/specific/external/CQExternal.java | 30 ++++----------- .../concept/specific/external/IdColumn.java | 6 +-- .../PersistentIdMapDeserializer.java | 38 ------------------- .../serializer/PersistentIdMapSerializer.java | 37 ------------------ .../conquery/io/result/ResultUtil.java | 26 ++++++------- .../conquery/models/config/NoIdMapping.java | 3 +- .../models/config/SimpleIdMapping.java | 12 +++--- .../identifiable/mapping/CsvEntityId.java | 21 ---------- .../identifiable/mapping/EntityIdMap.java | 22 +++++------ .../identifiable/mapping/EntityPrintId.java | 19 +++++----- .../identifiable/mapping/IdMappingConfig.java | 6 +-- .../arrow/ArrowResultGenerationTest.java | 2 +- .../result/csv/CsvResultGenerationTest.java | 2 +- .../result/excel/ExcelResultRenderTest.java | 2 +- .../identifiable/IdMapSerialisationTest.java | 7 ++-- 15 files changed, 58 insertions(+), 175 deletions(-) delete mode 100644 backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/PersistentIdMapDeserializer.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/PersistentIdMapSerializer.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/CsvEntityId.java diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java index 094f6f026f..feb4bc4255 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java @@ -14,7 +14,6 @@ import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.dictionary.EncodedDictionary; import com.bakdata.conquery.models.error.ConqueryError; -import com.bakdata.conquery.models.identifiable.mapping.CsvEntityId; import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; import com.bakdata.conquery.models.identifiable.mapping.UnresolvedEntityId; import com.bakdata.conquery.models.query.QueryPlanContext; @@ -82,10 +81,10 @@ public void resolve(QueryResolveContext context) { final DateFormat dateFormat = dateColumns.length > 0 ? dateColumns[0].getFormat() : DateFormat.ALL; - final IdColumn idColumn = this.format.stream() - .filter(IdColumn.class::isInstance) - .map(IdColumn.class::cast) - .collect(MoreCollectors.onlyElement()); + final IdColumn idColumn = format.stream() + .filter(IdColumn.class::isInstance) + .map(IdColumn.class::cast) + .collect(MoreCollectors.onlyElement()); final EncodedDictionary primary = context.getNamespace().getStorage().getPrimaryDictionary(); @@ -99,7 +98,7 @@ public void resolve(QueryResolveContext context) { // ignore the first row, because this is the header for (int i = 1; i < values.length; i++) { final String[] row = values[i]; - final UnresolvedEntityId externalId = idColumn.read(row); + final String externalId = idColumn.read(row); //read the dates from the row @@ -107,11 +106,11 @@ public void resolve(QueryResolveContext context) { CDateSet dates = dateFormat.readDates(dateColumns, row, dateFormats); - Optional id = mapping.toInternal(externalId); + Optional id = mapping.toInternal(externalId); int resolvedId; - if (id.isPresent() && (resolvedId = primary.getId(id.get().getCsvId())) != -1) { + if (id.isPresent() && (resolvedId = primary.getId(id.get())) != -1) { valuesResolved.put(resolvedId, dates); } else { @@ -140,21 +139,6 @@ public void resolve(QueryResolveContext context) { public void collectResultInfos(ResultInfoCollector collector) { } - // @RequiredArgsConstructor - // @Getter - // public enum FormatColumn { - // ID(true, null), - // EVENT_DATE(false, DateFormat.EVENT_DATE), - // START_DATE(false, DateFormat.START_END_DATE), - // END_DATE(false, DateFormat.START_END_DATE), - // DATE_RANGE(false, DateFormat.DATE_RANGE), - // DATE_SET(false, DateFormat.DATE_SET), - // IGNORE(true, null); - // - // private final boolean duplicatesAllowed; - // private final DateFormat dateFormat; - // } - @JsonIgnore @ValidationMethod(message = "Values and Format are not of same width.") diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/IdColumn.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/IdColumn.java index 5531120f7e..6572656e11 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/IdColumn.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/IdColumn.java @@ -1,13 +1,11 @@ package com.bakdata.conquery.apiv1.query.concept.specific.external; import com.bakdata.conquery.io.cps.CPSType; -import com.bakdata.conquery.models.identifiable.mapping.UnresolvedEntityId; @CPSType(id = "ID", base = FormatColumn.class) public class IdColumn extends FormatColumn { - - public UnresolvedEntityId read(String[] row) { - return new UnresolvedEntityId(row[getPosition()]); + public String read(String[] row) { + return row[getPosition()]; } } diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/PersistentIdMapDeserializer.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/PersistentIdMapDeserializer.java deleted file mode 100644 index 69cee6fe60..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/PersistentIdMapDeserializer.java +++ /dev/null @@ -1,38 +0,0 @@ -package com.bakdata.conquery.io.jackson.serializer; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import com.bakdata.conquery.models.identifiable.mapping.CsvEntityId; -import com.bakdata.conquery.models.identifiable.mapping.EntityPrintId; -import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; -import com.bakdata.conquery.models.identifiable.mapping.UnresolvedEntityId; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonDeserializer; - -public class PersistentIdMapDeserializer extends JsonDeserializer { - - private static final TypeReference arrayOfMapEntryType = new TypeReference>() {}; - private static final TypeReference mapOfCsvToExternalIdType = new TypeReference>() {}; - - @Override - public EntityIdMap deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { - - Map csvIdToExternalIdMap = p.readValueAs(mapOfCsvToExternalIdType); - Map externalIdPartCsvIdMap = new HashMap<>(); - - List mapAsList = p.readValueAs(arrayOfMapEntryType); - mapAsList.forEach(externalIdMapEntry -> { - externalIdPartCsvIdMap.put(externalIdMapEntry.getSufficientExternalEntityId(), externalIdMapEntry.getCsvEntityId()); - }); - EntityIdMap map = new EntityIdMap(); - map.getCsvIdToExternalIdMap().putAll(csvIdToExternalIdMap); - map.getExternalIdPartCsvIdMap().putAll(externalIdPartCsvIdMap); - return map; - } -} diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/PersistentIdMapSerializer.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/PersistentIdMapSerializer.java deleted file mode 100644 index 7af582ed73..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/PersistentIdMapSerializer.java +++ /dev/null @@ -1,37 +0,0 @@ -package com.bakdata.conquery.io.jackson.serializer; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import com.bakdata.conquery.models.identifiable.mapping.CsvEntityId; -import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; -import com.bakdata.conquery.models.identifiable.mapping.UnresolvedEntityId; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.JsonSerializer; -import com.fasterxml.jackson.databind.SerializerProvider; -import lombok.Data; - -public class PersistentIdMapSerializer extends JsonSerializer { - - @Override - public void serialize(EntityIdMap value, JsonGenerator gen, SerializerProvider provider) throws IOException { - gen.writeObject(value.getCsvIdToExternalIdMap()); - List mapAsList = new ArrayList<>(); - - for (Map.Entry entry : value.getExternalIdPartCsvIdMap().entrySet()) { - UnresolvedEntityId key = entry.getKey(); - CsvEntityId val = entry.getValue(); - mapAsList.add(new ExternalIdMapEntry(key, val)); - } - - gen.writeObject(mapAsList); - } - - @Data - public static class ExternalIdMapEntry { - private final UnresolvedEntityId sufficientExternalEntityId; - private final CsvEntityId csvEntityId; - } -} diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/ResultUtil.java b/backend/src/main/java/com/bakdata/conquery/io/result/ResultUtil.java index 2c46d37e8e..cc0b805c12 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/ResultUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/ResultUtil.java @@ -9,11 +9,10 @@ import com.bakdata.conquery.models.dictionary.EncodedDictionary; import com.bakdata.conquery.models.execution.ManagedExecution; -import com.bakdata.conquery.models.identifiable.mapping.CsvEntityId; +import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; import com.bakdata.conquery.models.identifiable.mapping.EntityPrintId; import com.bakdata.conquery.models.identifiable.mapping.IdMappingConfig; import com.bakdata.conquery.models.identifiable.mapping.IdMappingState; -import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; import com.bakdata.conquery.models.query.SingleTableResult; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.worker.Namespace; @@ -24,24 +23,23 @@ @Slf4j public class ResultUtil { - + public static EntityPrintId createId(Namespace namespace, EntityResult cer, IdMappingConfig idMappingConfig, IdMappingState mappingState) { EncodedDictionary dict = namespace.getStorage().getPrimaryDictionary(); final EntityIdMap idMapping = namespace.getStorage().getIdMapping(); - - - return idMappingConfig.toExternal(new CsvEntityId(dict.getElement(cer.getEntityId())), namespace, - mappingState, idMapping + return idMappingConfig.toExternal( + dict.getElement(cer.getEntityId()), namespace, + mappingState, idMapping ); } public static Response makeResponseWithFileName(StreamingOutput out, String label, String fileExtension) { Response.ResponseBuilder response = Response.ok(out); - if(!(Strings.isNullOrEmpty(label) || label.isBlank())) { + if (!(Strings.isNullOrEmpty(label) || label.isBlank())) { // Set filename from label if the label was set, otherwise the browser will name the file according to the request path response.header("Content-Disposition", String.format( - "attachment; filename=\"%s\"",FileUtil.makeSafeFileName(label, fileExtension))); + "attachment; filename=\"%s\"", FileUtil.makeSafeFileName(label, fileExtension))); } return response.build(); } @@ -51,14 +49,15 @@ public static Response makeResponseWithFileName(StreamingOutput out, String labe * Defaults to StandardCharsets.UTF_8. */ public static Charset determineCharset(String userAgent, String queryCharset) { - if(queryCharset != null) { + if (queryCharset != null) { try { return Charset.forName(queryCharset); - }catch (Exception e) { + } + catch (Exception e) { log.warn("Unable to map '{}' to a charset. Defaulting to UTF-8", queryCharset); } } - if(userAgent != null && userAgent.toLowerCase().contains("windows") ) { + if (userAgent != null && userAgent.toLowerCase().contains("windows")) { return StandardCharsets.ISO_8859_1; } return StandardCharsets.UTF_8; @@ -67,10 +66,11 @@ public static Charset determineCharset(String userAgent, String queryCharset) { /** * Throws a "Bad Request" response if the execution result is not a single table. + * * @param exec the execution to test */ public static void checkSingleTableResult(ManagedExecution exec) { - if (!(exec instanceof SingleTableResult)){ + if (!(exec instanceof SingleTableResult)) { throw new BadRequestException("Execution cannot be rendered as the requested format"); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/NoIdMapping.java b/backend/src/main/java/com/bakdata/conquery/models/config/NoIdMapping.java index 8bfffccbd2..b6cd730c2a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/NoIdMapping.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/NoIdMapping.java @@ -4,7 +4,6 @@ import java.util.List; import com.bakdata.conquery.io.cps.CPSType; -import com.bakdata.conquery.models.identifiable.mapping.CsvEntityId; import com.bakdata.conquery.models.identifiable.mapping.IdMappingConfig; import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; import com.univocity.parsers.common.record.Record; @@ -14,7 +13,7 @@ public class NoIdMapping extends IdMappingConfig { private static final String[] HEADER = new String[]{"result"}; @Override - protected void processRecord(Record record, CsvEntityId id, EntityIdMap mapping) { + protected void processRecord(Record record, String id, EntityIdMap mapping) { // Do nothing. } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/SimpleIdMapping.java b/backend/src/main/java/com/bakdata/conquery/models/config/SimpleIdMapping.java index 0b29966dee..4355276bcc 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/SimpleIdMapping.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/SimpleIdMapping.java @@ -3,9 +3,9 @@ import java.util.List; import com.bakdata.conquery.io.cps.CPSType; -import com.bakdata.conquery.models.identifiable.mapping.CsvEntityId; -import com.bakdata.conquery.models.identifiable.mapping.IdMappingConfig; import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; +import com.bakdata.conquery.models.identifiable.mapping.EntityPrintId; +import com.bakdata.conquery.models.identifiable.mapping.IdMappingConfig; import com.univocity.parsers.common.record.Record; /** @@ -17,11 +17,13 @@ public class SimpleIdMapping extends IdMappingConfig { @Override - protected void processRecord(Record record, CsvEntityId id, EntityIdMap mapping) { + protected void processRecord(Record record, String id, EntityIdMap mapping) { + + final EntityPrintId entityPrintId = EntityPrintId.from(id); - mapping.addOutputMapping(id, id.getCsvId()); + mapping.addOutputMapping(id, entityPrintId); - mapping.addInputMapping(id, id.getCsvId()); + mapping.addInputMapping(id, id); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/CsvEntityId.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/CsvEntityId.java deleted file mode 100644 index e3afd02a6a..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/CsvEntityId.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.bakdata.conquery.models.identifiable.mapping; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonValue; - -import lombok.Data; -import lombok.Getter; -import lombok.RequiredArgsConstructor; - -/** - * This class holds the csvId of an Entity. - */ -@Data @RequiredArgsConstructor(onConstructor_=@JsonCreator) -public class CsvEntityId { - - /** - * The csvId of an entity. - */ - @Getter(onMethod_=@JsonValue) - private final String csvId; -} \ No newline at end of file diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java index fdfd432801..6e6d2b48ba 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java @@ -4,8 +4,6 @@ import java.util.Map; import java.util.Optional; -import com.bakdata.conquery.io.jackson.serializer.PersistentIdMapDeserializer; -import com.bakdata.conquery.io.jackson.serializer.PersistentIdMapSerializer; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import lombok.EqualsAndHashCode; @@ -21,37 +19,35 @@ @EqualsAndHashCode @RequiredArgsConstructor @Slf4j -@JsonSerialize(using = PersistentIdMapSerializer.class) -@JsonDeserialize(using = PersistentIdMapDeserializer.class) public class EntityIdMap { /** * The map from csv entity ids to external entity ids. */ - private final Map csvIdToExternalIdMap = new HashMap<>(); + private final Map csvIdToExternalIdMap = new HashMap<>(); /** * The map from external entity ids to csv entity ids. */ - private final Map externalIdPartCsvIdMap = new HashMap<>(); + private final Map externalIdPartCsvIdMap = new HashMap<>(); /** * Map an internal id to an external. */ - public EntityPrintId toExternal(CsvEntityId internal) { + public EntityPrintId toExternal(String internal) { return csvIdToExternalIdMap.get(internal); } /** * Map an external to an internal id. + * @param external */ - public Optional toInternal(UnresolvedEntityId external) { + public Optional toInternal(String external) { return Optional.ofNullable(externalIdPartCsvIdMap.get(external)); } - public void addOutputMapping(CsvEntityId csvEntityId, String... parts) { - EntityPrintId externalEntityId = new EntityPrintId(parts); + public void addOutputMapping(String csvEntityId, EntityPrintId externalEntityId) { final EntityPrintId prior = csvIdToExternalIdMap.put(csvEntityId, externalEntityId); if (prior != null && prior.equals(externalEntityId)) { @@ -59,10 +55,10 @@ public void addOutputMapping(CsvEntityId csvEntityId, String... parts) { } } - public void addInputMapping(CsvEntityId csvEntityId, String part) { - UnresolvedEntityId externalEntityId = new UnresolvedEntityId(part); - final CsvEntityId prior = externalIdPartCsvIdMap.put(externalEntityId, csvEntityId); + public void addInputMapping(String csvEntityId, String externalEntityId) { + + final String prior = externalIdPartCsvIdMap.put(externalEntityId, csvEntityId); if (prior != null && prior.equals(csvEntityId)) { log.warn("Duplicate mapping for {} to {} and {}", externalEntityId, csvEntityId, prior); diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityPrintId.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityPrintId.java index e24df607bd..02b4c9e0e4 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityPrintId.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityPrintId.java @@ -2,8 +2,11 @@ import java.util.Arrays; +import javax.validation.constraints.NotNull; + import com.fasterxml.jackson.annotation.JsonCreator; +import lombok.AccessLevel; import lombok.Data; import lombok.RequiredArgsConstructor; @@ -11,23 +14,19 @@ * An external Id for a Entity. */ @Data -@RequiredArgsConstructor(onConstructor_ = @JsonCreator) +@RequiredArgsConstructor(onConstructor_ = @JsonCreator, access = AccessLevel.PROTECTED) public class EntityPrintId implements Comparable { + public static EntityPrintId from(String... parts) { + return new EntityPrintId(parts); + } + /** * The external Entity Id. */ + @NotNull private final String[] externalId; - /** - * Casts a given csv Entity Id into an ExternalEntityId. - * - * @param csvEntityId the given csvEntityId. - * @return the casted ExternalEntityId. - */ - public static EntityPrintId from(CsvEntityId csvEntityId) { - return new EntityPrintId(new String[] {csvEntityId.getCsvId() }); - } @Override public int compareTo(EntityPrintId o) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingConfig.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingConfig.java index 9ec2055df8..f870e6a991 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingConfig.java @@ -31,7 +31,7 @@ public EntityIdMap generateIdMapping(CsvParser parser) throws IllegalArgumentExc while((record = parser.parseNextRecord()) != null){ final String id = record.getString("id"); - final CsvEntityId csvEntityId = new CsvEntityId(id); + final String csvEntityId = new String(id); processRecord(record, csvEntityId, mapping); } @@ -41,7 +41,7 @@ public EntityIdMap generateIdMapping(CsvParser parser) throws IllegalArgumentExc return mapping; } - protected abstract void processRecord(Record record, CsvEntityId id, EntityIdMap mapping); + protected abstract void processRecord(Record record, String id, EntityIdMap mapping); @JsonIgnore public abstract List getPrintIdFields(); @@ -58,7 +58,7 @@ public IdMappingState initToExternal(User user, ManagedExecution execution) { /** * Converts the internal ID to the an external. */ - public EntityPrintId toExternal(CsvEntityId csvEntityId, Namespace namespace, IdMappingState state, EntityIdMap mapping) { + public EntityPrintId toExternal(String csvEntityId, Namespace namespace, IdMappingState state, EntityIdMap mapping) { // The state may be uses by implementations of this class if (mapping == null) { diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java index c7fee059e5..5f9791602b 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/arrow/ArrowResultGenerationTest.java @@ -111,7 +111,7 @@ void writeAndRead() throws IOException { Locale.ROOT, null, CONFIG, - (cer) -> new EntityPrintId(new String[]{Integer.toString(cer.getEntityId()), Integer.toString(cer.getEntityId())}), + (cer) -> EntityPrintId.from(Integer.toString(cer.getEntityId()), Integer.toString(cer.getEntityId())), (selectInfo) -> selectInfo.getSelect().getLabel()); // The Shard nodes send Object[] but since Jackson is used for deserialization, nested collections are always a list because they are not further specialized List results = getTestEntityResults(); diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/csv/CsvResultGenerationTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/csv/CsvResultGenerationTest.java index 8f37f71405..6ad4d8726b 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/csv/CsvResultGenerationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/csv/CsvResultGenerationTest.java @@ -51,7 +51,7 @@ void writeAndRead() throws IOException { Locale.GERMAN, null, CONFIG, - (cer) -> new EntityPrintId(new String[]{Integer.toString(cer.getEntityId()), Integer.toString(cer.getEntityId())}), + (cer) -> EntityPrintId.from(Integer.toString(cer.getEntityId()), Integer.toString(cer.getEntityId())), (selectInfo) -> selectInfo.getSelect().getLabel()); // The Shard nodes send Object[] but since Jackson is used for deserialization, nested collections are always a list because they are not further specialized List results = getTestEntityResults(); diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java index 737cad8747..cf8a8eba18 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java @@ -60,7 +60,7 @@ void writeAndRead() throws IOException { Locale.GERMAN, null, CONFIG, - (cer) -> new EntityPrintId(new String[]{Integer.toString(cer.getEntityId()), Integer.toString(cer.getEntityId())}), + (cer) -> EntityPrintId.from(Integer.toString(cer.getEntityId()), Integer.toString(cer.getEntityId())), (selectInfo) -> selectInfo.getSelect().getLabel()); // The Shard nodes send Object[] but since Jackson is used for deserialization, nested collections are always a list because they are not further specialized List results = getTestEntityResults(); diff --git a/backend/src/test/java/com/bakdata/conquery/models/identifiable/IdMapSerialisationTest.java b/backend/src/test/java/com/bakdata/conquery/models/identifiable/IdMapSerialisationTest.java index a07e9cd65d..6e3eaf0367 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/identifiable/IdMapSerialisationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/identifiable/IdMapSerialisationTest.java @@ -1,15 +1,16 @@ package com.bakdata.conquery.models.identifiable; -import com.bakdata.conquery.models.identifiable.mapping.CsvEntityId; import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; +import com.bakdata.conquery.models.identifiable.mapping.EntityPrintId; public class IdMapSerialisationTest { public static EntityIdMap createTestPersistentMap() { EntityIdMap entityIdMap = new EntityIdMap(); - entityIdMap.addInputMapping(new CsvEntityId("test1"), "a"); - entityIdMap.addOutputMapping(new CsvEntityId("test2"), "c"); + entityIdMap.addInputMapping("test1", "a"); + + entityIdMap.addOutputMapping("test2", EntityPrintId.from("c")); return entityIdMap; } From 560a4fa011b8c76e2955301c1eaf1369278f72ff Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Wed, 7 Jul 2021 17:49:09 +0200 Subject: [PATCH 04/82] Migrate format back to inline Types Also expose available FormatColumns under FontendConfig --- .../concept/specific/external/CQExternal.java | 107 +++++++------ .../specific/external/FormatColumn.java | 25 +++ .../conquery/io/cps/CPSTypeIdResolver.java | 146 +++++++++--------- .../models/config/FrontendConfig.java | 93 ++++++++++- .../models/execution/ManagedExecution.java | 26 ++-- .../identifiable/mapping/IdMappingConfig.java | 6 +- .../integration/common/LoadingUtil.java | 2 +- .../SIMPLE_CQEXTERNAL_QUERY.test.json | 2 +- 8 files changed, 260 insertions(+), 147 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java index feb4bc4255..b9c7bacd9c 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java @@ -2,20 +2,22 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Date; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import javax.validation.constraints.NotEmpty; import com.bakdata.conquery.apiv1.query.CQElement; import com.bakdata.conquery.io.cps.CPSType; +import com.bakdata.conquery.io.cps.CPSTypeIdResolver; import com.bakdata.conquery.io.jackson.InternalOnly; import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.dictionary.EncodedDictionary; import com.bakdata.conquery.models.error.ConqueryError; import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; -import com.bakdata.conquery.models.identifiable.mapping.UnresolvedEntityId; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; import com.bakdata.conquery.models.query.queryplan.ConceptQueryPlan; @@ -28,15 +30,23 @@ import io.dropwizard.validation.ValidationMethod; import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; import lombok.Getter; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +/** + * Allows uploading lists of entities. + */ @Slf4j @CPSType(id = "EXTERNAL", base = CQElement.class) public class CQExternal extends CQElement { + /** + * List of Type-Ids of Format Columns. + */ @Getter @NotEmpty - private final List format; + private final List format; + @Getter @NotEmpty private final String[][] values; @@ -45,13 +55,9 @@ public class CQExternal extends CQElement { @InternalOnly private Map valuesResolved; - public CQExternal(@NotEmpty List format, @NotEmpty String[][] values) { + public CQExternal(@NotEmpty List format, @NotEmpty String[][] values) { this.format = format; this.values = values; - - for (int index = 0; index < format.size(); index++) { - format.get(index).setPosition(index); - } } @Override @@ -66,25 +72,29 @@ public QPNode createQueryPlan(QueryPlanContext context, ConceptQueryPlan plan) { @Override public void resolve(QueryResolveContext context) { - valuesResolved = new Int2ObjectOpenHashMap<>(); + List resolvedFormats = instantiateFormatIds(format); + + for (int index = 0; index < resolvedFormats.size(); index++) { + resolvedFormats.get(index).setPosition(index); + } - //TODO probably easy to simplify. + valuesResolved = new Int2ObjectOpenHashMap<>(); - final DateColumn[] dateColumns = format.stream() - .filter(DateColumn.class::isInstance) - .map(DateColumn.class::cast) - .toArray(DateColumn[]::new); + final DateColumn[] dateColumns = resolvedFormats.stream() + .filter(DateColumn.class::isInstance) + .map(DateColumn.class::cast) + .toArray(DateColumn[]::new); //TODO verify dateColumns match final DateFormat dateFormat = dateColumns.length > 0 ? dateColumns[0].getFormat() : DateFormat.ALL; - final IdColumn idColumn = format.stream() - .filter(IdColumn.class::isInstance) - .map(IdColumn.class::cast) - .collect(MoreCollectors.onlyElement()); + final IdColumn idColumn = resolvedFormats.stream() + .filter(IdColumn.class::isInstance) + .map(IdColumn.class::cast) + .collect(MoreCollectors.onlyElement()); final EncodedDictionary primary = context.getNamespace().getStorage().getPrimaryDictionary(); @@ -100,7 +110,6 @@ public void resolve(QueryResolveContext context) { final String[] row = values[i]; final String externalId = idColumn.read(row); - //read the dates from the row try { @@ -135,44 +144,48 @@ public void resolve(QueryResolveContext context) { } } + /** + * Helper method to flatten API surface, allowing plain passing of type-ids instead of using objects of only type. + */ + @SneakyThrows + private static List instantiateFormatIds(@NotEmpty List formats) { + List resolvedFormats = new ArrayList<>(); + for (String s : formats) { + + Class clazz = CPSTypeIdResolver.getImplementation(FormatColumn.class, s); + // We've already established at startup, that this should not fail. + FormatColumn newInstance = clazz.getConstructor().newInstance(); + resolvedFormats.add(newInstance); + } + + return resolvedFormats; + } + @Override public void collectResultInfos(ResultInfoCollector collector) { } @JsonIgnore - @ValidationMethod(message = "Values and Format are not of same width.") - public boolean isAllSameLength() { - final int expected = format.size(); - return Arrays.stream(values).mapToInt(a -> a.length).allMatch(v -> expected == v); + @ValidationMethod(message = "Must contain only valid FormatColumn Ids.") + public boolean isValidFormatIds() { + return format.stream().map(id -> CPSTypeIdResolver.getImplementation(FormatColumn.class, id)).noneMatch(Objects::isNull); } @JsonIgnore - @ValidationMethod(message = "Must use one Id.") - public boolean isOnlyOneId() { - return format.stream().filter(IdColumn.class::isInstance).count() == 1; + @ValidationMethod(message = "Must use one IdColumn.") + public boolean isOnlyOneIdColumn() { + return format.stream() + .map(id -> CPSTypeIdResolver.getImplementation(FormatColumn.class, id)) + .filter(IdColumn.class::isAssignableFrom) + .count() == 1; } - //TODO implement this - - // @ValidationMethod(message = "Wrong usage of Dates.") - // public boolean isDatesAccessorExclusive() { - // if (format.contains(FormatColumn.DATE_RANGE)) { - // return !(format.contains(FormatColumn.DATE_SET) || format.contains(FormatColumn.START_DATE) || format.contains(FormatColumn.END_DATE)); - // } - // - // if (format.contains(FormatColumn.DATE_SET)) { - // return !(format.contains(FormatColumn.START_DATE) || format.contains(FormatColumn.END_DATE)); - // } - // - // if (format.stream().filter(FormatColumn.START_DATE::equals).count() > 1) { - // return false; - // } - // - // if (format.stream().filter(FormatColumn.END_DATE::equals).count() > 1) { - // return false; - // } - // - // return true; - // } + + @JsonIgnore + @ValidationMethod(message = "Values and Format are not of same width.") + public boolean isAllSameLength() { + final int expected = format.size(); + return Arrays.stream(values).mapToInt(a -> a.length).allMatch(v -> expected == v); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/FormatColumn.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/FormatColumn.java index 62f304a28d..83b8701c56 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/FormatColumn.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/FormatColumn.java @@ -1,18 +1,43 @@ package com.bakdata.conquery.apiv1.query.concept.specific.external; +import java.lang.reflect.InvocationTargetException; +import java.util.Set; + +import com.bakdata.conquery.apiv1.forms.Form; import com.bakdata.conquery.io.cps.CPSBase; +import com.bakdata.conquery.io.cps.CPSType; +import com.bakdata.conquery.io.cps.CPSTypeIdResolver; import com.bakdata.conquery.io.jackson.InternalOnly; import com.fasterxml.jackson.annotation.JsonTypeInfo; import lombok.Data; import lombok.Getter; import lombok.Setter; +import lombok.extern.slf4j.Slf4j; @CPSBase @JsonTypeInfo(use = JsonTypeInfo.Id.CUSTOM, property = "type") @Data +@Slf4j public abstract class FormatColumn { @InternalOnly @Setter @Getter private int position; + static { + // For each implementation of FormatColumn, test if we can instantiate it using default constructor. Fail start-up if not possible. + final Set> implementations = CPSTypeIdResolver.listImplementations(FormatColumn.class); + + for (Class impl : implementations) { + final String id = impl.getAnnotation(CPSType.class).id(); + try { + // Try and get no-args constructor + impl.getConstructor().newInstance(); + } + catch (Exception e) { + log.error("FormatColumn {} has no default constructor.", id); + throw new RuntimeException(e); + } + } + } + } diff --git a/backend/src/main/java/com/bakdata/conquery/io/cps/CPSTypeIdResolver.java b/backend/src/main/java/com/bakdata/conquery/io/cps/CPSTypeIdResolver.java index 767bcaa76c..d3c258f398 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/cps/CPSTypeIdResolver.java +++ b/backend/src/main/java/com/bakdata/conquery/io/cps/CPSTypeIdResolver.java @@ -34,7 +34,7 @@ public class CPSTypeIdResolver implements TypeIdResolver { private static HashMap, CPSMap> globalMap; public static final ScanResult SCAN_RESULT; - + private JavaType baseType; private CPSMap cpsMap; @@ -42,21 +42,21 @@ public class CPSTypeIdResolver implements TypeIdResolver { public void init(JavaType baseType) { this.baseType = baseType; this.cpsMap = new CPSMap(); - + //this creates an aggregate map of all the children Iterable> types = Traverser.forGraph( - (SuccessorsFunction>) node -> { - Class superclass = node.getSuperclass(); - List> interfaces = Arrays.asList(node.getInterfaces()); - return superclass == null - ? interfaces - : Iterables.concat(interfaces, Collections.singleton(superclass)); - } + (SuccessorsFunction>) node -> { + Class superclass = node.getSuperclass(); + List> interfaces = Arrays.asList(node.getInterfaces()); + return superclass == null + ? interfaces + : Iterables.concat(interfaces, Collections.singleton(superclass)); + } ).breadthFirst(baseType.getRawClass()); - - for(Class type : types) { + + for (Class type : types) { CPSMap local = globalMap.get(type); - if(local != null) { + if (local != null) { cpsMap.merge(local); } } @@ -69,141 +69,147 @@ public void init(JavaType baseType) { //scan classpaths for annotated child classes SCAN_RESULT = new ClassGraph() - .enableClassInfo() - .enableAnnotationInfo() - //TODO if ever necessary, use an environment variable also - .acceptPackages( - "com.bakdata", - "tests" - ) - .scan(); - + .enableClassInfo() + .enableAnnotationInfo() + //TODO if ever necessary, use an environment variable also + .acceptPackages( + "com.bakdata", + "tests" + ) + .scan(); + log.info("Scanned: {} classes in classpath", SCAN_RESULT.getAllClasses().size()); Set> types = new HashSet<>(); types.addAll(SCAN_RESULT.getClassesWithAnnotation(CPSTypes.class.getName()).loadClasses()); types.addAll(SCAN_RESULT.getClassesWithAnnotation(CPSType.class.getName()).loadClasses()); - + globalMap = new HashMap<>(); - for(Class type:types) { + for (Class type : types) { CPSType[] annos = type.getAnnotationsByType(CPSType.class); - for(CPSType anno:annos) { - CPSMap map = globalMap.computeIfAbsent(anno.base(), b->new CPSMap()); - + for (CPSType anno : annos) { + CPSMap map = globalMap.computeIfAbsent(anno.base(), b -> new CPSMap()); + //check if base is marked as base CPSBase baseAnno = anno.base().getAnnotation(CPSBase.class); - if(baseAnno==null) { - throw new IllegalStateException("The class "+anno.base()+" is used as a CPSBase in "+type+" but not annotated as such."); + if (baseAnno == null) { + throw new IllegalStateException("The class " + anno.base() + " is used as a CPSBase in " + type + " but not annotated as such."); } - if(!anno.base().isAssignableFrom(type)) { - throw new IllegalStateException("The class "+anno.base()+" is used as a CPSBase in "+type+" but type is no subclass of it."); + if (!anno.base().isAssignableFrom(type)) { + throw new IllegalStateException("The class " + anno.base() + " is used as a CPSBase in " + type + " but type is no subclass of it."); } - if(anno.subTyped() && !SubTyped.class.isAssignableFrom(type)) { - throw new IllegalStateException("The class "+type+" is flagged to support a subtyping information but does not implement "+ SubTyped.class.getName()); + if (anno.subTyped() && !SubTyped.class.isAssignableFrom(type)) { + throw new IllegalStateException("The class " + + type + + " is flagged to support a subtyping information but does not implement " + + SubTyped.class.getName()); } - + map.add(anno.id(), type); } } - + List> bases = SCAN_RESULT.getClassesWithAnnotation(CPSBase.class.getName()).loadClasses(); - for(Class b:bases) { + for (Class b : bases) { CPSMap map = globalMap.get(b); - if(map==null) { + if (map == null) { log.warn("\tBase Class {}:\tNo registered types", b); } else { log.info("\tBase Class {}", b.getSimpleName()); map.calculateInverse(); - for(Entry, String> e:map) { + for (Entry, String> e : map) { log.info("\t\t{}\t->\t{}", e.getValue(), e.getKey().getSimpleName()); } } - + } } @Override public JavaType typeFromId(DatabindContext context, String id) { Class result = cpsMap.getClassFromId(truncateSubTypeInformation(id)); - if(result == null) { - throw new IllegalStateException("There is no type "+id+" for "+baseType.getTypeName()+". Try: "+getDescForKnownTypeIds()); + if (result == null) { + throw new IllegalStateException("There is no type " + id + " for " + baseType.getTypeName() + ". Try: " + getDescForKnownTypeIds()); } String subTypeInfo = extractSubTypeInformation(id); - if(!Strings.isNullOrEmpty(subTypeInfo)) { - + if (!Strings.isNullOrEmpty(subTypeInfo)) { + context.setAttribute(ATTRIBUTE_SUB_TYPE, subTypeInfo); } return TypeFactory.defaultInstance().constructSpecializedType(baseType, result); } - + public static String truncateSubTypeInformation(@NonNull String fullType) { int seperatorIndex = fullType.indexOf(SEPARATOR_SUB_TYPE); - if(seperatorIndex < 0) { + if (seperatorIndex < 0) { // Separator not found return fullType; } return fullType.substring(0, seperatorIndex); } - + public static String extractSubTypeInformation(@NonNull String fullType) { int seperatorIndex = fullType.indexOf(SEPARATOR_SUB_TYPE); - if(seperatorIndex < 0) { + if (seperatorIndex < 0) { // Separator not found return null; } // +1 because we want to skip the separator - return fullType.substring(seperatorIndex+1); - + return fullType.substring(seperatorIndex + 1); + } - - public static String createSubTyped(@NonNull String type,@NonNull String sub) { + + public static String createSubTyped(@NonNull String type, @NonNull String sub) { return String.join(SEPARATOR_SUB_TYPE, type, sub); } - + public static Set> listImplementations(Class base) { CPSMap map = globalMap.get(base); - if(map == null) { + if (map == null) { log.warn("No implementations for {}", base); return Collections.emptySet(); } - return (Set>)(Set)map.getClasses(); + return (Set>) (Set) map.getClasses(); } - + + public static Class getImplementation(Class base, String subType) { + return (Class) globalMap.get(base).getClassFromId(subType); + } + public static Set, Class>> listImplementations() { - return globalMap - .entrySet() - .stream() - ., Class>>flatMap(e-> - e.getValue() - .getClasses() + return globalMap.entrySet() .stream() - .map(v->Pair.of(e.getKey(), v)) - ) - .collect(Collectors.toSet()); + ., Class>>flatMap(e -> e.getValue() + .getClasses() + .stream() + .map(v -> Pair.of(e.getKey(), v)) + ) + .collect(Collectors.toSet()); } @Override public String idFromValueAndType(Object value, Class suggestedType) { String result = cpsMap.getTypeIdForClass(suggestedType); - if(result == null) { + if (result == null) { //check if other base CPSType anno = value.getClass().getAnnotation(CPSType.class); - if(anno == null) - throw new IllegalStateException("There is no id for the class "+suggestedType+" for "+baseType.getTypeName()+"."); + if (anno == null) { + throw new IllegalStateException("There is no id for the class " + suggestedType + " for " + baseType.getTypeName() + "."); + } return anno.id(); } CPSType anno = suggestedType.getAnnotation(CPSType.class); - if(anno != null && anno.subTyped()) { - return createSubTyped(result, ((SubTyped)value).getSubType()); + if (anno != null && anno.subTyped()) { + return createSubTyped(result, ((SubTyped) value).getSubType()); } return result; } - + @Override public String getDescForKnownTypeIds() { return new TreeSet<>(cpsMap.getTypeIds()).toString(); } - + @Override public String idFromValue(Object value) { return idFromValueAndType(value, value.getClass()); diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java index 66f12f74f9..7dbba4645a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java @@ -1,28 +1,107 @@ package com.bakdata.conquery.models.config; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; + import javax.validation.Valid; +import javax.validation.constraints.NotEmpty; import javax.validation.constraints.NotNull; +import com.bakdata.conquery.apiv1.query.concept.specific.external.DateColumn; +import com.bakdata.conquery.apiv1.query.concept.specific.external.FormatColumn; +import com.bakdata.conquery.apiv1.query.concept.specific.external.IdColumn; +import com.bakdata.conquery.io.cps.CPSType; +import com.bakdata.conquery.io.cps.CPSTypeIdResolver; import com.bakdata.conquery.util.VersionInfo; - +import com.fasterxml.jackson.annotation.JsonIgnore; import groovy.transform.ToString; +import io.dropwizard.validation.ValidationMethod; import lombok.Data; import lombok.Getter; import lombok.Setter; -@ToString @Getter @Setter +@ToString +@Getter +@Setter public class FrontendConfig { - + + private String version = VersionInfo.INSTANCE.getProjectVersion(); - @Valid @NotNull + @Valid + @NotNull private CurrencyConfig currency = new CurrencyConfig(); - + + private UploadConfig queryUpload = new UploadConfig(); + + @Data + public static class UploadConfig { + private final static List DEFAULT_IDS = new ArrayList<>(); + + + static { + // Collect all implementations of IdClass. + + final Set> idClasses = CPSTypeIdResolver.listImplementations(FormatColumn.class); + + for (Class idClass : idClasses) { + + if (!IdColumn.class.isAssignableFrom(idClass)) { + continue; + } + + final String id = idClass.getAnnotation(CPSType.class).id(); + DEFAULT_IDS.add(new ColumnConfig(id, Map.of("en", id), Map.of("en", id))); + } + } + + @NotEmpty + private List ids = DEFAULT_IDS; + + @NotNull + private ColumnConfig dateStart = + new ColumnConfig(DateColumn.DateStart.class.getAnnotation(CPSType.class).id(), Map.of("en", "Begin"), Map.of("en", "Begin of Date")); + + @NotNull + private ColumnConfig dateEnd = + new ColumnConfig(DateColumn.DateEnd.class.getAnnotation(CPSType.class).id(), Map.of("en", "Begin"), Map.of("en", "Begin of Date")); + + @NotNull + private ColumnConfig dateRange = + new ColumnConfig(DateColumn.DateRange.class.getAnnotation(CPSType.class).id(), Map.of("en", "Date Range"), Map.of("en", "Full Date Range")); + + @NotNull + private ColumnConfig dateSet = + new ColumnConfig(DateColumn.DateSet.class.getAnnotation(CPSType.class).id(), Map.of("en", "Dateset"), Map.of("en", "Set of Date-Ranges")); + + @NotNull + private ColumnConfig eventDate = + new ColumnConfig(DateColumn.EventDate.class.getAnnotation(CPSType.class).id(), Map.of("en", "Event Date"), Map.of("en", "Single day")); + + @Data + public static class ColumnConfig { + @NotEmpty + private final String name; + + private final Map label; + private final Map description; + + @JsonIgnore + @ValidationMethod + public boolean isExistingColumnFormat() { + return CPSTypeIdResolver.getImplementation(FormatColumn.class, name) != null; + } + } + } + @Data - public - static class CurrencyConfig { + public static class CurrencyConfig { private String prefix = "€"; private String thousandSeparator = "."; private String decimalSeparator = ","; private int decimalScale = 2; } + + } \ No newline at end of file diff --git a/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java b/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java index cc8b68e31b..b51497c60d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java +++ b/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java @@ -30,10 +30,10 @@ import com.bakdata.conquery.models.auth.permissions.Ability; import com.bakdata.conquery.models.auth.permissions.ConqueryPermission; import com.bakdata.conquery.models.auth.permissions.ExecutionPermission; -import com.bakdata.conquery.models.datasets.concepts.Concept; -import com.bakdata.conquery.models.datasets.concepts.ConceptElement; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.datasets.Dataset; +import com.bakdata.conquery.models.datasets.concepts.Concept; +import com.bakdata.conquery.models.datasets.concepts.ConceptElement; import com.bakdata.conquery.models.error.ConqueryErrorInfo; import com.bakdata.conquery.models.forms.managed.ManagedForm; import com.bakdata.conquery.models.i18n.I18n; @@ -66,18 +66,6 @@ import org.apache.commons.lang3.ArrayUtils; import org.apache.shiro.authz.Permission; -import javax.annotation.Nullable; -import javax.validation.constraints.NotNull; -import javax.ws.rs.core.UriBuilder; -import java.time.Duration; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.temporal.ChronoUnit; -import java.util.*; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - @Getter @Setter @ToString @@ -151,8 +139,14 @@ public void initExecutable(DatasetRegistry datasetRegistry, ConqueryConfig confi executionManager = datasetRegistry.get(getDataset().getId()).getExecutionManager(); - doInitExecutable(datasetRegistry, config); - initialized = true; + try { + doInitExecutable(datasetRegistry, config); + initialized = true; + } + catch (Exception e) { + log.error("Failed to initialize", e); + //TODO what do? + } } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingConfig.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingConfig.java index f870e6a991..20567851f1 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingConfig.java @@ -31,13 +31,9 @@ public EntityIdMap generateIdMapping(CsvParser parser) throws IllegalArgumentExc while((record = parser.parseNextRecord()) != null){ final String id = record.getString("id"); - final String csvEntityId = new String(id); - - processRecord(record, csvEntityId, mapping); + processRecord(record, id, mapping); } - //TODO mapping.checkIntegrity(Arrays.asList(getIdAccessors())); - return mapping; } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java b/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java index bad4ab8d7e..e71591876d 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java @@ -75,7 +75,7 @@ public static void importPreviousQueries(StandaloneSupport support, RequiredData final CsvParser parser = support.getConfig().getCsv().withParseHeaders(false).withSkipHeader(false).createParser(); String[][] data = parser.parseAll(queryResults.stream()).toArray(new String[0][]); - ConceptQuery q = new ConceptQuery(new CQExternal(Arrays.asList(new IdColumn(), new DateColumn.DateSet()), data)); + ConceptQuery q = new ConceptQuery(new CQExternal(Arrays.asList("ID", "DATE_SET"), data)); ManagedExecution managed = support.getNamespace().getExecutionManager().createQuery(support.getNamespace().getNamespaces(),q, queryId, user, support.getNamespace().getDataset()); user.addPermission(support.getMetaStorage(), ExecutionPermission.onInstance(AbilitySets.QUERY_CREATOR, managed.getId())); diff --git a/backend/src/test/resources/tests/query/SIMPLE_CQEXTERNAL_QUERY/SIMPLE_CQEXTERNAL_QUERY.test.json b/backend/src/test/resources/tests/query/SIMPLE_CQEXTERNAL_QUERY/SIMPLE_CQEXTERNAL_QUERY.test.json index 476fe2e83f..75dce85ed8 100644 --- a/backend/src/test/resources/tests/query/SIMPLE_CQEXTERNAL_QUERY/SIMPLE_CQEXTERNAL_QUERY.test.json +++ b/backend/src/test/resources/tests/query/SIMPLE_CQEXTERNAL_QUERY/SIMPLE_CQEXTERNAL_QUERY.test.json @@ -7,7 +7,7 @@ "root": { "type": "EXTERNAL", "format": [ - "id","EVENT_DATE" + "ID","EVENT_DATE" ], "values": [ ["result","dates"], From c0d85cde2399489fb268e5e53d496aa16e152956 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Wed, 7 Jul 2021 18:02:32 +0200 Subject: [PATCH 05/82] merge fixes --- .../conquery/apiv1/query/QueryDescription.java | 1 - .../concept/specific/external/IgnoreColumn.java | 11 +++++++++++ .../conquery/models/query/ManagedQuery.java | 8 +------- .../conquery/api/StoredQueriesProcessorTest.java | 1 - .../conquery/integration/common/LoadingUtil.java | 14 ++------------ 5 files changed, 14 insertions(+), 21 deletions(-) create mode 100644 backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/IgnoreColumn.java diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/QueryDescription.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/QueryDescription.java index 0cfbe62302..1a0749d09c 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/QueryDescription.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/QueryDescription.java @@ -3,7 +3,6 @@ import java.util.Set; import java.util.stream.Collectors; -import com.bakdata.conquery.apiv1.query.concept.specific.CQExternal; import com.bakdata.conquery.io.cps.CPSBase; import com.bakdata.conquery.io.jackson.InternalOnly; import com.bakdata.conquery.models.auth.entities.User; diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/IgnoreColumn.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/IgnoreColumn.java new file mode 100644 index 0000000000..5ef653ab12 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/IgnoreColumn.java @@ -0,0 +1,11 @@ +package com.bakdata.conquery.apiv1.query.concept.specific.external; + +import com.bakdata.conquery.io.cps.CPSType; + +@CPSType(id = "IGNORE", base = FormatColumn.class) +public class IgnoreColumn extends FormatColumn { + + public String read(String[] row) { + return null; + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java b/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java index 827c54a85c..e2c5a96dcf 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java @@ -19,8 +19,8 @@ import com.bakdata.conquery.apiv1.query.QueryDescription; import com.bakdata.conquery.apiv1.query.SecondaryIdQuery; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; -import com.bakdata.conquery.apiv1.query.concept.specific.CQExternal; import com.bakdata.conquery.apiv1.query.concept.specific.CQReusedQuery; +import com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal; import com.bakdata.conquery.internationalization.CQElementC10n; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.storage.MetaStorage; @@ -34,12 +34,6 @@ import com.bakdata.conquery.models.i18n.I18n; import com.bakdata.conquery.models.identifiable.ids.NamespacedIdentifiable; import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; -import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; -import com.bakdata.conquery.apiv1.query.SecondaryIdQuery; -import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; -import com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal; -import com.bakdata.conquery.apiv1.query.concept.specific.CQReusedQuery; -import com.bakdata.conquery.models.query.queryplan.QueryPlan; import com.bakdata.conquery.models.messages.namespaces.WorkerMessage; import com.bakdata.conquery.models.messages.namespaces.specific.ExecuteQuery; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; diff --git a/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java b/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java index 97c6e8bc8f..92c3874af7 100644 --- a/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java +++ b/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java @@ -22,7 +22,6 @@ import com.bakdata.conquery.apiv1.query.SecondaryIdQuery; import com.bakdata.conquery.apiv1.query.concept.specific.CQAnd; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; -import com.bakdata.conquery.apiv1.query.concept.specific.CQExternal; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.models.auth.AuthorizationController; import com.bakdata.conquery.models.auth.AuthorizationHelper; diff --git a/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java b/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java index 22eb7cf2e6..4550f3fd44 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java @@ -16,29 +16,22 @@ import java.util.Map; import java.util.UUID; -import javax.swing.text.Utilities; -import javax.validation.Valid; -import javax.validation.constraints.NotEmpty; import javax.ws.rs.client.Entity; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import com.bakdata.conquery.ConqueryConstants; -import com.bakdata.conquery.apiv1.query.concept.specific.external.DateColumn; -import com.bakdata.conquery.apiv1.query.concept.specific.external.IdColumn; import com.bakdata.conquery.apiv1.query.ConceptQuery; import com.bakdata.conquery.apiv1.query.Query; -import com.bakdata.conquery.apiv1.query.concept.specific.CQExternal; -import com.bakdata.conquery.apiv1.query.concept.specific.CQExternal.FormatColumn; +import com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal; import com.bakdata.conquery.integration.json.ConqueryTestSpec; import com.bakdata.conquery.io.jackson.Jackson; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.auth.permissions.AbilitySets; import com.bakdata.conquery.models.auth.permissions.ExecutionPermission; -import com.bakdata.conquery.models.datasets.Table; -import com.bakdata.conquery.models.datasets.concepts.Concept; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.datasets.SecondaryIdDescription; +import com.bakdata.conquery.models.datasets.Table; import com.bakdata.conquery.models.datasets.concepts.Concept; import com.bakdata.conquery.models.exceptions.JSONException; import com.bakdata.conquery.models.execution.ExecutionState; @@ -46,9 +39,6 @@ import com.bakdata.conquery.models.preproc.TableImportDescriptor; import com.bakdata.conquery.models.preproc.TableInputDescriptor; import com.bakdata.conquery.models.preproc.outputs.OutputDescription; -import com.bakdata.conquery.models.query.ExecutionManager; -import com.bakdata.conquery.apiv1.query.ConceptQuery; -import com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal; import com.bakdata.conquery.models.worker.SingletonNamespaceCollection; import com.bakdata.conquery.resources.ResourceConstants; import com.bakdata.conquery.resources.admin.rest.AdminDatasetResource; From cd4cdb6db49f6b4f657c6631823a88757b40cfed Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 7 Jul 2021 16:04:18 +0000 Subject: [PATCH 06/82] Update AutoDoc --- docs/Config JSON.md | 19 ++++++++++--------- docs/REST API JSONs.md | 21 +++++++++++---------- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/docs/Config JSON.md b/docs/Config JSON.md index 1e37d50489..264cd0fd45 100644 --- a/docs/Config JSON.md +++ b/docs/Config JSON.md @@ -141,7 +141,7 @@ An `IdMappingConfig` is used to define how multi column entity IDs are printed a Different types of IdMappingConfig can be used by setting `type` to one of the following values: -### NO_ID_MAPPING [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/NoIdMapping.java#L12) +### NO_ID_MAPPING [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/NoIdMapping.java#L11)
Details

@@ -256,7 +256,7 @@ Supported Fields: | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L68) | storage | `@Valid @NotNull StoreFactory` | | | |

-### Type CurrencyConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L20) +### Type CurrencyConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L98)
Details

@@ -267,13 +267,13 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L26) | decimalScale | `int` | `2` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L25) | decimalSeparator | `String` | `","` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L23) | prefix | `String` | `"€"` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L24) | thousandSeparator | `String` | `"."` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L103) | decimalScale | `int` | `2` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L102) | decimalSeparator | `String` | `","` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L100) | prefix | `String` | `"€"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L101) | thousandSeparator | `String` | `"."` | | |

-### Type FrontendConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L13) +### Type FrontendConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L25)
Details

@@ -284,8 +284,9 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L17) | currency | [CurrencyConfig](#Type-CurrencyConfig) | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L16) | version | `String` | `"0.0.0-SNAPSHOT"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L32) | currency | [CurrencyConfig](#Type-CurrencyConfig) | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L36) | queryUpload | `UploadConfig` | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L31) | version | `String` | `"0.0.0-SNAPSHOT"` | | |

### Type LocaleConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L11) diff --git a/docs/REST API JSONs.md b/docs/REST API JSONs.md index 81788ada58..ef647b37fc 100644 --- a/docs/REST API JSONs.md +++ b/docs/REST API JSONs.md @@ -509,8 +509,8 @@ Supported Fields: | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQElement.java#L31-L33) | label | `String` | ? | | Allows the user to define labels. |

-### EXTERNAL [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java#L34) - +### EXTERNAL [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java#L36-L38) +Allows uploading lists of entities.
Details

@@ -755,7 +755,7 @@ Supported Fields: | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptResource.java#L68) | concepts | list of `String` | `null` | | |

-### Type CurrencyConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L20) +### Type CurrencyConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L98)
Details

@@ -766,10 +766,10 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L26) | decimalScale | `int` | `2` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L25) | decimalSeparator | `String` | `","` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L23) | prefix | `String` | `"€"` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L24) | thousandSeparator | `String` | `"."` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L103) | decimalScale | `int` | `2` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L102) | decimalSeparator | `String` | `","` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L100) | prefix | `String` | `"€"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L101) | thousandSeparator | `String` | `"."` | | |

### Type ExecutionStatus [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/ExecutionStatus.java#L18) @@ -885,7 +885,7 @@ No fields can be set for this type.

-### Type FrontendConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L13) +### Type FrontendConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L25)
Details

@@ -896,8 +896,9 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L17) | currency | [CurrencyConfig](#Type-CurrencyConfig) | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L16) | version | `String` | `"0.0.0-SNAPSHOT"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L32) | currency | [CurrencyConfig](#Type-CurrencyConfig) | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L36) | queryUpload | `UploadConfig` | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L31) | version | `String` | `"0.0.0-SNAPSHOT"` | | |

### Type FullExecutionStatus [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/FullExecutionStatus.java#L20-L24) From ac83e180d946825630af081c6e9557126ab2b535 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Thu, 8 Jul 2021 12:10:17 +0200 Subject: [PATCH 07/82] Add Id kind as part of IdMapping resolve step, to dissambiguate between keys --- .../concept/specific/external/CQExternal.java | 39 ++++++------ .../concept/specific/external/IdColumn.java | 4 +- .../identifiable/mapping/EntityIdMap.java | 59 +++++++++++++++---- .../identifiable/mapping/IdMappingConfig.java | 5 +- .../admin/ui/DatasetsUIResource.java | 4 +- 5 files changed, 74 insertions(+), 37 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java index b9c7bacd9c..0728d7df07 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java @@ -102,40 +102,43 @@ public void resolve(QueryResolveContext context) { final DateFormats dateFormats = context.getConfig().getPreprocessor().getParsers().getDateFormats(); - List nonResolved = new ArrayList<>(); - + List unresolved = new ArrayList<>(); // ignore the first row, because this is the header for (int i = 1; i < values.length; i++) { final String[] row = values[i]; - final String externalId = idColumn.read(row); + final String[] externalId = idColumn.read(row); - //read the dates from the row - try { + final Optional id = mapping.toInternal(externalId); - CDateSet dates = dateFormat.readDates(dateColumns, row, dateFormats); + if (id.isEmpty()){ + continue; + } - Optional id = mapping.toInternal(externalId); + final int resolvedId; - int resolvedId; + if ((resolvedId = primary.getId(id.get())) == -1) { + unresolved.add(row); + continue; + } + + //read the dates from the row + try { + CDateSet dates = dateFormat.readDates(dateColumns, row, dateFormats); - if (id.isPresent() && (resolvedId = primary.getId(id.get())) != -1) { - valuesResolved.put(resolvedId, dates); - } - else { - nonResolved.add(row); - } + valuesResolved.put(resolvedId, dates); } catch (Exception e) { - log.warn("Failed to parse id from {}", row, e); + log.warn("Failed to parse Date from {}", row, e); + unresolved.add(row); } } - if (!nonResolved.isEmpty()) { + if (!unresolved.isEmpty()) { log.warn( "Could not resolve {} of the {} rows. Not resolved: {}", - nonResolved.size(), + unresolved.size(), values.length - 1, - nonResolved.subList(0, Math.min(nonResolved.size(), 10)) + unresolved.subList(0, Math.min(unresolved.size(), 10)) ); } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/IdColumn.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/IdColumn.java index 6572656e11..e90633ff69 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/IdColumn.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/IdColumn.java @@ -5,7 +5,7 @@ @CPSType(id = "ID", base = FormatColumn.class) public class IdColumn extends FormatColumn { - public String read(String[] row) { - return row[getPosition()]; + public String[] read(String[] row) { + return new String[]{row[getPosition()]}; } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java index 6e6d2b48ba..71f5a8b5be 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java @@ -1,14 +1,16 @@ package com.bakdata.conquery.models.identifiable.mapping; +import java.util.Collection; import java.util.HashMap; import java.util.Map; import java.util.Optional; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonValue; +import lombok.Data; import lombok.EqualsAndHashCode; import lombok.Getter; -import lombok.RequiredArgsConstructor; +import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; /** @@ -17,38 +19,64 @@ */ @Getter @EqualsAndHashCode -@RequiredArgsConstructor @Slf4j +@NoArgsConstructor public class EntityIdMap { + /** * The map from csv entity ids to external entity ids. */ - private final Map csvIdToExternalIdMap = new HashMap<>(); + private final Map internalToPrint = new HashMap<>(); /** * The map from external entity ids to csv entity ids. */ - private final Map externalIdPartCsvIdMap = new HashMap<>(); + private final Map external2Internal = new HashMap<>(); + + + @Data + protected static class Container { + private final Collection> external2Internal; + private final Map internalToPrint; + } + + @JsonCreator + protected EntityIdMap fromContainer(Container container) { + final EntityIdMap out = new EntityIdMap(); + + for (Map.Entry entry : container.getExternal2Internal()) { + out.getExternal2Internal().put(entry.getKey(), entry.getValue()); + } + + out.getInternalToPrint().putAll(container.getInternalToPrint()); + + return out; + } + + @JsonValue + protected Container getContainer() { + return new Container(external2Internal.entrySet(), internalToPrint); + } /** * Map an internal id to an external. */ public EntityPrintId toExternal(String internal) { - return csvIdToExternalIdMap.get(internal); + return internalToPrint.get(internal); } /** - * Map an external to an internal id. + * Resolve an external to an internal id. * @param external */ - public Optional toInternal(String external) { - return Optional.ofNullable(externalIdPartCsvIdMap.get(external)); + public Optional toInternal(String... external) { + return Optional.ofNullable(external2Internal.get(new ExternalId(external))); } public void addOutputMapping(String csvEntityId, EntityPrintId externalEntityId) { - final EntityPrintId prior = csvIdToExternalIdMap.put(csvEntityId, externalEntityId); + final EntityPrintId prior = internalToPrint.put(csvEntityId, externalEntityId); if (prior != null && prior.equals(externalEntityId)) { log.warn("Duplicate mapping for {} to {} and {}", csvEntityId, externalEntityId, prior); @@ -56,13 +84,18 @@ public void addOutputMapping(String csvEntityId, EntityPrintId externalEntityId) } - public void addInputMapping(String csvEntityId, String externalEntityId) { + public void addInputMapping(String csvEntityId, String... externalEntityId) { - final String prior = externalIdPartCsvIdMap.put(externalEntityId, csvEntityId); + final String prior = external2Internal.put(new ExternalId(externalEntityId), csvEntityId); if (prior != null && prior.equals(csvEntityId)) { log.warn("Duplicate mapping for {} to {} and {}", externalEntityId, csvEntityId, prior); } } + @Data + private static class ExternalId { + private final String[] parts; + } + } diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingConfig.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingConfig.java index 20567851f1..d0985f7ee7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingConfig.java @@ -24,8 +24,6 @@ public EntityIdMap generateIdMapping(CsvParser parser) throws IllegalArgumentExc EntityIdMap mapping = new EntityIdMap(); - //TODO Check headers match parser.getContext().headers(); - Record record; while((record = parser.parseNextRecord()) != null){ @@ -37,6 +35,9 @@ public EntityIdMap generateIdMapping(CsvParser parser) throws IllegalArgumentExc return mapping; } + /** + * Process a single record and generate all found ids. + */ protected abstract void processRecord(Record record, String id, EntityIdMap mapping); @JsonIgnore diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/DatasetsUIResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/DatasetsUIResource.java index e0d12a0613..9c9c5409b4 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/DatasetsUIResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/DatasetsUIResource.java @@ -114,8 +114,8 @@ imp, getNamespace().getStorage().getAllConcepts() @Path("mapping") public View getIdMapping() { EntityIdMap mapping = namespace.getStorage().getIdMapping(); - if (mapping != null && mapping.getCsvIdToExternalIdMap() != null) { - return new UIView<>("idmapping.html.ftl", uiProcessor.getUIContext(), mapping.getCsvIdToExternalIdMap()); + if (mapping != null && mapping.getInternalToPrint() != null) { + return new UIView<>("idmapping.html.ftl", uiProcessor.getUIContext(), mapping.getInternalToPrint()); } return new UIView<>("add_idmapping.html.ftl", uiProcessor.getUIContext(), namespace.getDataset().getId()); } From df98f556dbe2ceebd4b08a5dbdfd984846030a4d Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Thu, 8 Jul 2021 17:29:37 +0200 Subject: [PATCH 08/82] don't use weird reflection magic --- .../concept/specific/external/CQExternal.java | 104 +++++++++--------- .../concept/specific/external/DateColumn.java | 29 +++-- .../concept/specific/external/DateFormat.java | 59 +++++----- .../specific/external/FormatColumn.java | 37 +------ .../concept/specific/external/IdColumn.java | 11 +- .../specific/external/IgnoreColumn.java | 3 +- .../models/config/FrontendConfig.java | 46 ++++---- .../conquery/models/config/NoIdMapping.java | 3 + .../models/config/SimpleIdMapping.java | 26 +++-- .../identifiable/mapping/IdMappingConfig.java | 24 +++- 10 files changed, 170 insertions(+), 172 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java index 0728d7df07..7e812eef3b 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java @@ -2,17 +2,14 @@ import java.util.ArrayList; import java.util.Arrays; -import java.util.Date; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import javax.validation.constraints.NotEmpty; import com.bakdata.conquery.apiv1.query.CQElement; import com.bakdata.conquery.io.cps.CPSType; -import com.bakdata.conquery.io.cps.CPSTypeIdResolver; import com.bakdata.conquery.io.jackson.InternalOnly; import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.dictionary.EncodedDictionary; @@ -26,12 +23,11 @@ import com.bakdata.conquery.models.query.resultinfo.ResultInfoCollector; import com.bakdata.conquery.util.DateFormats; import com.fasterxml.jackson.annotation.JsonIgnore; -import com.google.common.collect.MoreCollectors; import io.dropwizard.validation.ValidationMethod; import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; import lombok.Getter; -import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; /** * Allows uploading lists of entities. @@ -71,47 +67,34 @@ public QPNode createQueryPlan(QueryPlanContext context, ConceptQueryPlan plan) { @Override public void resolve(QueryResolveContext context) { - - List resolvedFormats = instantiateFormatIds(format); - - - for (int index = 0; index < resolvedFormats.size(); index++) { - resolvedFormats.get(index).setPosition(index); - } - valuesResolved = new Int2ObjectOpenHashMap<>(); - final DateColumn[] dateColumns = resolvedFormats.stream() - .filter(DateColumn.class::isInstance) - .map(DateColumn.class::cast) - .toArray(DateColumn[]::new); - //TODO verify dateColumns match + final EntityIdMap mapping = context.getNamespace().getStorage().getIdMapping(); + + final Map stringFormatColumnMap = context.getConfig().getIdMapping().getFormatColumns(); - final DateFormat dateFormat = dateColumns.length > 0 ? dateColumns[0].getFormat() : DateFormat.ALL; + final FormatColumn[] resolvedFormats = resolveFormats(stringFormatColumnMap); + final DateFormat dateFormat = getDateFormat(resolvedFormats); - final IdColumn idColumn = resolvedFormats.stream() - .filter(IdColumn.class::isInstance) - .map(IdColumn.class::cast) - .collect(MoreCollectors.onlyElement()); + final int idIndex = getIdIndex(resolvedFormats); + final IdColumn idColumn = (IdColumn) resolvedFormats[idIndex]; final EncodedDictionary primary = context.getNamespace().getStorage().getPrimaryDictionary(); - final EntityIdMap mapping = context.getNamespace().getStorage().getIdMapping(); final DateFormats dateFormats = context.getConfig().getPreprocessor().getParsers().getDateFormats(); - List unresolved = new ArrayList<>(); // ignore the first row, because this is the header for (int i = 1; i < values.length; i++) { final String[] row = values[i]; - final String[] externalId = idColumn.read(row); + final String[] externalId = idColumn.read(row, idIndex); final Optional id = mapping.toInternal(externalId); - if (id.isEmpty()){ + if (id.isEmpty()) { continue; } @@ -124,7 +107,7 @@ public void resolve(QueryResolveContext context) { //read the dates from the row try { - CDateSet dates = dateFormat.readDates(dateColumns, row, dateFormats); + CDateSet dates = dateFormat.readDates(resolvedFormats, row, dateFormats); valuesResolved.put(resolvedId, dates); } @@ -133,6 +116,7 @@ public void resolve(QueryResolveContext context) { unresolved.add(row); } } + if (!unresolved.isEmpty()) { log.warn( "Could not resolve {} of the {} rows. Not resolved: {}", @@ -147,41 +131,53 @@ public void resolve(QueryResolveContext context) { } } - /** - * Helper method to flatten API surface, allowing plain passing of type-ids instead of using objects of only type. - */ - @SneakyThrows - private static List instantiateFormatIds(@NotEmpty List formats) { - List resolvedFormats = new ArrayList<>(); - for (String s : formats) { - - Class clazz = CPSTypeIdResolver.getImplementation(FormatColumn.class, s); - // We've already established at startup, that this should not fail. - FormatColumn newInstance = clazz.getConstructor().newInstance(); - resolvedFormats.add(newInstance); + @NotNull + private DateFormat getDateFormat(FormatColumn[] resolvedFormats) { + DateFormat dateFormat = null; + + for (FormatColumn col : resolvedFormats) { + if (!(col instanceof DateColumn)) { + continue; + } + + if (dateFormat != null && !dateFormat.equals(((DateColumn) col).getFormat())) { + throw new IllegalStateException("Use of multiple Date Formats."); + } + + dateFormat = ((DateColumn) col).getFormat(); } - return resolvedFormats; + return dateFormat == null ? DateFormat.ALL : dateFormat; } - @Override - public void collectResultInfos(ResultInfoCollector collector) { + private int getIdIndex(FormatColumn[] resolvedFormats) { + for (int index = 0; index < resolvedFormats.length; index++) { + if (resolvedFormats[index] instanceof IdColumn) { + return index; + } + } + + throw new IllegalStateException("No IdColumn provided"); } + @NotNull + private FormatColumn[] resolveFormats(Map stringFormatColumnMap) { + final List resolvedFormats = new ArrayList<>(); - @JsonIgnore - @ValidationMethod(message = "Must contain only valid FormatColumn Ids.") - public boolean isValidFormatIds() { - return format.stream().map(id -> CPSTypeIdResolver.getImplementation(FormatColumn.class, id)).noneMatch(Objects::isNull); + for (String columnType : format) { + FormatColumn formatColumn = stringFormatColumnMap.get(columnType); + + if (formatColumn == null) { + throw new IllegalStateException(String.format("Don't know of format %s", columnType)); + } + + resolvedFormats.add(formatColumn); + } + return resolvedFormats.toArray(FormatColumn[]::new); } - @JsonIgnore - @ValidationMethod(message = "Must use one IdColumn.") - public boolean isOnlyOneIdColumn() { - return format.stream() - .map(id -> CPSTypeIdResolver.getImplementation(FormatColumn.class, id)) - .filter(IdColumn.class::isAssignableFrom) - .count() == 1; + @Override + public void collectResultInfos(ResultInfoCollector collector) { } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateColumn.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateColumn.java index ec752f10d0..679189365c 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateColumn.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateColumn.java @@ -1,49 +1,58 @@ package com.bakdata.conquery.apiv1.query.concept.specific.external; import com.bakdata.conquery.io.cps.CPSType; +import lombok.AllArgsConstructor; import lombok.Getter; +import lombok.With; @Getter public abstract class DateColumn extends FormatColumn { private final DateFormat format; - public DateColumn(DateFormat format) { + protected DateColumn(DateFormat format) { this.format = format; } - @CPSType(id = "DATE_SET", base = FormatColumn.class) + public static class DateSet extends DateColumn { + public static String HANDLE = "DATE_SET"; + public DateSet() { super(DateFormat.DATE_SET); } } - @CPSType(id = "DATE_RANGE", base = FormatColumn.class) public static class DateRange extends DateColumn { + public static String HANDLE = "DATE_RANGE"; + public DateRange() { super(DateFormat.DATE_RANGE); } } - @CPSType(id = "EVENT_DATE", base = FormatColumn.class) public static class EventDate extends DateColumn { + public static String HANDLE = "EVENT_DATE"; + + public EventDate() { super(DateFormat.EVENT_DATE); } } - @CPSType(id = "START_DATE", base = FormatColumn.class) - public static class DateStart extends DateColumn { - public DateStart() { + public static class StartDate extends DateColumn { + public static String HANDLE = "START_DATE"; + + public StartDate() { super(DateFormat.START_END_DATE); } } - @CPSType(id = "END_DATE", base = FormatColumn.class) - public static class DateEnd extends DateColumn { - public DateEnd() { + public static class EndDate extends DateColumn { + public static String HANDLE = "END_DATE"; + + public EndDate() { super(DateFormat.START_END_DATE); } } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java index cd4c15dbe7..5a0253b276 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java @@ -3,44 +3,35 @@ import java.time.LocalDate; import java.util.Arrays; import java.util.Collections; +import java.util.Enumeration; +import com.bakdata.conquery.apiv1.forms.Form; import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.preproc.parser.specific.DateRangeParser; import com.bakdata.conquery.util.DateFormats; import com.google.common.base.Preconditions; import com.google.common.collect.MoreCollectors; +import org.apache.commons.collections.EnumerationUtils; public enum DateFormat { EVENT_DATE { @Override - public CDateSet readDates(DateColumn[] dateIndices, String[] row, DateFormats dateFormats) { - Preconditions.checkArgument(dateIndices.length == 1); + public CDateSet readDates(FormatColumn[] formats, String[] row, DateFormats dateFormats) { + final int index = findIndex(formats, DateColumn.EventDate.class); - return CDateSet.create(Collections.singleton(CDateRange.exactly(dateFormats.parseToLocalDate(row[dateIndices[0].getPosition()])))); + return CDateSet.create(Collections.singleton(CDateRange.exactly(dateFormats.parseToLocalDate(row[index])))); } }, START_END_DATE { @Override - public CDateSet readDates(DateColumn[] dateIndices, String[] row, DateFormats dateFormats) { - Preconditions.checkArgument(dateIndices.length == 1 || dateIndices.length == 2); + public CDateSet readDates(FormatColumn[] formats, String[] row, DateFormats dateFormats) { + final int startIndex = findIndex(formats, DateColumn.StartDate.class); + final int endIndex = findIndex(formats, DateColumn.EndDate.class); - LocalDate start = Arrays.stream(dateIndices) - .filter(DateColumn.DateStart.class::isInstance) - .map(DateColumn::getPosition) - .map(idx -> row[idx]) - .map(dateFormats::parseToLocalDate) - .collect(MoreCollectors.toOptional()) - .orElse(null); - - LocalDate end = Arrays.stream(dateIndices) - .filter(DateColumn.DateEnd.class::isInstance) - .map(DateColumn::getPosition) - .map(idx -> row[idx]) - .map(dateFormats::parseToLocalDate) - .collect(MoreCollectors.toOptional()) - .orElse(null); + LocalDate start = dateFormats.parseToLocalDate(row[startIndex]); + LocalDate end = dateFormats.parseToLocalDate(row[endIndex]); if (start == null && end == null) { return null; @@ -51,28 +42,36 @@ public CDateSet readDates(DateColumn[] dateIndices, String[] row, DateFormats da }, DATE_RANGE { @Override - public CDateSet readDates(DateColumn[] dateIndices, String[] row, DateFormats dateFormats) { - Preconditions.checkArgument(dateIndices.length == 1); + public CDateSet readDates(FormatColumn[] formats, String[] row, DateFormats dateFormats) { + final int index = findIndex(formats, DateColumn.DateRange.class); - return CDateSet.create(Collections.singleton(DateRangeParser.parseISORange(row[dateIndices[0].getPosition()], dateFormats))); + return CDateSet.create(Collections.singleton(DateRangeParser.parseISORange(row[index], dateFormats))); } }, DATE_SET { @Override - public CDateSet readDates(DateColumn[] dateIndices, String[] row, DateFormats dateFormats) { - Preconditions.checkArgument(dateIndices.length == 1); + public CDateSet readDates(FormatColumn[] formats, String[] row, DateFormats dateFormats) { + final int index = findIndex(formats, DateColumn.DateSet.class); - return CDateSet.parse(row[dateIndices[0].getPosition()], dateFormats); + return CDateSet.parse(row[index], dateFormats); } }, ALL { @Override - public CDateSet readDates(DateColumn[] dateIndices, String[] row, DateFormats dateFormats) { - Preconditions.checkArgument(dateIndices.length == 0); - + public CDateSet readDates(FormatColumn[] formats, String[] row, DateFormats dateFormats) { return CDateSet.createFull(); } }; - public abstract CDateSet readDates(DateColumn[] dateIndices, String[] row, DateFormats dateFormats); + public abstract CDateSet readDates(FormatColumn[] formats, String[] row, DateFormats dateFormats); + + protected static int findIndex(FormatColumn[] formats, Class clazz){ + for (int index = 0; index < formats.length; index++) { + if (clazz.isInstance(formats[index])) { + return index; + } + } + + return -1; + } } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/FormatColumn.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/FormatColumn.java index 83b8701c56..27f769c8e8 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/FormatColumn.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/FormatColumn.java @@ -1,43 +1,10 @@ package com.bakdata.conquery.apiv1.query.concept.specific.external; -import java.lang.reflect.InvocationTargetException; -import java.util.Set; - -import com.bakdata.conquery.apiv1.forms.Form; -import com.bakdata.conquery.io.cps.CPSBase; -import com.bakdata.conquery.io.cps.CPSType; -import com.bakdata.conquery.io.cps.CPSTypeIdResolver; -import com.bakdata.conquery.io.jackson.InternalOnly; -import com.fasterxml.jackson.annotation.JsonTypeInfo; import lombok.Data; -import lombok.Getter; -import lombok.Setter; +import lombok.With; import lombok.extern.slf4j.Slf4j; -@CPSBase -@JsonTypeInfo(use = JsonTypeInfo.Id.CUSTOM, property = "type") -@Data -@Slf4j -public abstract class FormatColumn { - - @InternalOnly @Setter @Getter - private int position; - static { - // For each implementation of FormatColumn, test if we can instantiate it using default constructor. Fail start-up if not possible. - final Set> implementations = CPSTypeIdResolver.listImplementations(FormatColumn.class); - - for (Class impl : implementations) { - final String id = impl.getAnnotation(CPSType.class).id(); - try { - // Try and get no-args constructor - impl.getConstructor().newInstance(); - } - catch (Exception e) { - log.error("FormatColumn {} has no default constructor.", id); - throw new RuntimeException(e); - } - } - } +public abstract class FormatColumn { } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/IdColumn.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/IdColumn.java index e90633ff69..1877249878 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/IdColumn.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/IdColumn.java @@ -1,11 +1,14 @@ package com.bakdata.conquery.apiv1.query.concept.specific.external; -import com.bakdata.conquery.io.cps.CPSType; +import lombok.AllArgsConstructor; +import lombok.Value; +import lombok.With; -@CPSType(id = "ID", base = FormatColumn.class) public class IdColumn extends FormatColumn { - public String[] read(String[] row) { - return new String[]{row[getPosition()]}; + public static String HANDLE = "ID"; + + public String[] read(String[] row, int position) { + return new String[]{row[position]}; } } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/IgnoreColumn.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/IgnoreColumn.java index 5ef653ab12..4a1c314d84 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/IgnoreColumn.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/IgnoreColumn.java @@ -2,9 +2,10 @@ import com.bakdata.conquery.io.cps.CPSType; -@CPSType(id = "IGNORE", base = FormatColumn.class) public class IgnoreColumn extends FormatColumn { + public static final String HANDLE = "IGNORED"; + public String read(String[] row) { return null; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java index 7dbba4645a..dc958d672f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java @@ -14,7 +14,9 @@ import com.bakdata.conquery.apiv1.query.concept.specific.external.IdColumn; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.cps.CPSTypeIdResolver; +import com.bakdata.conquery.models.identifiable.mapping.IdMappingConfig; import com.bakdata.conquery.util.VersionInfo; +import com.fasterxml.jackson.annotation.JsonBackReference; import com.fasterxml.jackson.annotation.JsonIgnore; import groovy.transform.ToString; import io.dropwizard.validation.ValidationMethod; @@ -27,7 +29,6 @@ @Setter public class FrontendConfig { - private String version = VersionInfo.INSTANCE.getProjectVersion(); @Valid @NotNull @@ -35,56 +36,49 @@ public class FrontendConfig { private UploadConfig queryUpload = new UploadConfig(); - @Data + @Getter @Setter public static class UploadConfig { - private final static List DEFAULT_IDS = new ArrayList<>(); - - - static { - // Collect all implementations of IdClass. - - final Set> idClasses = CPSTypeIdResolver.listImplementations(FormatColumn.class); - - for (Class idClass : idClasses) { - - if (!IdColumn.class.isAssignableFrom(idClass)) { - continue; - } - - final String id = idClass.getAnnotation(CPSType.class).id(); - DEFAULT_IDS.add(new ColumnConfig(id, Map.of("en", id), Map.of("en", id))); - } - } @NotEmpty - private List ids = DEFAULT_IDS; + private List ids = List.of(new ColumnConfig(IdColumn.HANDLE, Map.of("en", "Id"), Map.of("en", "Id of the Entity."))); @NotNull private ColumnConfig dateStart = - new ColumnConfig(DateColumn.DateStart.class.getAnnotation(CPSType.class).id(), Map.of("en", "Begin"), Map.of("en", "Begin of Date")); + new ColumnConfig(DateColumn.StartDate.HANDLE, Map.of("en", "Begin"), Map.of("en", "Begin of Date-range")); @NotNull private ColumnConfig dateEnd = - new ColumnConfig(DateColumn.DateEnd.class.getAnnotation(CPSType.class).id(), Map.of("en", "Begin"), Map.of("en", "Begin of Date")); + new ColumnConfig(DateColumn.EndDate.HANDLE, Map.of("en", "Begin"), Map.of("en", "End of Date-range")); @NotNull private ColumnConfig dateRange = - new ColumnConfig(DateColumn.DateRange.class.getAnnotation(CPSType.class).id(), Map.of("en", "Date Range"), Map.of("en", "Full Date Range")); + new ColumnConfig(DateColumn.DateRange.HANDLE, Map.of("en", "Date Range"), Map.of("en", "Full Date Range")); @NotNull private ColumnConfig dateSet = - new ColumnConfig(DateColumn.DateSet.class.getAnnotation(CPSType.class).id(), Map.of("en", "Dateset"), Map.of("en", "Set of Date-Ranges")); + new ColumnConfig(DateColumn.DateSet.HANDLE, Map.of("en", "Dateset"), Map.of("en", "Set of Date-Ranges")); @NotNull private ColumnConfig eventDate = - new ColumnConfig(DateColumn.EventDate.class.getAnnotation(CPSType.class).id(), Map.of("en", "Event Date"), Map.of("en", "Single day")); + new ColumnConfig(DateColumn.EventDate.HANDLE, Map.of("en", "Event Date"), Map.of("en", "Single event")); @Data public static class ColumnConfig { + + /** + * Name of the Column, used to resolve the specific entry at {@link IdMappingConfig#getFormatColumns()} + */ @NotEmpty private final String name; + /** + * Map of Localized labels. + */ private final Map label; + + /** + * Map of Localized description. + */ private final Map description; @JsonIgnore diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/NoIdMapping.java b/backend/src/main/java/com/bakdata/conquery/models/config/NoIdMapping.java index b6cd730c2a..394ba14338 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/NoIdMapping.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/NoIdMapping.java @@ -8,6 +8,9 @@ import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; import com.univocity.parsers.common.record.Record; +/** + * Disables Id-Mapping completely. + */ @CPSType(base = IdMappingConfig.class, id = "NO_ID_MAPPING") public class NoIdMapping extends IdMappingConfig { private static final String[] HEADER = new String[]{"result"}; diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/SimpleIdMapping.java b/backend/src/main/java/com/bakdata/conquery/models/config/SimpleIdMapping.java index 4355276bcc..f85eeb104f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/SimpleIdMapping.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/SimpleIdMapping.java @@ -1,7 +1,10 @@ package com.bakdata.conquery.models.config; -import java.util.List; +import java.util.HashMap; +import java.util.Map; +import com.bakdata.conquery.apiv1.query.concept.specific.external.FormatColumn; +import com.bakdata.conquery.apiv1.query.concept.specific.external.IdColumn; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; import com.bakdata.conquery.models.identifiable.mapping.EntityPrintId; @@ -9,12 +12,19 @@ import com.univocity.parsers.common.record.Record; /** - * Identity mapping. - * - * TODO should this actually map to a separate column? + * Maps input to itself for upload. */ @CPSType(base = IdMappingConfig.class, id = "SIMPLE") -public class SimpleIdMapping extends IdMappingConfig { +public class SimpleIdMapping extends NoIdMapping { + + @Override + public Map getFormatColumns() { + final HashMap out = new HashMap<>(super.getFormatColumns()); + + out.put(IdColumn.HANDLE, new IdColumn()); + + return out; + } @Override protected void processRecord(Record record, String id, EntityIdMap mapping) { @@ -26,10 +36,4 @@ protected void processRecord(Record record, String id, EntityIdMap mapping) { mapping.addInputMapping(id, id); } - - @Override - public List getPrintIdFields() { - return List.of("result"); - } - } diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingConfig.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingConfig.java index d0985f7ee7..81495d0e86 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingConfig.java @@ -2,6 +2,9 @@ import java.util.*; +import com.bakdata.conquery.apiv1.query.concept.specific.external.DateColumn; +import com.bakdata.conquery.apiv1.query.concept.specific.external.FormatColumn; +import com.bakdata.conquery.apiv1.query.concept.specific.external.IgnoreColumn; import com.bakdata.conquery.io.cps.CPSBase; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.execution.ManagedExecution; @@ -20,7 +23,23 @@ @JsonTypeInfo(use = JsonTypeInfo.Id.CUSTOM, property = "type") public abstract class IdMappingConfig { - public EntityIdMap generateIdMapping(CsvParser parser) throws IllegalArgumentException { + @JsonIgnore + public Map getFormatColumns() { + return Map.of( + DateColumn.DateSet.HANDLE, new DateColumn.DateSet(), + DateColumn.DateRange.HANDLE, new DateColumn.DateRange(), + DateColumn.EventDate.HANDLE, new DateColumn.EventDate(), + DateColumn.StartDate.HANDLE, new DateColumn.StartDate(), + DateColumn.EndDate.HANDLE, new DateColumn.EndDate(), + IgnoreColumn.HANDLE, new IgnoreColumn() + ); + } + + + /** + * Read incoming CSV-file extracting Id-Mappings for in and Output. + */ + public EntityIdMap generateIdMapping(CsvParser parser) { EntityIdMap mapping = new EntityIdMap(); @@ -40,6 +59,9 @@ public EntityIdMap generateIdMapping(CsvParser parser) throws IllegalArgumentExc */ protected abstract void processRecord(Record record, String id, EntityIdMap mapping); + /** + * Headers for Output CSV. + */ @JsonIgnore public abstract List getPrintIdFields(); From 5c51afa4237971a5e190a6249931d184ec188f15 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 8 Jul 2021 15:32:15 +0000 Subject: [PATCH 09/82] Update AutoDoc --- docs/Config JSON.md | 26 +++++++++++++------------- docs/REST API JSONs.md | 20 ++++++++++---------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/docs/Config JSON.md b/docs/Config JSON.md index 264cd0fd45..b82a65e60c 100644 --- a/docs/Config JSON.md +++ b/docs/Config JSON.md @@ -141,8 +141,8 @@ An `IdMappingConfig` is used to define how multi column entity IDs are printed a Different types of IdMappingConfig can be used by setting `type` to one of the following values: -### NO_ID_MAPPING [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/NoIdMapping.java#L11) - +### NO_ID_MAPPING [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/NoIdMapping.java#L11-L13) +Disables Id-Mapping completely.
Details

@@ -152,8 +152,8 @@ No fields can be set for this type.

-### SIMPLE [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/SimpleIdMapping.java#L11-L15) -Identity mapping. TODO should this actually map to a separate column? +### SIMPLE [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/SimpleIdMapping.java#L14-L16) +Maps input to itself for upload.
Details

@@ -256,7 +256,7 @@ Supported Fields: | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L68) | storage | `@Valid @NotNull StoreFactory` | | | |

-### Type CurrencyConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L98) +### Type CurrencyConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L92)
Details

@@ -267,13 +267,13 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L103) | decimalScale | `int` | `2` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L102) | decimalSeparator | `String` | `","` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L100) | prefix | `String` | `"€"` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L101) | thousandSeparator | `String` | `"."` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L97) | decimalScale | `int` | `2` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L96) | decimalSeparator | `String` | `","` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L94) | prefix | `String` | `"€"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L95) | thousandSeparator | `String` | `"."` | | |

-### Type FrontendConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L25) +### Type FrontendConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L27)
Details

@@ -284,9 +284,9 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L32) | currency | [CurrencyConfig](#Type-CurrencyConfig) | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L36) | queryUpload | `UploadConfig` | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L31) | version | `String` | `"0.0.0-SNAPSHOT"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L33) | currency | [CurrencyConfig](#Type-CurrencyConfig) | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L37) | queryUpload | `UploadConfig` | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L32) | version | `String` | `"0.0.0-SNAPSHOT"` | | |

### Type LocaleConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L11) diff --git a/docs/REST API JSONs.md b/docs/REST API JSONs.md index ef647b37fc..536882392a 100644 --- a/docs/REST API JSONs.md +++ b/docs/REST API JSONs.md @@ -509,7 +509,7 @@ Supported Fields: | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQElement.java#L31-L33) | label | `String` | ? | | Allows the user to define labels. |

-### EXTERNAL [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java#L36-L38) +### EXTERNAL [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java#L32-L34) Allows uploading lists of entities.
Details

@@ -755,7 +755,7 @@ Supported Fields: | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptResource.java#L68) | concepts | list of `String` | `null` | | |

-### Type CurrencyConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L98) +### Type CurrencyConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L92)
Details

@@ -766,10 +766,10 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L103) | decimalScale | `int` | `2` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L102) | decimalSeparator | `String` | `","` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L100) | prefix | `String` | `"€"` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L101) | thousandSeparator | `String` | `"."` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L97) | decimalScale | `int` | `2` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L96) | decimalSeparator | `String` | `","` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L94) | prefix | `String` | `"€"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L95) | thousandSeparator | `String` | `"."` | | |

### Type ExecutionStatus [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/ExecutionStatus.java#L18) @@ -885,7 +885,7 @@ No fields can be set for this type.

-### Type FrontendConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L25) +### Type FrontendConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L27)
Details

@@ -896,9 +896,9 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L32) | currency | [CurrencyConfig](#Type-CurrencyConfig) | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L36) | queryUpload | `UploadConfig` | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L31) | version | `String` | `"0.0.0-SNAPSHOT"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L33) | currency | [CurrencyConfig](#Type-CurrencyConfig) | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L37) | queryUpload | `UploadConfig` | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L32) | version | `String` | `"0.0.0-SNAPSHOT"` | | |

### Type FullExecutionStatus [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/FullExecutionStatus.java#L20-L24) From 6b1685aac6c775df3233321e41859a6a34f1aaee Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Thu, 15 Jul 2021 17:04:05 +0200 Subject: [PATCH 10/82] Don't use fancy CPS instead define mappings in config --- .../java/com/bakdata/conquery/Conquery.java | 2 - .../concept/specific/external/CQExternal.java | 92 ++-------- .../concept/specific/external/DateFormat.java | 49 ++--- .../serializer/CDateRangeDeserializer.java | 10 +- .../serializer/CDateSetDeserializer.java | 6 +- .../serializer/FormatedDateDeserializer.java | 6 +- .../conquery/io/storage/NamespaceStorage.java | 7 + .../conquery/models/common/CDateSet.java | 8 +- .../models/common/daterange/CDateRange.java | 3 - .../models/config/ConqueryConfig.java | 8 +- .../conquery/models/config/NoIdMapping.java | 5 - .../conquery/models/config/ParserConfig.java | 26 +-- .../models/config/SimpleIdMapping.java | 19 -- .../identifiable/mapping/EntityIdMap.java | 15 +- .../identifiable/mapping/IdMappingConfig.java | 170 +++++++++++++++--- .../conquery/models/preproc/Preprocessor.java | 8 +- .../models/preproc/outputs/CopyOutput.java | 4 +- .../preproc/outputs/DateRangeOutput.java | 8 +- .../preproc/outputs/EpochDateRangeOutput.java | 4 +- .../models/preproc/outputs/EpochOutput.java | 4 +- .../models/preproc/outputs/LineOutput.java | 4 +- .../models/preproc/outputs/NullOutput.java | 4 +- .../preproc/outputs/OutputDescription.java | 6 +- .../preproc/parser/specific/DateParser.java | 8 +- .../parser/specific/DateRangeParser.java | 16 +- .../{DateFormats.java => DateReader.java} | 31 ++-- .../json/AbstractQueryEngineTest.java | 1 - 27 files changed, 268 insertions(+), 256 deletions(-) rename backend/src/main/java/com/bakdata/conquery/util/{DateFormats.java => DateReader.java} (85%) diff --git a/backend/src/main/java/com/bakdata/conquery/Conquery.java b/backend/src/main/java/com/bakdata/conquery/Conquery.java index d8de33ae49..af84bae5d4 100644 --- a/backend/src/main/java/com/bakdata/conquery/Conquery.java +++ b/backend/src/main/java/com/bakdata/conquery/Conquery.java @@ -15,11 +15,9 @@ import com.bakdata.conquery.io.jackson.Jackson; import com.bakdata.conquery.io.jackson.MutableInjectableValues; import com.bakdata.conquery.models.config.ConqueryConfig; -import com.bakdata.conquery.util.DateFormats; import com.bakdata.conquery.util.UrlRewriteBundle; import io.dropwizard.Application; import io.dropwizard.ConfiguredBundle; -import io.dropwizard.configuration.EnvironmentVariableSubstitutor; import io.dropwizard.configuration.JsonConfigurationFactory; import io.dropwizard.configuration.SubstitutingSourceProvider; import io.dropwizard.servlets.assets.AssetServlet; diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java index 7e812eef3b..88e90c99b1 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java @@ -4,7 +4,6 @@ import java.util.Arrays; import java.util.List; import java.util.Map; -import java.util.Optional; import javax.validation.constraints.NotEmpty; @@ -12,18 +11,19 @@ import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.jackson.InternalOnly; import com.bakdata.conquery.models.common.CDateSet; -import com.bakdata.conquery.models.dictionary.EncodedDictionary; import com.bakdata.conquery.models.error.ConqueryError; import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; +import com.bakdata.conquery.models.identifiable.mapping.IdMappingConfig; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; import com.bakdata.conquery.models.query.queryplan.ConceptQueryPlan; import com.bakdata.conquery.models.query.queryplan.QPNode; import com.bakdata.conquery.models.query.queryplan.specific.ExternalNode; import com.bakdata.conquery.models.query.resultinfo.ResultInfoCollector; -import com.bakdata.conquery.util.DateFormats; +import com.bakdata.conquery.util.DateReader; import com.fasterxml.jackson.annotation.JsonIgnore; import io.dropwizard.validation.ValidationMethod; +import it.unimi.dsi.fastutil.ints.Int2ObjectMap; import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -69,52 +69,40 @@ public QPNode createQueryPlan(QueryPlanContext context, ConceptQueryPlan plan) { public void resolve(QueryResolveContext context) { valuesResolved = new Int2ObjectOpenHashMap<>(); - final EntityIdMap mapping = context.getNamespace().getStorage().getIdMapping(); - final Map stringFormatColumnMap = context.getConfig().getIdMapping().getFormatColumns(); - - final FormatColumn[] resolvedFormats = resolveFormats(stringFormatColumnMap); + final DateReader dateReader = context.getConfig().getPreprocessor().getParsers().getDateReader(); - final DateFormat dateFormat = getDateFormat(resolvedFormats); + final IdMappingConfig mappingConfig = context.getConfig().getIdMapping(); - final int idIndex = getIdIndex(resolvedFormats); - final IdColumn idColumn = (IdColumn) resolvedFormats[idIndex]; + // extract dates from rows + final Int2ObjectMap rowDates = mappingConfig.readDates(values, format, dateReader); + final int idIndex = mappingConfig.getIdIndex(format); - final EncodedDictionary primary = context.getNamespace().getStorage().getPrimaryDictionary(); - final DateFormats dateFormats = context.getConfig().getPreprocessor().getParsers().getDateFormats(); + final IdMappingConfig.InputMapper reader = mappingConfig.getIdMapper(format.get(idIndex)); - List unresolved = new ArrayList<>(); + final List unresolved = new ArrayList<>(); // ignore the first row, because this is the header - for (int i = 1; i < values.length; i++) { - final String[] row = values[i]; - final String[] externalId = idColumn.read(row, idIndex); + for (int rowNum = 1; rowNum < values.length; rowNum++) { + final String[] row = values[rowNum]; + final String externalId = reader.read(row[idIndex]); - final Optional id = mapping.toInternal(externalId); + final int resolvedId = mapping.resolve(reader.getName(), externalId); - if (id.isEmpty()) { + if (resolvedId == -1) { + unresolved.add(row); continue; } - final int resolvedId; - - if ((resolvedId = primary.getId(id.get())) == -1) { + if (!rowDates.containsKey(rowNum)) { unresolved.add(row); continue; } //read the dates from the row - try { - CDateSet dates = dateFormat.readDates(resolvedFormats, row, dateFormats); - - valuesResolved.put(resolvedId, dates); - } - catch (Exception e) { - log.warn("Failed to parse Date from {}", row, e); - unresolved.add(row); - } + valuesResolved.put(resolvedId, rowDates.get(rowNum)); } if (!unresolved.isEmpty()) { @@ -131,50 +119,6 @@ public void resolve(QueryResolveContext context) { } } - @NotNull - private DateFormat getDateFormat(FormatColumn[] resolvedFormats) { - DateFormat dateFormat = null; - - for (FormatColumn col : resolvedFormats) { - if (!(col instanceof DateColumn)) { - continue; - } - - if (dateFormat != null && !dateFormat.equals(((DateColumn) col).getFormat())) { - throw new IllegalStateException("Use of multiple Date Formats."); - } - - dateFormat = ((DateColumn) col).getFormat(); - } - - return dateFormat == null ? DateFormat.ALL : dateFormat; - } - - private int getIdIndex(FormatColumn[] resolvedFormats) { - for (int index = 0; index < resolvedFormats.length; index++) { - if (resolvedFormats[index] instanceof IdColumn) { - return index; - } - } - - throw new IllegalStateException("No IdColumn provided"); - } - - @NotNull - private FormatColumn[] resolveFormats(Map stringFormatColumnMap) { - final List resolvedFormats = new ArrayList<>(); - - for (String columnType : format) { - FormatColumn formatColumn = stringFormatColumnMap.get(columnType); - - if (formatColumn == null) { - throw new IllegalStateException(String.format("Don't know of format %s", columnType)); - } - - resolvedFormats.add(formatColumn); - } - return resolvedFormats.toArray(FormatColumn[]::new); - } @Override public void collectResultInfos(ResultInfoCollector collector) { diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java index 5a0253b276..96461ebfaf 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java @@ -1,37 +1,31 @@ package com.bakdata.conquery.apiv1.query.concept.specific.external; import java.time.LocalDate; -import java.util.Arrays; import java.util.Collections; -import java.util.Enumeration; -import com.bakdata.conquery.apiv1.forms.Form; import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.preproc.parser.specific.DateRangeParser; -import com.bakdata.conquery.util.DateFormats; -import com.google.common.base.Preconditions; -import com.google.common.collect.MoreCollectors; -import org.apache.commons.collections.EnumerationUtils; +import com.bakdata.conquery.util.DateReader; public enum DateFormat { EVENT_DATE { @Override - public CDateSet readDates(FormatColumn[] formats, String[] row, DateFormats dateFormats) { - final int index = findIndex(formats, DateColumn.EventDate.class); + public CDateSet readDates(int[] formats, String[] row, DateReader dateReader) { + final int index = formats[0]; - return CDateSet.create(Collections.singleton(CDateRange.exactly(dateFormats.parseToLocalDate(row[index])))); + return CDateSet.create(Collections.singleton(CDateRange.exactly(dateReader.parseToLocalDate(row[index])))); } }, START_END_DATE { @Override - public CDateSet readDates(FormatColumn[] formats, String[] row, DateFormats dateFormats) { + public CDateSet readDates(int[] formats, String[] row, DateReader dateReader) { - final int startIndex = findIndex(formats, DateColumn.StartDate.class); - final int endIndex = findIndex(formats, DateColumn.EndDate.class); + final int startIndex = formats[0]; + final int endIndex = formats[1]; - LocalDate start = dateFormats.parseToLocalDate(row[startIndex]); - LocalDate end = dateFormats.parseToLocalDate(row[endIndex]); + LocalDate start = dateReader.parseToLocalDate(row[startIndex]); + LocalDate end = dateReader.parseToLocalDate(row[endIndex]); if (start == null && end == null) { return null; @@ -42,36 +36,27 @@ public CDateSet readDates(FormatColumn[] formats, String[] row, DateFormats date }, DATE_RANGE { @Override - public CDateSet readDates(FormatColumn[] formats, String[] row, DateFormats dateFormats) { - final int index = findIndex(formats, DateColumn.DateRange.class); + public CDateSet readDates(int[] formats, String[] row, DateReader dateReader) { + final int index = formats[0]; - return CDateSet.create(Collections.singleton(DateRangeParser.parseISORange(row[index], dateFormats))); + return CDateSet.create(Collections.singleton(DateRangeParser.parseISORange(row[index], dateReader))); } }, DATE_SET { @Override - public CDateSet readDates(FormatColumn[] formats, String[] row, DateFormats dateFormats) { - final int index = findIndex(formats, DateColumn.DateSet.class); + public CDateSet readDates(int[] formats, String[] row, DateReader dateReader) { + final int index = formats[0]; - return CDateSet.parse(row[index], dateFormats); + return CDateSet.parse(row[index], dateReader); } }, ALL { @Override - public CDateSet readDates(FormatColumn[] formats, String[] row, DateFormats dateFormats) { + public CDateSet readDates(int[] formats, String[] row, DateReader dateReader) { return CDateSet.createFull(); } }; - public abstract CDateSet readDates(FormatColumn[] formats, String[] row, DateFormats dateFormats); + public abstract CDateSet readDates(int[] formats, String[] row, DateReader dateReader); - protected static int findIndex(FormatColumn[] formats, Class clazz){ - for (int index = 0; index < formats.length; index++) { - if (clazz.isInstance(formats[index])) { - return index; - } - } - - return -1; - } } diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/CDateRangeDeserializer.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/CDateRangeDeserializer.java index ee2f502587..4d445ad9c4 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/CDateRangeDeserializer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/CDateRangeDeserializer.java @@ -2,7 +2,7 @@ import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.preproc.parser.specific.DateRangeParser; -import com.bakdata.conquery.util.DateFormats; +import com.bakdata.conquery.util.DateReader; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.DeserializationContext; @@ -13,16 +13,16 @@ public class CDateRangeDeserializer extends StdDeserializer { - private final DateFormats dateFormats; + private final DateReader dateReader; - protected CDateRangeDeserializer(DateFormats dateFormats) { + protected CDateRangeDeserializer(DateReader dateReader) { super(CDateRange.class); - this.dateFormats = dateFormats; + this.dateReader = dateReader; } @SneakyThrows @Override public CDateRange deserialize(JsonParser p, DeserializationContext ctxt) throws IOException, JsonProcessingException { - return DateRangeParser.parseISORange(p.getText(), dateFormats); + return DateRangeParser.parseISORange(p.getText(), dateReader); } } diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/CDateSetDeserializer.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/CDateSetDeserializer.java index 0eecf28ebf..356cd167e7 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/CDateSetDeserializer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/CDateSetDeserializer.java @@ -4,7 +4,7 @@ import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.common.daterange.CDateRange; -import com.bakdata.conquery.util.DateFormats; +import com.bakdata.conquery.util.DateReader; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.JsonToken; @@ -15,10 +15,10 @@ public class CDateSetDeserializer extends StdDeserializer { private static final long serialVersionUID = 1L; - private final DateFormats formats; + private final DateReader formats; - public CDateSetDeserializer(DateFormats formats) { + public CDateSetDeserializer(DateReader formats) { super(CDateSet.class); this.formats = formats; } diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/FormatedDateDeserializer.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/FormatedDateDeserializer.java index 21022449c9..ead4a01e79 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/FormatedDateDeserializer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/FormatedDateDeserializer.java @@ -1,6 +1,6 @@ package com.bakdata.conquery.io.jackson.serializer; -import com.bakdata.conquery.util.DateFormats; +import com.bakdata.conquery.util.DateReader; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.deser.std.StdDeserializer; @@ -11,9 +11,9 @@ public class FormatedDateDeserializer extends StdDeserializer { - private final DateFormats formats; + private final DateReader formats; - public FormatedDateDeserializer(DateFormats formats) { + public FormatedDateDeserializer(DateReader formats) { super(LocalDate.class); this.formats = formats; } diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java index 0c311b7492..c85f6188dc 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java @@ -36,6 +36,13 @@ public NamespaceStorage(Validator validator, StoreFactory storageFactory, String idMapping = storageFactory.createIdMappingStore(pathName); structure = storageFactory.createStructureStore(pathName, new SingletonNamespaceCollection(getCentralRegistry())); workerToBuckets = storageFactory.createWorkerToBucketsStore(pathName); + + + decorateIdMapping(idMapping); + } + + private void decorateIdMapping(SingletonStore idMapping) { + idMapping.onAdd(mapping -> mapping.setDictionary(getPrimaryDictionary())); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/common/CDateSet.java b/backend/src/main/java/com/bakdata/conquery/models/common/CDateSet.java index 7b04f46740..d54efeba74 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/common/CDateSet.java +++ b/backend/src/main/java/com/bakdata/conquery/models/common/CDateSet.java @@ -16,7 +16,7 @@ import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.preproc.parser.specific.DateRangeParser; -import com.bakdata.conquery.util.DateFormats; +import com.bakdata.conquery.util.DateReader; import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.base.Joiner; import com.google.common.collect.ForwardingCollection; @@ -24,7 +24,7 @@ import lombok.EqualsAndHashCode; /** - * (De-)Serializers are are registered programmatically because they depend on {@link com.bakdata.conquery.util.DateFormats} + * (De-)Serializers are are registered programmatically because they depend on {@link DateReader} */ @EqualsAndHashCode public class CDateSet { @@ -445,13 +445,13 @@ public int getMaxValue() { return rangesByLowerBound.lastEntry().getValue().getMaxValue(); } - public static CDateSet parse(String value, DateFormats dateFormats) { + public static CDateSet parse(String value, DateReader dateReader) { List ranges = PARSE_PATTERN .matcher(value) .results() .map(mr -> { try { - return DateRangeParser.parseISORange(mr.group(2), dateFormats); + return DateRangeParser.parseISORange(mr.group(2), dateReader); } catch(Exception e) { throw new RuntimeException(e); diff --git a/backend/src/main/java/com/bakdata/conquery/models/common/daterange/CDateRange.java b/backend/src/main/java/com/bakdata/conquery/models/common/daterange/CDateRange.java index 51a77739ee..bf42591f8f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/common/daterange/CDateRange.java +++ b/backend/src/main/java/com/bakdata/conquery/models/common/daterange/CDateRange.java @@ -15,9 +15,6 @@ import com.bakdata.conquery.models.common.IRange; import com.bakdata.conquery.models.common.QuarterUtils; import com.bakdata.conquery.models.common.Range; -import com.bakdata.conquery.models.exceptions.ParsingException; -import com.bakdata.conquery.models.preproc.parser.specific.DateRangeParser; -import com.bakdata.conquery.util.DateFormats; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonValue; diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java index a05b2f6fdf..6e1275af23 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java @@ -20,7 +20,7 @@ import com.bakdata.conquery.models.config.auth.DevelopmentAuthorizationConfig; import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.identifiable.mapping.IdMappingConfig; -import com.bakdata.conquery.util.DateFormats; +import com.bakdata.conquery.util.DateReader; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.module.SimpleModule; import com.google.common.collect.MoreCollectors; @@ -137,10 +137,10 @@ public ObjectMapper configureObjectMapper(ObjectMapper objectMapper) { public static class ConfiguredModule extends SimpleModule { public ConfiguredModule(ConqueryConfig config){ - DateFormats dateFormats = config.getPreprocessor().getParsers().getDateFormats(); - addDeserializer(LocalDate.class, new FormatedDateDeserializer(dateFormats)); + DateReader dateReader = config.getPreprocessor().getParsers().getDateReader(); + addDeserializer(LocalDate.class, new FormatedDateDeserializer(dateReader)); - addDeserializer(CDateSet.class, new CDateSetDeserializer(dateFormats)); + addDeserializer(CDateSet.class, new CDateSetDeserializer(dateReader)); addSerializer(CDateSet.class, new CDateSetSerializer()); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/NoIdMapping.java b/backend/src/main/java/com/bakdata/conquery/models/config/NoIdMapping.java index 394ba14338..f7a84dbb06 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/NoIdMapping.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/NoIdMapping.java @@ -11,14 +11,9 @@ /** * Disables Id-Mapping completely. */ -@CPSType(base = IdMappingConfig.class, id = "NO_ID_MAPPING") public class NoIdMapping extends IdMappingConfig { private static final String[] HEADER = new String[]{"result"}; - @Override - protected void processRecord(Record record, String id, EntityIdMap mapping) { - // Do nothing. - } @Override public List getPrintIdFields() { diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ParserConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/ParserConfig.java index 216b706e4a..2080c75c0d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ParserConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ParserConfig.java @@ -1,31 +1,11 @@ package com.bakdata.conquery.models.config; -import com.bakdata.conquery.models.common.daterange.CDateRange; -import com.bakdata.conquery.models.exceptions.ParsingException; -import com.bakdata.conquery.util.DateFormats; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonDeserializer; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import com.google.common.base.Preconditions; -import com.google.common.base.Strings; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; +import com.bakdata.conquery.util.DateReader; import lombok.Data; import lombok.Setter; -import lombok.experimental.UtilityClass; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import javax.validation.constraints.NotNull; -import java.io.IOException; -import java.time.LocalDate; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeFormatterBuilder; -import java.time.format.DateTimeParseException; + import java.util.*; @Data @@ -51,7 +31,7 @@ public class ParserConfig { * Date formats that are available for parsing. */ @NotNull - private DateFormats dateFormats = new DateFormats(List.of( + private DateReader dateReader = new DateReader(List.of( "yyyy-MM-dd", "yyyyMMdd", "dd.MM.yyyy" )); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/SimpleIdMapping.java b/backend/src/main/java/com/bakdata/conquery/models/config/SimpleIdMapping.java index f85eeb104f..597b3d941a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/SimpleIdMapping.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/SimpleIdMapping.java @@ -14,26 +14,7 @@ /** * Maps input to itself for upload. */ -@CPSType(base = IdMappingConfig.class, id = "SIMPLE") public class SimpleIdMapping extends NoIdMapping { - @Override - public Map getFormatColumns() { - final HashMap out = new HashMap<>(super.getFormatColumns()); - out.put(IdColumn.HANDLE, new IdColumn()); - - return out; - } - - @Override - protected void processRecord(Record record, String id, EntityIdMap mapping) { - - final EntityPrintId entityPrintId = EntityPrintId.from(id); - - mapping.addOutputMapping(id, entityPrintId); - - mapping.addInputMapping(id, id); - - } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java index 71f5a8b5be..f490741758 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java @@ -5,12 +5,14 @@ import java.util.Map; import java.util.Optional; +import com.bakdata.conquery.models.dictionary.EncodedDictionary; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonValue; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.NoArgsConstructor; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; /** @@ -23,7 +25,8 @@ @NoArgsConstructor public class EntityIdMap { - + @Setter + private EncodedDictionary dictionary; /** * The map from csv entity ids to external entity ids. @@ -75,6 +78,16 @@ public Optional toInternal(String... external) { return Optional.ofNullable(external2Internal.get(new ExternalId(external))); } + public int resolve(String... external) { + String value = external2Internal.get(new ExternalId(external)); + + if (value != null) { + return dictionary.getId(value); + } + + return -1; + } + public void addOutputMapping(String csvEntityId, EntityPrintId externalEntityId) { final EntityPrintId prior = internalToPrint.put(csvEntityId, externalEntityId); diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingConfig.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingConfig.java index 81495d0e86..d2462d4460 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingConfig.java @@ -1,40 +1,52 @@ package com.bakdata.conquery.models.identifiable.mapping; -import java.util.*; +import java.util.Collections; +import java.util.List; +import java.util.Map; import com.bakdata.conquery.apiv1.query.concept.specific.external.DateColumn; -import com.bakdata.conquery.apiv1.query.concept.specific.external.FormatColumn; -import com.bakdata.conquery.apiv1.query.concept.specific.external.IgnoreColumn; +import com.bakdata.conquery.apiv1.query.concept.specific.external.DateFormat; import com.bakdata.conquery.io.cps.CPSBase; import com.bakdata.conquery.models.auth.entities.User; +import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.worker.Namespace; +import com.bakdata.conquery.util.DateReader; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.google.common.base.Strings; import com.univocity.parsers.common.record.Record; import com.univocity.parsers.csv.CsvParser; -import lombok.RequiredArgsConstructor; +import it.unimi.dsi.fastutil.ints.Int2ObjectAVLTreeMap; +import it.unimi.dsi.fastutil.ints.Int2ObjectMap; +import it.unimi.dsi.fastutil.ints.IntArrayList; +import it.unimi.dsi.fastutil.ints.IntList; +import lombok.Data; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; @Slf4j -@CPSBase -@RequiredArgsConstructor -@JsonTypeInfo(use = JsonTypeInfo.Id.CUSTOM, property = "type") -public abstract class IdMappingConfig { +@NoArgsConstructor +@Setter +@Getter +public class IdMappingConfig { - @JsonIgnore - public Map getFormatColumns() { - return Map.of( - DateColumn.DateSet.HANDLE, new DateColumn.DateSet(), - DateColumn.DateRange.HANDLE, new DateColumn.DateRange(), - DateColumn.EventDate.HANDLE, new DateColumn.EventDate(), - DateColumn.StartDate.HANDLE, new DateColumn.StartDate(), - DateColumn.EndDate.HANDLE, new DateColumn.EndDate(), - IgnoreColumn.HANDLE, new IgnoreColumn() - ); - } + public final Map dateFormats = Map.of( + DateColumn.DateSet.HANDLE, DateFormat.DATE_SET, + DateColumn.DateRange.HANDLE, DateFormat.DATE_RANGE, + DateColumn.EventDate.HANDLE, DateFormat.EVENT_DATE, + DateColumn.StartDate.HANDLE, DateFormat.START_END_DATE, + DateColumn.EndDate.HANDLE, DateFormat.START_END_DATE + ); + + private List mappers = List.of(new InputMapper("ID", "id", "0", -1, Collections.emptyMap(), Collections.emptyMap())); + + private OutputMapper outputMapper = new OutputMapper(); /** * Read incoming CSV-file extracting Id-Mappings for in and Output. @@ -45,25 +57,29 @@ public EntityIdMap generateIdMapping(CsvParser parser) { Record record; - while((record = parser.parseNextRecord()) != null){ + while ((record = parser.parseNextRecord()) != null) { final String id = record.getString("id"); - processRecord(record, id, mapping); + for (InputMapper mapper : mappers) { + final String otherId = record.getString(mapper.getField()); + final String transformed = mapper.read(otherId); + + mapping.addInputMapping(id, mapper.getName(), transformed); + } + + mapping.addOutputMapping(id, outputMapper.extractOutputId(record)); } return mapping; } - /** - * Process a single record and generate all found ids. - */ - protected abstract void processRecord(Record record, String id, EntityIdMap mapping); - /** * Headers for Output CSV. */ @JsonIgnore - public abstract List getPrintIdFields(); + public List getPrintIdFields() { + return List.of(outputMapper.getHeaders()); + } /** * Is called once before a mapping is used before a query result is created to @@ -93,4 +109,104 @@ public EntityPrintId toExternal(String csvEntityId, Namespace namespace, IdMappi return externalEntityId; } + public InputMapper getIdMapper(String name) { + return mappers.stream().filter(mapper -> mapper.getName().equals(name)).findFirst().orElse(null); + } + + public int getIdIndex(List format) { + for (int index = 0; index < format.size(); index++) { + final String current = format.get(index); + + if (mappers.stream().map(InputMapper::getName).anyMatch(current::equals)) { + return index; + } + } + + return -1; + } + + + public Int2ObjectMap readDates(String[][] values, List format, DateReader dateReader) { + Int2ObjectMap out = new Int2ObjectAVLTreeMap<>(); + + DateFormat dateFormat = null; + + + IntList dateColumns = new IntArrayList(format.size()); + + for (int col = 0; col < format.size(); col++) { + String desc = format.get(col); + + dateFormat = resolveDateFormat(desc); + + if (dateFormat == null) { + continue; + } + + dateColumns.add(col); + } + + dateFormat = dateFormat == null ? DateFormat.ALL : dateFormat; + final int[] datePositions = dateColumns.toIntArray(); + + for (int row = 1; row < values.length; row++) { + try { + final CDateSet dates = dateFormat.readDates(datePositions, values[row], dateReader); + + if (dates == null) { + continue; + } + + out.put(row, dates); + } + catch (Exception e) { + log.warn("Failed to parse Date from {}", row, e); + } + } + + return out; + } + + public DateFormat resolveDateFormat(String desc) { + return dateFormats.get(desc); + } + + @Data + public static class InputMapper { + private final String name; + + private final String field; + + private final String pad; + private final int length; + + private final Map label; + private final Map description; + + public String read(String value) { + if (Strings.isNullOrEmpty(value)) { + return null; + } + + if (length == -1) { + return value; + } + + return StringUtils.leftPad(value, length, pad); + } + } + + @CPSBase + @JsonTypeInfo(use = JsonTypeInfo.Id.CUSTOM, property = "type") + @NoArgsConstructor + public static class OutputMapper { + @JsonIgnore + public String[] getHeaders() { + return new String[]{"result"}; + } + + public EntityPrintId extractOutputId(Record record) { + return new EntityPrintId(new String[]{record.getString("id")}); + } + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/Preprocessor.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/Preprocessor.java index ec09ddc9cd..900737ded3 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/Preprocessor.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/Preprocessor.java @@ -19,7 +19,7 @@ import com.bakdata.conquery.models.exceptions.ParsingException; import com.bakdata.conquery.models.preproc.outputs.OutputDescription; import com.bakdata.conquery.models.preproc.parser.Parser; -import com.bakdata.conquery.util.DateFormats; +import com.bakdata.conquery.util.DateReader; import com.bakdata.conquery.util.io.ConqueryMDC; import com.bakdata.conquery.util.io.FileUtil; import com.bakdata.conquery.util.io.LogUtil; @@ -124,13 +124,13 @@ public static void preprocess(PreprocessingJob preprocessingJob, ProgressBar tot final GroovyPredicate filter = input.createFilter(headers); - DateFormats dateFormats = config.getPreprocessor().getParsers().getDateFormats(); - final OutputDescription.Output primaryOut = input.getPrimary().createForHeaders(headerMap, dateFormats); + DateReader dateReader = config.getPreprocessor().getParsers().getDateReader(); + final OutputDescription.Output primaryOut = input.getPrimary().createForHeaders(headerMap, dateReader); final List outputs = new ArrayList<>(); // Instantiate Outputs based on descriptors (apply header positions) for (OutputDescription op : input.getOutput()) { - outputs.add(op.createForHeaders(headerMap, dateFormats)); + outputs.add(op.createForHeaders(headerMap, dateReader)); } String[] row; diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/CopyOutput.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/CopyOutput.java index 949dae4afb..2d4060c7cc 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/CopyOutput.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/CopyOutput.java @@ -6,7 +6,7 @@ import com.bakdata.conquery.models.events.MajorTypeId; import com.bakdata.conquery.models.exceptions.ParsingException; import com.bakdata.conquery.models.preproc.parser.Parser; -import com.bakdata.conquery.util.DateFormats; +import com.bakdata.conquery.util.DateReader; import it.unimi.dsi.fastutil.objects.Object2IntArrayMap; import lombok.Data; import lombok.NoArgsConstructor; @@ -47,7 +47,7 @@ public int hashCode(){ private MajorTypeId inputType; @Override - public Output createForHeaders(Object2IntArrayMap headers, DateFormats dateFormats) { + public Output createForHeaders(Object2IntArrayMap headers, DateReader dateReader) { assertRequiredHeaders(headers, inputColumn); final int column = headers.getInt(inputColumn); diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/DateRangeOutput.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/DateRangeOutput.java index a97f29e513..a48dd337ca 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/DateRangeOutput.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/DateRangeOutput.java @@ -9,7 +9,7 @@ import com.bakdata.conquery.models.events.MajorTypeId; import com.bakdata.conquery.models.exceptions.ParsingException; import com.bakdata.conquery.models.preproc.parser.Parser; -import com.bakdata.conquery.util.DateFormats; +import com.bakdata.conquery.util.DateReader; import com.google.common.base.Strings; import it.unimi.dsi.fastutil.objects.Object2IntArrayMap; import lombok.Data; @@ -43,7 +43,7 @@ public int hashCode(){ } @Override - public Output createForHeaders(Object2IntArrayMap headers, DateFormats dateFormats) { + public Output createForHeaders(Object2IntArrayMap headers, DateReader dateReader) { assertRequiredHeaders(headers, startColumn, endColumn); final int startIndex = headers.getInt(startColumn); @@ -61,8 +61,8 @@ protected Object parseLine(String[] row, Parser type, long sourceLine) throws Pa throw new IllegalArgumentException("Open Ranges are not allowed."); } - LocalDate start = dateFormats.parseToLocalDate(row[startIndex]); - LocalDate end = dateFormats.parseToLocalDate(row[endIndex]); + LocalDate start = dateReader.parseToLocalDate(row[startIndex]); + LocalDate end = dateReader.parseToLocalDate(row[endIndex]); return CDateRange.of(start, end); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/EpochDateRangeOutput.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/EpochDateRangeOutput.java index a8fef5d2e1..46b9a8bef8 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/EpochDateRangeOutput.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/EpochDateRangeOutput.java @@ -7,7 +7,7 @@ import com.bakdata.conquery.models.events.MajorTypeId; import com.bakdata.conquery.models.exceptions.ParsingException; import com.bakdata.conquery.models.preproc.parser.Parser; -import com.bakdata.conquery.util.DateFormats; +import com.bakdata.conquery.util.DateReader; import com.google.common.base.Strings; import it.unimi.dsi.fastutil.objects.Object2IntArrayMap; import lombok.Data; @@ -41,7 +41,7 @@ public int hashCode(){ } @Override - public Output createForHeaders(Object2IntArrayMap headers, DateFormats dateFormats) { + public Output createForHeaders(Object2IntArrayMap headers, DateReader dateReader) { assertRequiredHeaders(headers, startColumn, endColumn); final int startIndex = headers.getInt(startColumn); diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/EpochOutput.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/EpochOutput.java index d28ff81618..6b77946b2e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/EpochOutput.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/EpochOutput.java @@ -6,7 +6,7 @@ import com.bakdata.conquery.models.events.MajorTypeId; import com.bakdata.conquery.models.exceptions.ParsingException; import com.bakdata.conquery.models.preproc.parser.Parser; -import com.bakdata.conquery.util.DateFormats; +import com.bakdata.conquery.util.DateReader; import it.unimi.dsi.fastutil.objects.Object2IntArrayMap; import lombok.Data; import lombok.ToString; @@ -32,7 +32,7 @@ public int hashCode(){ } @Override - public Output createForHeaders(Object2IntArrayMap headers, DateFormats dateFormats) { + public Output createForHeaders(Object2IntArrayMap headers, DateReader dateReader) { assertRequiredHeaders(headers, inputColumn); final int columnIndex = headers.getInt(inputColumn); diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/LineOutput.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/LineOutput.java index b8f996a9d2..9cbb0ecf57 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/LineOutput.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/LineOutput.java @@ -4,7 +4,7 @@ import com.bakdata.conquery.models.events.MajorTypeId; import com.bakdata.conquery.models.exceptions.ParsingException; import com.bakdata.conquery.models.preproc.parser.Parser; -import com.bakdata.conquery.util.DateFormats; +import com.bakdata.conquery.util.DateReader; import com.fasterxml.jackson.annotation.JsonCreator; import it.unimi.dsi.fastutil.objects.Object2IntArrayMap; import lombok.NoArgsConstructor; @@ -30,7 +30,7 @@ public boolean isRequired() { } @Override - public Output createForHeaders(Object2IntArrayMap headers, DateFormats dateFormats) { + public Output createForHeaders(Object2IntArrayMap headers, DateReader dateReader) { return new Output() { @Override protected Object parseLine(String[] row, Parser type, long sourceLine) throws ParsingException { diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/NullOutput.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/NullOutput.java index bb3bd6ede6..cf7fa73795 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/NullOutput.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/NullOutput.java @@ -6,7 +6,7 @@ import com.bakdata.conquery.models.events.MajorTypeId; import com.bakdata.conquery.models.exceptions.ParsingException; import com.bakdata.conquery.models.preproc.parser.Parser; -import com.bakdata.conquery.util.DateFormats; +import com.bakdata.conquery.util.DateReader; import it.unimi.dsi.fastutil.objects.Object2IntArrayMap; import lombok.Data; import org.apache.commons.lang3.builder.HashCodeBuilder; @@ -37,7 +37,7 @@ public boolean isRequired() { } @Override - public Output createForHeaders(Object2IntArrayMap headers, DateFormats dateFormats) { + public Output createForHeaders(Object2IntArrayMap headers, DateReader dateReader) { return new Output() { @Override protected Object parseLine(String[] row, Parser type, long sourceLine) throws ParsingException { diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/OutputDescription.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/OutputDescription.java index c10b77a8e7..21aabbd52d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/OutputDescription.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/outputs/OutputDescription.java @@ -11,7 +11,7 @@ import com.bakdata.conquery.models.exceptions.ParsingException; import com.bakdata.conquery.models.preproc.ColumnDescription; import com.bakdata.conquery.models.preproc.parser.Parser; -import com.bakdata.conquery.util.DateFormats; +import com.bakdata.conquery.util.DateReader; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonTypeInfo; import it.unimi.dsi.fastutil.objects.Object2IntArrayMap; @@ -110,10 +110,10 @@ protected void assertRequiredHeaders(Object2IntArrayMap actualHeaders, S /** * Instantiate the corresponding {@link Output} for the rows. * @param headers A map from column names to column indices. - * @param dateFormats + * @param dateReader * @return the output for the specific headers. */ - public abstract Output createForHeaders(Object2IntArrayMap headers, DateFormats dateFormats); + public abstract Output createForHeaders(Object2IntArrayMap headers, DateReader dateReader); /** * The resulting type after {@link Output} has been applied. diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/DateParser.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/DateParser.java index fe12fee1a9..6c9da6a11b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/DateParser.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/DateParser.java @@ -10,7 +10,7 @@ import com.bakdata.conquery.models.exceptions.ParsingException; import com.bakdata.conquery.models.preproc.parser.ColumnValues; import com.bakdata.conquery.models.preproc.parser.Parser; -import com.bakdata.conquery.util.DateFormats; +import com.bakdata.conquery.util.DateReader; import lombok.SneakyThrows; import lombok.ToString; @@ -18,12 +18,12 @@ public class DateParser extends Parser { private IntegerParser subType; - private DateFormats dateFormats; + private DateReader dateReader; public DateParser(ParserConfig config) { super(config); subType = new IntegerParser(config); - dateFormats = config.getDateFormats(); + dateReader = config.getDateReader(); } @@ -35,7 +35,7 @@ public void setLines(int lines) { @Override protected Integer parseValue(@Nonnull String value) throws ParsingException { - return CDate.ofLocalDate(dateFormats.parseToLocalDate(value)); + return CDate.ofLocalDate(dateReader.parseToLocalDate(value)); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/DateRangeParser.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/DateRangeParser.java index b123947d37..9fb8b9e473 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/DateRangeParser.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/DateRangeParser.java @@ -10,20 +10,18 @@ import com.bakdata.conquery.models.exceptions.ParsingException; import com.bakdata.conquery.models.preproc.parser.ColumnValues; import com.bakdata.conquery.models.preproc.parser.Parser; -import com.bakdata.conquery.util.DateFormats; +import com.bakdata.conquery.util.DateReader; import lombok.ToString; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -import java.util.Date; - @Slf4j @ToString(callSuper = true) public class DateRangeParser extends Parser { private final DateParser minParser; private final DateParser maxParser; - private final DateFormats dateFormats; + private final DateReader dateReader; private boolean onlyQuarters = true; private int maxValue = Integer.MIN_VALUE; @@ -34,15 +32,15 @@ public DateRangeParser(ParserConfig config) { super(config); minParser = new DateParser(config); maxParser = new DateParser(config); - dateFormats = config.getDateFormats(); + dateReader = config.getDateReader(); } @Override protected CDateRange parseValue(@Nonnull String value) { - return DateRangeParser.parseISORange(value, dateFormats); + return DateRangeParser.parseISORange(value, dateReader); } - public static CDateRange parseISORange(String value, DateFormats dateFormats) { + public static CDateRange parseISORange(String value, DateReader dateReader) { if (value == null) { return null; } @@ -52,8 +50,8 @@ public static CDateRange parseISORange(String value, DateFormats dateFormats) { } return CDateRange.of( - dateFormats.parseToLocalDate(parts[0]), - dateFormats.parseToLocalDate(parts[1]) + dateReader.parseToLocalDate(parts[0]), + dateReader.parseToLocalDate(parts[1]) ); } diff --git a/backend/src/main/java/com/bakdata/conquery/util/DateFormats.java b/backend/src/main/java/com/bakdata/conquery/util/DateReader.java similarity index 85% rename from backend/src/main/java/com/bakdata/conquery/util/DateFormats.java rename to backend/src/main/java/com/bakdata/conquery/util/DateReader.java index 1eb424e9bd..bd1ec3be4b 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/DateFormats.java +++ b/backend/src/main/java/com/bakdata/conquery/util/DateReader.java @@ -10,11 +10,12 @@ import java.util.Locale; import java.util.Set; +import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.NotNull; + import com.bakdata.conquery.models.exceptions.ParsingException; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.databind.annotation.JsonAppend; -import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; @@ -23,15 +24,12 @@ import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; -import javax.validation.constraints.NotEmpty; -import javax.validation.constraints.NotNull; - /** * Utility class for parsing multiple dateformats. Parsing is cached in two ways: First parsed values are cached. Second, the last used parser is cached since it's likely that it will be used again, we therefore try to use it first, then try all others. */ @Slf4j @NoArgsConstructor -public class DateFormats { +public class DateReader { @NotNull @NotEmpty @@ -61,13 +59,12 @@ public class DateFormats { */ @JsonIgnore private final LoadingCache DATE_CACHE = CacheBuilder.newBuilder() - .softValues() - .concurrencyLevel(10) - .initialCapacity(64000) - .build(CacheLoader.from(this::tryParse)); + .weakValues() + .concurrencyLevel(10) + .build(CacheLoader.from(this::tryParse)); @JsonCreator - public DateFormats(@NotEmpty List formats) { + public DateReader(@NotEmpty List formats) { final Set formatters = new HashSet<>(); @@ -82,13 +79,13 @@ public DateFormats(@NotEmpty List formats) { * Try parsing the String value to a LocalDate. */ public LocalDate parseToLocalDate(String value) throws ParsingException { - if(Strings.isNullOrEmpty(value)) { + if (Strings.isNullOrEmpty(value)) { return null; } final LocalDate out = DATE_CACHE.getUnchecked(value); - if(out.equals(ERROR_DATE)) { + if (out.equals(ERROR_DATE)) { throw new IllegalArgumentException(String.format("Failed to parse `%s` as LocalDate.", value)); } @@ -97,7 +94,7 @@ public LocalDate parseToLocalDate(String value) throws ParsingException { /** * Try and parse with the last successful parser. If not successful try and parse with other parsers and update the last successful parser. - * + *

* Method is private as it is only directly accessed via the Cache. */ private LocalDate tryParse(String value) { @@ -107,7 +104,8 @@ private LocalDate tryParse(String value) { if (formatter != null) { try { return LocalDate.parse(value, formatter); - } catch (DateTimeParseException e) { + } + catch (DateTimeParseException e) { //intentionally left blank } } @@ -118,7 +116,8 @@ private LocalDate tryParse(String value) { LocalDate res = LocalDate.parse(value, format); lastFormat.set(format); return res; - } catch (DateTimeParseException e) { + } + catch (DateTimeParseException e) { //intentionally left blank } } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java b/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java index 854d05cda8..b350b108d4 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java @@ -47,7 +47,6 @@ public void overrideConfig(ConqueryConfig config) { @Override public void executeTest(StandaloneSupport standaloneSupport) throws IOException { DatasetRegistry namespaces = standaloneSupport.getNamespace().getNamespaces(); - Dataset dataset = standaloneSupport.getDataset(); Query query = getQuery(); From 45fad7028e4596ca5c454a3d4e44716f9e3c4868 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Fri, 16 Jul 2021 12:55:38 +0200 Subject: [PATCH 11/82] fix serialization of EntityIdMap.java using JsonValue/JsonCreator --- .../identifiable/mapping/EntityIdMap.java | 57 ++++++++++++------- 1 file changed, 37 insertions(+), 20 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java index f490741758..8233eb0002 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java @@ -1,17 +1,20 @@ package com.bakdata.conquery.models.identifiable.mapping; -import java.util.Collection; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Optional; import com.bakdata.conquery.models.dictionary.EncodedDictionary; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonValue; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.NoArgsConstructor; +import lombok.RequiredArgsConstructor; import lombok.Setter; import lombok.extern.slf4j.Slf4j; @@ -26,41 +29,61 @@ public class EntityIdMap { @Setter + @JsonIgnore private EncodedDictionary dictionary; /** * The map from csv entity ids to external entity ids. */ + @JsonIgnore private final Map internalToPrint = new HashMap<>(); /** * The map from external entity ids to csv entity ids. */ + @JsonIgnore private final Map external2Internal = new HashMap<>(); + /** + * Helper class for serialization. + */ @Data - protected static class Container { - private final Collection> external2Internal; + @Getter + @RequiredArgsConstructor(onConstructor_ = @JsonCreator) + private static class Container { + private final List keys; + private final List values; private final Map internalToPrint; } + /** + * Constructor to deserialize from {@link Container}. + */ @JsonCreator - protected EntityIdMap fromContainer(Container container) { - final EntityIdMap out = new EntityIdMap(); + private EntityIdMap(Container container) { - for (Map.Entry entry : container.getExternal2Internal()) { - out.getExternal2Internal().put(entry.getKey(), entry.getValue()); + for (int index = 0; index < container.keys.size(); index++) { + getExternal2Internal().put(container.keys.get(index), container.values.get(index)); } - out.getInternalToPrint().putAll(container.getInternalToPrint()); - - return out; + getInternalToPrint().putAll(container.internalToPrint); } + /** + * JsonValue to Serialize as {@link Container}, as Jackson cannot handle complex classes as Map-keys (ie {@link ExternalId}. + */ @JsonValue - protected Container getContainer() { - return new Container(external2Internal.entrySet(), internalToPrint); + private Container getContainer() { + final List keys = new ArrayList<>(external2Internal.size()); + final List values = new ArrayList<>(external2Internal.size()); + + external2Internal.forEach((key, value) -> { + keys.add(key); + values.add(value); + }); + + return new Container(keys, values, internalToPrint); } /** @@ -70,14 +93,6 @@ public EntityPrintId toExternal(String internal) { return internalToPrint.get(internal); } - /** - * Resolve an external to an internal id. - * @param external - */ - public Optional toInternal(String... external) { - return Optional.ofNullable(external2Internal.get(new ExternalId(external))); - } - public int resolve(String... external) { String value = external2Internal.get(new ExternalId(external)); @@ -107,7 +122,9 @@ public void addInputMapping(String csvEntityId, String... externalEntityId) { } @Data + @RequiredArgsConstructor(onConstructor_ = @JsonCreator) private static class ExternalId { + @Getter private final String[] parts; } From 9871230c2619d9836ebf3b80aaab7107c93e366c Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Tue, 20 Jul 2021 15:55:03 +0200 Subject: [PATCH 12/82] some more nudging the Mapping into FrontendConfig --- .../concept/specific/external/CQExternal.java | 55 ++++++++- .../models/config/FrontendConfig.java | 93 ++++++++------- .../identifiable/mapping/EntityIdMap.java | 16 ++- .../identifiable/mapping/IdMappingConfig.java | 106 ++++-------------- .../resources/api/ConfigResource.java | 5 +- .../identifiable/IdMapSerialisationTest.java | 2 +- 6 files changed, 131 insertions(+), 146 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java index 88e90c99b1..3f0e336bef 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java @@ -11,6 +11,8 @@ import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.jackson.InternalOnly; import com.bakdata.conquery.models.common.CDateSet; +import com.bakdata.conquery.models.config.ColumnConfig; +import com.bakdata.conquery.models.config.FrontendConfig; import com.bakdata.conquery.models.error.ConqueryError; import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; import com.bakdata.conquery.models.identifiable.mapping.IdMappingConfig; @@ -23,11 +25,13 @@ import com.bakdata.conquery.util.DateReader; import com.fasterxml.jackson.annotation.JsonIgnore; import io.dropwizard.validation.ValidationMethod; +import it.unimi.dsi.fastutil.ints.Int2ObjectAVLTreeMap; import it.unimi.dsi.fastutil.ints.Int2ObjectMap; import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; +import it.unimi.dsi.fastutil.ints.IntArrayList; +import it.unimi.dsi.fastutil.ints.IntList; import lombok.Getter; import lombok.extern.slf4j.Slf4j; -import org.jetbrains.annotations.NotNull; /** * Allows uploading lists of entities. @@ -65,6 +69,47 @@ public QPNode createQueryPlan(QueryPlanContext context, ConceptQueryPlan plan) { } + private Int2ObjectMap readDates(String[][] values, List format, DateReader dateReader, FrontendConfig.UploadConfig queryUpload) { + Int2ObjectMap out = new Int2ObjectAVLTreeMap<>(); + + DateFormat dateFormat = null; + + + IntList dateColumns = new IntArrayList(format.size()); + + for (int col = 0; col < format.size(); col++) { + String desc = format.get(col); + + dateFormat = queryUpload.resolveDateFormat(desc); + + if (dateFormat == null) { + continue; + } + + dateColumns.add(col); + } + + dateFormat = dateFormat == null ? DateFormat.ALL : dateFormat; + final int[] datePositions = dateColumns.toIntArray(); + + for (int row = 1; row < values.length; row++) { + try { + final CDateSet dates = dateFormat.readDates(datePositions, values[row], dateReader); + + if (dates == null) { + continue; + } + + out.put(row, dates); + } + catch (Exception e) { + log.warn("Failed to parse Date from {}", row, e); + } + } + + return out; + } + @Override public void resolve(QueryResolveContext context) { valuesResolved = new Int2ObjectOpenHashMap<>(); @@ -76,20 +121,20 @@ public void resolve(QueryResolveContext context) { final IdMappingConfig mappingConfig = context.getConfig().getIdMapping(); // extract dates from rows - final Int2ObjectMap rowDates = mappingConfig.readDates(values, format, dateReader); + final Int2ObjectMap rowDates = readDates(values, format, dateReader, context.getConfig().getFrontend().getQueryUpload()); final int idIndex = mappingConfig.getIdIndex(format); - final IdMappingConfig.InputMapper reader = mappingConfig.getIdMapper(format.get(idIndex)); + final ColumnConfig reader = mappingConfig.getIdMapper(format.get(idIndex)); final List unresolved = new ArrayList<>(); // ignore the first row, because this is the header for (int rowNum = 1; rowNum < values.length; rowNum++) { final String[] row = values[rowNum]; - final String externalId = reader.read(row[idIndex]); + final EntityIdMap.ExternalId externalId = reader.read(row[idIndex]); - final int resolvedId = mapping.resolve(reader.getName(), externalId); + final int resolvedId = mapping.resolve(externalId); if (resolvedId == -1) { unresolved.add(row); diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java index dc958d672f..cbef3e2457 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java @@ -1,32 +1,29 @@ package com.bakdata.conquery.models.config; -import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Set; import javax.validation.Valid; import javax.validation.constraints.NotEmpty; import javax.validation.constraints.NotNull; import com.bakdata.conquery.apiv1.query.concept.specific.external.DateColumn; -import com.bakdata.conquery.apiv1.query.concept.specific.external.FormatColumn; -import com.bakdata.conquery.apiv1.query.concept.specific.external.IdColumn; -import com.bakdata.conquery.io.cps.CPSType; -import com.bakdata.conquery.io.cps.CPSTypeIdResolver; -import com.bakdata.conquery.models.identifiable.mapping.IdMappingConfig; +import com.bakdata.conquery.apiv1.query.concept.specific.external.DateFormat; import com.bakdata.conquery.util.VersionInfo; -import com.fasterxml.jackson.annotation.JsonBackReference; -import com.fasterxml.jackson.annotation.JsonIgnore; import groovy.transform.ToString; -import io.dropwizard.validation.ValidationMethod; +import lombok.AllArgsConstructor; import lombok.Data; import lombok.Getter; +import lombok.NoArgsConstructor; import lombok.Setter; +import lombok.With; @ToString @Getter @Setter +@With +@AllArgsConstructor +@NoArgsConstructor public class FrontendConfig { private String version = VersionInfo.INSTANCE.getProjectVersion(); @@ -36,57 +33,59 @@ public class FrontendConfig { private UploadConfig queryUpload = new UploadConfig(); - @Getter @Setter + @Getter + @Setter + @With + @AllArgsConstructor + @NoArgsConstructor public static class UploadConfig { @NotEmpty - private List ids = List.of(new ColumnConfig(IdColumn.HANDLE, Map.of("en", "Id"), Map.of("en", "Id of the Entity."))); + private List ids; @NotNull - private ColumnConfig dateStart = - new ColumnConfig(DateColumn.StartDate.HANDLE, Map.of("en", "Begin"), Map.of("en", "Begin of Date-range")); + private ColumnConfig dateStart = ColumnConfig.builder() + .name(DateColumn.StartDate.HANDLE) + .label(Map.of("en", "Begin")) + .description(Map.of("en", "Begin of Date-range")) + .build(); @NotNull - private ColumnConfig dateEnd = - new ColumnConfig(DateColumn.EndDate.HANDLE, Map.of("en", "Begin"), Map.of("en", "End of Date-range")); + private ColumnConfig dateEnd = ColumnConfig.builder() + .name(DateColumn.EndDate.HANDLE) + .label(Map.of("en", "End")) + .description(Map.of("en", "End of Date-range")) + .build(); + @NotNull - private ColumnConfig dateRange = - new ColumnConfig(DateColumn.DateRange.HANDLE, Map.of("en", "Date Range"), Map.of("en", "Full Date Range")); + private ColumnConfig dateRange = ColumnConfig.builder() + .name(DateColumn.DateRange.HANDLE) + .label(Map.of("en", "Date Range")) + .description(Map.of("en", "Full Date Range")) + .build(); + @NotNull - private ColumnConfig dateSet = - new ColumnConfig(DateColumn.DateSet.HANDLE, Map.of("en", "Dateset"), Map.of("en", "Set of Date-Ranges")); + private ColumnConfig dateSet = ColumnConfig.builder() + .name(DateColumn.DateSet.HANDLE) + .label(Map.of("en", "Dateset")) + .description(Map.of("en", "Set of Date-Ranges")) + .build(); + @NotNull - private ColumnConfig eventDate = - new ColumnConfig(DateColumn.EventDate.HANDLE, Map.of("en", "Event Date"), Map.of("en", "Single event")); - - @Data - public static class ColumnConfig { - - /** - * Name of the Column, used to resolve the specific entry at {@link IdMappingConfig#getFormatColumns()} - */ - @NotEmpty - private final String name; - - /** - * Map of Localized labels. - */ - private final Map label; - - /** - * Map of Localized description. - */ - private final Map description; - - @JsonIgnore - @ValidationMethod - public boolean isExistingColumnFormat() { - return CPSTypeIdResolver.getImplementation(FormatColumn.class, name) != null; - } + private ColumnConfig eventDate = ColumnConfig.builder() + .name(DateColumn.EventDate.HANDLE) + .label(Map.of("en", "Event Date")) + .description(Map.of("en", "Single event")) + .build(); + + public DateFormat resolveDateFormat(String handle) { + return DateFormat.ALL; //TODO } + + } @Data diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java index 8233eb0002..2737e79a1b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java @@ -4,7 +4,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import com.bakdata.conquery.models.dictionary.EncodedDictionary; import com.fasterxml.jackson.annotation.JsonCreator; @@ -93,8 +92,13 @@ public EntityPrintId toExternal(String internal) { return internalToPrint.get(internal); } - public int resolve(String... external) { - String value = external2Internal.get(new ExternalId(external)); + /** + * Resolve external ID to Entity Id. + * + * Return -1 when not resolved. + */ + public int resolve(ExternalId key) { + String value = external2Internal.get(key); if (value != null) { return dictionary.getId(value); @@ -112,9 +116,9 @@ public void addOutputMapping(String csvEntityId, EntityPrintId externalEntityId) } - public void addInputMapping(String csvEntityId, String... externalEntityId) { + public void addInputMapping(String csvEntityId, ExternalId externalEntityId) { - final String prior = external2Internal.put(new ExternalId(externalEntityId), csvEntityId); + final String prior = external2Internal.put(externalEntityId, csvEntityId); if (prior != null && prior.equals(csvEntityId)) { log.warn("Duplicate mapping for {} to {} and {}", externalEntityId, csvEntityId, prior); @@ -123,7 +127,7 @@ public void addInputMapping(String csvEntityId, String... externalEntityId) { @Data @RequiredArgsConstructor(onConstructor_ = @JsonCreator) - private static class ExternalId { + public static class ExternalId { @Getter private final String[] parts; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingConfig.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingConfig.java index d2462d4460..d7d84b1408 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingConfig.java @@ -1,6 +1,5 @@ package com.bakdata.conquery.models.identifiable.mapping; -import java.util.Collections; import java.util.List; import java.util.Map; @@ -8,25 +7,17 @@ import com.bakdata.conquery.apiv1.query.concept.specific.external.DateFormat; import com.bakdata.conquery.io.cps.CPSBase; import com.bakdata.conquery.models.auth.entities.User; -import com.bakdata.conquery.models.common.CDateSet; +import com.bakdata.conquery.models.config.ColumnConfig; import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.worker.Namespace; -import com.bakdata.conquery.util.DateReader; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import com.google.common.base.Strings; import com.univocity.parsers.common.record.Record; import com.univocity.parsers.csv.CsvParser; -import it.unimi.dsi.fastutil.ints.Int2ObjectAVLTreeMap; -import it.unimi.dsi.fastutil.ints.Int2ObjectMap; -import it.unimi.dsi.fastutil.ints.IntArrayList; -import it.unimi.dsi.fastutil.ints.IntList; -import lombok.Data; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; @Slf4j @@ -44,7 +35,14 @@ public class IdMappingConfig { DateColumn.EndDate.HANDLE, DateFormat.START_END_DATE ); - private List mappers = List.of(new InputMapper("ID", "id", "0", -1, Collections.emptyMap(), Collections.emptyMap())); + private List mappers = List.of( + ColumnConfig.builder() + .name("ID") + .mapping(ColumnConfig.Mapping.builder() + .field("id") + .resolvable(true) + .build()) + .build()); private OutputMapper outputMapper = new OutputMapper(); @@ -60,11 +58,12 @@ public EntityIdMap generateIdMapping(CsvParser parser) { while ((record = parser.parseNextRecord()) != null) { final String id = record.getString("id"); - for (InputMapper mapper : mappers) { - final String otherId = record.getString(mapper.getField()); - final String transformed = mapper.read(otherId); + for (ColumnConfig columnConfig : mappers) { - mapping.addInputMapping(id, mapper.getName(), transformed); + final String otherId = record.getString(columnConfig.getMapping().getField()); + final EntityIdMap.ExternalId transformed = columnConfig.read(otherId); + + mapping.addInputMapping(id, transformed); } mapping.addOutputMapping(id, outputMapper.extractOutputId(record)); @@ -109,15 +108,18 @@ public EntityPrintId toExternal(String csvEntityId, Namespace namespace, IdMappi return externalEntityId; } - public InputMapper getIdMapper(String name) { - return mappers.stream().filter(mapper -> mapper.getName().equals(name)).findFirst().orElse(null); + public ColumnConfig getIdMapper(String name) { + return mappers.stream() + .filter(mapper -> mapper.getName().equals(name)) //TODO use map + .findFirst() + .orElse(null); } public int getIdIndex(List format) { for (int index = 0; index < format.size(); index++) { final String current = format.get(index); - if (mappers.stream().map(InputMapper::getName).anyMatch(current::equals)) { + if (mappers.stream().map(ColumnConfig::getName).anyMatch(current::equals)) { return index; } } @@ -126,75 +128,7 @@ public int getIdIndex(List format) { } - public Int2ObjectMap readDates(String[][] values, List format, DateReader dateReader) { - Int2ObjectMap out = new Int2ObjectAVLTreeMap<>(); - - DateFormat dateFormat = null; - - - IntList dateColumns = new IntArrayList(format.size()); - - for (int col = 0; col < format.size(); col++) { - String desc = format.get(col); - - dateFormat = resolveDateFormat(desc); - - if (dateFormat == null) { - continue; - } - - dateColumns.add(col); - } - - dateFormat = dateFormat == null ? DateFormat.ALL : dateFormat; - final int[] datePositions = dateColumns.toIntArray(); - - for (int row = 1; row < values.length; row++) { - try { - final CDateSet dates = dateFormat.readDates(datePositions, values[row], dateReader); - if (dates == null) { - continue; - } - - out.put(row, dates); - } - catch (Exception e) { - log.warn("Failed to parse Date from {}", row, e); - } - } - - return out; - } - - public DateFormat resolveDateFormat(String desc) { - return dateFormats.get(desc); - } - - @Data - public static class InputMapper { - private final String name; - - private final String field; - - private final String pad; - private final int length; - - private final Map label; - private final Map description; - - public String read(String value) { - if (Strings.isNullOrEmpty(value)) { - return null; - } - - if (length == -1) { - return value; - } - - return StringUtils.leftPad(value, length, pad); - } - } @CPSBase @JsonTypeInfo(use = JsonTypeInfo.Id.CUSTOM, property = "type") diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ConfigResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ConfigResource.java index 89188ca248..b75f7df718 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ConfigResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ConfigResource.java @@ -20,6 +20,9 @@ public class ConfigResource { @GET @Path("frontend") public FrontendConfig getFrontendConfig() { - return config.getFrontend(); + + + return config.getFrontend() + .withQueryUpload(config.getFrontend().getQueryUpload().withIds(config.getIdMapping().getMappers())); } } \ No newline at end of file diff --git a/backend/src/test/java/com/bakdata/conquery/models/identifiable/IdMapSerialisationTest.java b/backend/src/test/java/com/bakdata/conquery/models/identifiable/IdMapSerialisationTest.java index 6e3eaf0367..a92ce8f261 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/identifiable/IdMapSerialisationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/identifiable/IdMapSerialisationTest.java @@ -8,7 +8,7 @@ public class IdMapSerialisationTest { public static EntityIdMap createTestPersistentMap() { EntityIdMap entityIdMap = new EntityIdMap(); - entityIdMap.addInputMapping("test1", "a"); + entityIdMap.addInputMapping("test1", new EntityIdMap.ExternalId(new String[]{"a"})); entityIdMap.addOutputMapping("test2", EntityPrintId.from("c")); From 986ec60878386f6f4f894f0599e70d4370b3e8e0 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Tue, 20 Jul 2021 16:28:19 +0200 Subject: [PATCH 13/82] Also make OutputMapping rely on ColumnConfig --- .../conquery/models/config/ColumnConfig.java | 83 +++++++++++++++++++ .../identifiable/mapping/IdMappingConfig.java | 56 +++++++------ 2 files changed, 115 insertions(+), 24 deletions(-) create mode 100644 backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java new file mode 100644 index 0000000000..52b3abcdce --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java @@ -0,0 +1,83 @@ +package com.bakdata.conquery.models.config; + +import java.util.Collections; +import java.util.Map; + +import javax.validation.constraints.NotEmpty; + +import com.bakdata.conquery.apiv1.query.concept.specific.external.FormatColumn; +import com.bakdata.conquery.io.cps.CPSTypeIdResolver; +import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.google.common.base.Strings; +import io.dropwizard.logback.shaded.checkerframework.checker.nullness.qual.Nullable; +import io.dropwizard.validation.ValidationMethod; +import lombok.Builder; +import lombok.Data; +import org.apache.commons.lang3.StringUtils; + +@Data +@Builder +public class ColumnConfig { + + @Data + @Builder + public static class Mapping { + private final String field; + + @Builder.Default + private final String pad = null; + @Builder.Default + private final int length = -1; + + @Builder.Default + private final boolean resolvable = true; + + private final boolean fillAnon = false; + + } + + public EntityIdMap.ExternalId read(String value) { + if(!mapping.isResolvable()){ + return null; //TODO throw Exception? + } + + if (Strings.isNullOrEmpty(value)) { + return null; + } + + if (getMapping().getLength() == -1) { + return new EntityIdMap.ExternalId(new String[]{getName(), value}); + } + + String padded = StringUtils.leftPad(value, getMapping().getLength(), getMapping().getPad()); + + return new EntityIdMap.ExternalId(new String[]{getName(), padded}); + + } + + + @NotEmpty + private final String name; + + /** + * Map of Localized labels. + */ + @Builder.Default + private final Map label = Collections.emptyMap(); + + /** + * Map of Localized description. + */ + @Builder.Default + private final Map description = Collections.emptyMap(); + + @Nullable + private final Mapping mapping; + + @JsonIgnore + @ValidationMethod + public boolean isExistingColumnFormat() { + return CPSTypeIdResolver.getImplementation(FormatColumn.class, name) != null; + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingConfig.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingConfig.java index d7d84b1408..6a82aa2be4 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingConfig.java @@ -1,17 +1,18 @@ package com.bakdata.conquery.models.identifiable.mapping; +import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; import com.bakdata.conquery.apiv1.query.concept.specific.external.DateColumn; import com.bakdata.conquery.apiv1.query.concept.specific.external.DateFormat; -import com.bakdata.conquery.io.cps.CPSBase; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.config.ColumnConfig; import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.worker.Namespace; import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.univocity.parsers.common.record.Record; import com.univocity.parsers.csv.CsvParser; import lombok.Getter; @@ -42,9 +43,9 @@ public class IdMappingConfig { .field("id") .resolvable(true) .build()) - .build()); - - private OutputMapper outputMapper = new OutputMapper(); + .build() + ); + private List idFieldsCached; /** * Read incoming CSV-file extracting Id-Mappings for in and Output. @@ -55,29 +56,53 @@ public EntityIdMap generateIdMapping(CsvParser parser) { Record record; + while ((record = parser.parseNextRecord()) != null) { + List idParts = new ArrayList<>(mappers.size()); + final String id = record.getString("id"); for (ColumnConfig columnConfig : mappers) { final String otherId = record.getString(columnConfig.getMapping().getField()); + + idParts.add(otherId); + + if (otherId == null) { + continue; + } + + if (!columnConfig.getMapping().isResolvable()) { + continue; + } + final EntityIdMap.ExternalId transformed = columnConfig.read(otherId); mapping.addInputMapping(id, transformed); } - mapping.addOutputMapping(id, outputMapper.extractOutputId(record)); + mapping.addOutputMapping(id, new EntityPrintId(idParts.toArray(new String[0]))); } return mapping; } + + /** * Headers for Output CSV. */ @JsonIgnore public List getPrintIdFields() { - return List.of(outputMapper.getHeaders()); + if(idFieldsCached == null) { + idFieldsCached = mappers.stream() + .map(ColumnConfig::getMapping) + .filter(Objects::nonNull) + .map(ColumnConfig.Mapping::getField) + .collect(Collectors.toList()); + } + + return idFieldsCached; } /** @@ -126,21 +151,4 @@ public int getIdIndex(List format) { return -1; } - - - - - @CPSBase - @JsonTypeInfo(use = JsonTypeInfo.Id.CUSTOM, property = "type") - @NoArgsConstructor - public static class OutputMapper { - @JsonIgnore - public String[] getHeaders() { - return new String[]{"result"}; - } - - public EntityPrintId extractOutputId(Record record) { - return new EntityPrintId(new String[]{record.getString("id")}); - } - } } From 32f832dd15fab9772be6938e99325c2ca20a4689 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Fri, 23 Jul 2021 14:47:03 +0200 Subject: [PATCH 14/82] migrate all usage to queryUpload config also implement resolving of DateFormat --- .../concept/specific/external/CQExternal.java | 31 ++--- .../concept/specific/external/DateColumn.java | 4 +- .../concept/specific/external/DateFormat.java | 40 ++++++- .../conquery/io/result/ResultUtil.java | 4 +- .../io/result/arrow/ResultArrowProcessor.java | 40 +++---- .../io/result/csv/ResultCsvProcessor.java | 7 +- .../io/result/excel/ResultExcelProcessor.java | 12 +- .../models/config/ConqueryConfig.java | 3 - .../models/config/FrontendConfig.java | 102 +++++++++++++++-- .../conquery/models/config/NoIdMapping.java | 23 ---- .../models/config/SimpleIdMapping.java | 20 ---- .../identifiable/mapping/EntityIdMap.java | 43 +++++++ .../identifiable/mapping/IdMappingConfig.java | 107 +----------------- .../conquery/models/query/ManagedQuery.java | 6 +- .../admin/rest/AdminDatasetProcessor.java | 2 +- .../resources/api/ConfigResource.java | 3 +- .../json/AbstractQueryEngineTest.java | 13 +-- .../conquery/integration/json/FormTest.java | 7 +- 18 files changed, 228 insertions(+), 239 deletions(-) delete mode 100644 backend/src/main/java/com/bakdata/conquery/models/config/NoIdMapping.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/models/config/SimpleIdMapping.java diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java index 3f0e336bef..9cee81794e 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java @@ -4,6 +4,8 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; import javax.validation.constraints.NotEmpty; @@ -72,25 +74,14 @@ public QPNode createQueryPlan(QueryPlanContext context, ConceptQueryPlan plan) { private Int2ObjectMap readDates(String[][] values, List format, DateReader dateReader, FrontendConfig.UploadConfig queryUpload) { Int2ObjectMap out = new Int2ObjectAVLTreeMap<>(); - DateFormat dateFormat = null; + List dateFormats = format.stream().map(queryUpload::resolveDateFormat).collect(Collectors.toList()); - IntList dateColumns = new IntArrayList(format.size()); + //validate structures - for (int col = 0; col < format.size(); col++) { - String desc = format.get(col); + final int[] datePositions = DateFormat.select(dateFormats); - dateFormat = queryUpload.resolveDateFormat(desc); - - if (dateFormat == null) { - continue; - } - - dateColumns.add(col); - } - - dateFormat = dateFormat == null ? DateFormat.ALL : dateFormat; - final int[] datePositions = dateColumns.toIntArray(); + DateFormat dateFormat = datePositions.length > 0 ? dateFormats.get(datePositions[0]) : DateFormat.ALL; for (int row = 1; row < values.length; row++) { try { @@ -118,14 +109,14 @@ public void resolve(QueryResolveContext context) { final DateReader dateReader = context.getConfig().getPreprocessor().getParsers().getDateReader(); - final IdMappingConfig mappingConfig = context.getConfig().getIdMapping(); - // extract dates from rows - final Int2ObjectMap rowDates = readDates(values, format, dateReader, context.getConfig().getFrontend().getQueryUpload()); + final FrontendConfig.UploadConfig uploadConfig = context.getConfig().getFrontend().getQueryUpload(); + + final Int2ObjectMap rowDates = readDates(values, format, dateReader, uploadConfig); - final int idIndex = mappingConfig.getIdIndex(format); + final int idIndex = uploadConfig.getIdIndex(format); - final ColumnConfig reader = mappingConfig.getIdMapper(format.get(idIndex)); + final ColumnConfig reader = uploadConfig.getIdMapper(format.get(idIndex)); final List unresolved = new ArrayList<>(); diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateColumn.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateColumn.java index 679189365c..da784920b5 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateColumn.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateColumn.java @@ -45,7 +45,7 @@ public static class StartDate extends DateColumn { public static String HANDLE = "START_DATE"; public StartDate() { - super(DateFormat.START_END_DATE); + super(DateFormat.START_DATE); } } @@ -53,7 +53,7 @@ public static class EndDate extends DateColumn { public static String HANDLE = "END_DATE"; public EndDate() { - super(DateFormat.START_END_DATE); + super(DateFormat.END_DATE); } } } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java index 96461ebfaf..d172a177c3 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java @@ -2,6 +2,9 @@ import java.time.LocalDate; import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.common.daterange.CDateRange; @@ -17,10 +20,25 @@ public CDateSet readDates(int[] formats, String[] row, DateReader dateReader) { return CDateSet.create(Collections.singleton(CDateRange.exactly(dateReader.parseToLocalDate(row[index])))); } }, - START_END_DATE { + END_DATE { @Override public CDateSet readDates(int[] formats, String[] row, DateReader dateReader) { + if (formats.length == 1) { + return CDateSet.create(CDateRange.atMost(dateReader.parseToLocalDate(row[0]))); + } + + return END_DATE.readDates(formats, row, dateReader); + } + }, + START_DATE { + @Override + public CDateSet readDates(int[] formats, String[] row, DateReader dateReader) { + + if (formats.length == 1) { + return CDateSet.create(CDateRange.atLeast(dateReader.parseToLocalDate(row[0]))); + } + final int startIndex = formats[0]; final int endIndex = formats[1]; @@ -59,4 +77,24 @@ public CDateSet readDates(int[] formats, String[] row, DateReader dateReader) { public abstract CDateSet readDates(int[] formats, String[] row, DateReader dateReader); + public static int[] select(List dateFormat) { + final List distinct = dateFormat.stream().filter(Objects::nonNull).distinct().sorted().collect(Collectors.toList()); + + // => ALL + if (distinct.isEmpty()) { + return new int[0]; + } + + + if (distinct.size() == 1) { + return new int[]{dateFormat.indexOf(distinct.get(0))}; + } + + if (distinct.size() == 2 && distinct.get(0).equals(START_DATE) && distinct.get(1).equals(END_DATE)) { + return new int[]{dateFormat.indexOf(START_DATE), dateFormat.indexOf(END_DATE)}; + } + + throw new IllegalStateException("can only handle 1 or 2 format columns"); //TODO map error + + } } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/ResultUtil.java b/backend/src/main/java/com/bakdata/conquery/io/result/ResultUtil.java index cc0b805c12..974fd57aec 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/ResultUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/ResultUtil.java @@ -24,11 +24,11 @@ public class ResultUtil { - public static EntityPrintId createId(Namespace namespace, EntityResult cer, IdMappingConfig idMappingConfig, IdMappingState mappingState) { + public static EntityPrintId createId(Namespace namespace, EntityResult cer, IdMappingState mappingState) { EncodedDictionary dict = namespace.getStorage().getPrimaryDictionary(); final EntityIdMap idMapping = namespace.getStorage().getIdMapping(); - return idMappingConfig.toExternal( + return IdMappingConfig.toExternal( dict.getElement(cer.getEntityId()), namespace, mappingState, idMapping ); diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java index 769dc1691f..237d56e4da 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java @@ -3,15 +3,10 @@ import static com.bakdata.conquery.io.result.ResultUtil.makeResponseWithFileName; import static com.bakdata.conquery.io.result.arrow.ArrowRenderer.renderToStream; import static com.bakdata.conquery.models.auth.AuthorizationHelper.authorizeDownloadDatasets; -import static com.bakdata.conquery.resources.ResourceConstants.FILE_EXTENTION_ARROW_FILE; -import static com.bakdata.conquery.resources.ResourceConstants.FILE_EXTENTION_ARROW_STREAM; -import java.io.IOException; import java.io.OutputStream; -import java.nio.channels.Channels; import java.util.function.Function; -import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.Response; import javax.ws.rs.core.StreamingOutput; @@ -32,13 +27,9 @@ import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.util.io.ConqueryMDC; -import lombok.RequiredArgsConstructor; import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.arrow.vector.dictionary.DictionaryProvider; -import org.apache.arrow.vector.ipc.ArrowFileWriter; -import org.apache.arrow.vector.ipc.ArrowStreamWriter; import org.apache.arrow.vector.ipc.ArrowWriter; import org.apache.http.HttpStatus; @@ -75,29 +66,24 @@ public static & SingleTableResult> Response getAr } // Get the locale extracted by the LocaleFilter - IdMappingConfig idMappingConf = config.getIdMapping(); - IdMappingState mappingState = config.getIdMapping().initToExternal(user, exec); + IdMappingState mappingState = IdMappingConfig.initToExternal(user, exec); PrintSettings settings = new PrintSettings( pretty, I18n.LOCALE.get(), datasetRegistry, config, - (EntityResult cer) -> ResultUtil.createId(namespace, cer, config.getIdMapping(), mappingState)); - - - StreamingOutput out = new StreamingOutput() { - - @Override - public void write(OutputStream output) throws IOException, WebApplicationException { - renderToStream(writerProducer.apply(output), - settings, - config.getArrow().getBatchSize(), - idMappingConf.getPrintIdFields(), - exec.getResultInfo(), - exec.streamResults()); - - } - }; + (EntityResult cer) -> ResultUtil.createId(namespace, cer, mappingState) + ); + + + StreamingOutput out = output -> renderToStream( + writerProducer.apply(output), + settings, + config.getArrow().getBatchSize(), + config.getFrontend().getQueryUpload().getPrintIdFields(), + exec.getResultInfo(), + exec.streamResults() + ); return makeResponseWithFileName(out, exec.getLabelWithoutAutoLabelSuffix(), fileExtension); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java index b9e9b81ccc..3f6ac4b16b 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java @@ -19,6 +19,7 @@ import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.i18n.I18n; +import com.bakdata.conquery.models.identifiable.mapping.IdMappingConfig; import com.bakdata.conquery.models.identifiable.mapping.IdMappingState; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.SingleTableResult; @@ -49,7 +50,7 @@ public & SingleTableResult> Response getResult(Us // Check if user is permitted to download on all datasets that were referenced by the query authorizeDownloadDatasets(user, exec); - IdMappingState mappingState = config.getIdMapping().initToExternal(user, exec); + IdMappingState mappingState = IdMappingConfig.initToExternal(user, exec); // Get the locale extracted by the LocaleFilter PrintSettings settings = new PrintSettings( @@ -57,7 +58,7 @@ public & SingleTableResult> Response getResult(Us I18n.LOCALE.get(), datasetRegistry, config, - cer -> ResultUtil.createId(namespace, cer, config.getIdMapping(), mappingState) + cer -> ResultUtil.createId(namespace, cer, mappingState) ); Charset charset = determineCharset(userAgent, queryCharset); @@ -65,7 +66,7 @@ public & SingleTableResult> Response getResult(Us StreamingOutput out = os -> { try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(os, charset))) { CsvRenderer renderer = new CsvRenderer(config.getCsv().createWriter(writer), settings); - renderer.toCSV(config.getIdMapping().getPrintIdFields(), exec.getResultInfo(), exec.streamResults()); + renderer.toCSV(config.getFrontend().getQueryUpload().getPrintIdFields(), exec.getResultInfo(), exec.streamResults()); } catch (EofException e) { log.info("User canceled download"); diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java index 5ce26459d5..86e08ea07d 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java @@ -1,7 +1,6 @@ package com.bakdata.conquery.io.result.excel; import com.bakdata.conquery.io.result.ResultUtil; -import com.bakdata.conquery.io.result.excel.ExcelRenderer; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.auth.permissions.Ability; import com.bakdata.conquery.models.config.ConqueryConfig; @@ -9,7 +8,6 @@ import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.i18n.I18n; import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; -import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.identifiable.mapping.IdMappingConfig; import com.bakdata.conquery.models.identifiable.mapping.IdMappingState; import com.bakdata.conquery.models.query.PrintSettings; @@ -17,15 +15,12 @@ import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.Namespace; -import com.bakdata.conquery.util.ResourceUtil; import com.bakdata.conquery.util.io.ConqueryMDC; import lombok.RequiredArgsConstructor; -import javax.ws.rs.BadRequestException; import javax.ws.rs.core.Response; import javax.ws.rs.core.StreamingOutput; -import static com.bakdata.conquery.io.result.ResultUtil.checkSingleTableResult; import static com.bakdata.conquery.io.result.ResultUtil.makeResponseWithFileName; @RequiredArgsConstructor @@ -44,20 +39,19 @@ public & SingleTableResult> Response getExcelResu user.authorize(dataset, Ability.DOWNLOAD); user.authorize(exec, Ability.READ); - IdMappingConfig idMapping = config.getIdMapping(); - IdMappingState mappingState = idMapping.initToExternal(user, exec); + IdMappingState mappingState = IdMappingConfig.initToExternal(user, exec); PrintSettings settings = new PrintSettings( pretty, I18n.LOCALE.get(), datasetRegistry, config, - (EntityResult cer) -> ResultUtil.createId(namespace, cer, idMapping, mappingState)); + (EntityResult cer) -> ResultUtil.createId(namespace, cer, mappingState)); ExcelRenderer excelRenderer = new ExcelRenderer(config.getExcel()); StreamingOutput out = output -> excelRenderer.renderToStream( settings, - idMapping.getPrintIdFields(), + config.getFrontend().getQueryUpload().getPrintIdFields(), (ManagedExecution & SingleTableResult)exec, output ); diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java index 6e1275af23..54e931acc4 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java @@ -80,9 +80,6 @@ public class ConqueryConfig extends Configuration { private ConqueryMetricsConfig metricsConfig = new ConqueryMetricsConfig(); - @NotNull - @Valid - private IdMappingConfig idMapping = new NoIdMapping(); @Valid @NotNull private List authentication = List.of(new DevAuthConfig()); diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java index cbef3e2457..134edcde2e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java @@ -1,22 +1,28 @@ package com.bakdata.conquery.models.config; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; import javax.validation.Valid; import javax.validation.constraints.NotEmpty; import javax.validation.constraints.NotNull; -import com.bakdata.conquery.apiv1.query.concept.specific.external.DateColumn; import com.bakdata.conquery.apiv1.query.concept.specific.external.DateFormat; import com.bakdata.conquery.util.VersionInfo; +import com.fasterxml.jackson.annotation.JsonIgnore; import groovy.transform.ToString; +import io.dropwizard.validation.ValidationMethod; import lombok.AllArgsConstructor; import lombok.Data; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; import lombok.With; +import lombok.extern.slf4j.Slf4j; @ToString @Getter @@ -24,6 +30,7 @@ @With @AllArgsConstructor @NoArgsConstructor +@Slf4j public class FrontendConfig { private String version = VersionInfo.INSTANCE.getProjectVersion(); @@ -41,18 +48,64 @@ public class FrontendConfig { public static class UploadConfig { @NotEmpty - private List ids; + private List ids = List.of( + ColumnConfig.builder() + .name("ID") + .mapping(ColumnConfig.Mapping.builder() + .field("id") + .resolvable(true) + .build()) + .build() + ); + + @JsonIgnore + private List idFieldsCached; + + /** + * Headers for Output CSV. + */ + @JsonIgnore + public List getPrintIdFields() { + if (idFieldsCached == null) { + idFieldsCached = ids.stream() + .map(ColumnConfig::getMapping) + .filter(Objects::nonNull) + .map(ColumnConfig.Mapping::getField) + .collect(Collectors.toList()); + } + + return idFieldsCached; + } + + public ColumnConfig getIdMapper(String name) { + return ids.stream() + .filter(mapper -> mapper.getName().equals(name)) //TODO use map + .findFirst() + .orElse(null); + } + + public int getIdIndex(List format) { + for (int index = 0; index < format.size(); index++) { + final String current = format.get(index); + + if (ids.stream().map(ColumnConfig::getName).anyMatch(current::equals)) { + return index; + } + } + + return -1; + } @NotNull private ColumnConfig dateStart = ColumnConfig.builder() - .name(DateColumn.StartDate.HANDLE) + .name(DateFormat.START_DATE.name()) .label(Map.of("en", "Begin")) .description(Map.of("en", "Begin of Date-range")) .build(); @NotNull private ColumnConfig dateEnd = ColumnConfig.builder() - .name(DateColumn.EndDate.HANDLE) + .name(DateFormat.END_DATE.name()) .label(Map.of("en", "End")) .description(Map.of("en", "End of Date-range")) .build(); @@ -60,7 +113,7 @@ public static class UploadConfig { @NotNull private ColumnConfig dateRange = ColumnConfig.builder() - .name(DateColumn.DateRange.HANDLE) + .name(DateFormat.DATE_RANGE.name()) .label(Map.of("en", "Date Range")) .description(Map.of("en", "Full Date Range")) .build(); @@ -68,7 +121,7 @@ public static class UploadConfig { @NotNull private ColumnConfig dateSet = ColumnConfig.builder() - .name(DateColumn.DateSet.HANDLE) + .name(DateFormat.DATE_SET.name()) .label(Map.of("en", "Dateset")) .description(Map.of("en", "Set of Date-Ranges")) .build(); @@ -76,13 +129,46 @@ public static class UploadConfig { @NotNull private ColumnConfig eventDate = ColumnConfig.builder() - .name(DateColumn.EventDate.HANDLE) + .name(DateFormat.EVENT_DATE.name()) .label(Map.of("en", "Event Date")) .description(Map.of("en", "Single event")) .build(); + @ValidationMethod(message = "Duplicate Claims for Mapping Columns.") + @JsonIgnore + public boolean isAllColsUnique() { + Map dupes = new HashMap<>(); + + final ArrayList candidates = new ArrayList<>(ids); + candidates.addAll(List.of(dateStart, dateEnd, dateSet, dateRange, eventDate)); + + for (ColumnConfig config : candidates) { + final ColumnConfig prior = dupes.put(config.getName(), config); + + if (prior == null) { + continue; + } + + log.error("Duplicate claims for Name = `{}` ({} / {})", config.getName(), config, prior); + return false; + } + + return true; + } + + @ValidationMethod(message = "Not all Id-Columns have mappings.") + @JsonIgnore + public boolean isIdColsHaveMapping() { + return ids.stream().map(ColumnConfig::getMapping).allMatch(Objects::nonNull); + } + public DateFormat resolveDateFormat(String handle) { - return DateFormat.ALL; //TODO + try { + return DateFormat.valueOf(handle); + } + catch (IllegalArgumentException e) { + return null; // Does not exist + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/NoIdMapping.java b/backend/src/main/java/com/bakdata/conquery/models/config/NoIdMapping.java deleted file mode 100644 index f7a84dbb06..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/models/config/NoIdMapping.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.bakdata.conquery.models.config; - -import java.util.Arrays; -import java.util.List; - -import com.bakdata.conquery.io.cps.CPSType; -import com.bakdata.conquery.models.identifiable.mapping.IdMappingConfig; -import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; -import com.univocity.parsers.common.record.Record; - -/** - * Disables Id-Mapping completely. - */ -public class NoIdMapping extends IdMappingConfig { - private static final String[] HEADER = new String[]{"result"}; - - - @Override - public List getPrintIdFields() { - return Arrays.asList(HEADER); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/SimpleIdMapping.java b/backend/src/main/java/com/bakdata/conquery/models/config/SimpleIdMapping.java deleted file mode 100644 index 597b3d941a..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/models/config/SimpleIdMapping.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.bakdata.conquery.models.config; - -import java.util.HashMap; -import java.util.Map; - -import com.bakdata.conquery.apiv1.query.concept.specific.external.FormatColumn; -import com.bakdata.conquery.apiv1.query.concept.specific.external.IdColumn; -import com.bakdata.conquery.io.cps.CPSType; -import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; -import com.bakdata.conquery.models.identifiable.mapping.EntityPrintId; -import com.bakdata.conquery.models.identifiable.mapping.IdMappingConfig; -import com.univocity.parsers.common.record.Record; - -/** - * Maps input to itself for upload. - */ -public class SimpleIdMapping extends NoIdMapping { - - -} diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java index 2737e79a1b..5cbbeee4df 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java @@ -5,10 +5,13 @@ import java.util.List; import java.util.Map; +import com.bakdata.conquery.models.config.ColumnConfig; import com.bakdata.conquery.models.dictionary.EncodedDictionary; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonValue; +import com.univocity.parsers.common.record.Record; +import com.univocity.parsers.csv.CsvParser; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.Getter; @@ -43,6 +46,46 @@ public class EntityIdMap { @JsonIgnore private final Map external2Internal = new HashMap<>(); + /** + * Read incoming CSV-file extracting Id-Mappings for in and Output. + */ + public static EntityIdMap generateIdMapping(CsvParser parser, List mappers) { + + EntityIdMap mapping = new EntityIdMap(); + + Record record; + + + while ((record = parser.parseNextRecord()) != null) { + List idParts = new ArrayList<>(mappers.size()); + + final String id = record.getString("id"); + + for (ColumnConfig columnConfig : mappers) { + + final String otherId = record.getString(columnConfig.getMapping().getField()); + + idParts.add(otherId); + + if (otherId == null) { + continue; + } + + if (!columnConfig.getMapping().isResolvable()) { + continue; + } + + final ExternalId transformed = columnConfig.read(otherId); + + mapping.addInputMapping(id, transformed); + } + + mapping.addOutputMapping(id, new EntityPrintId(idParts.toArray(new String[0]))); + } + + return mapping; + } + /** * Helper class for serialization. diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingConfig.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingConfig.java index 6a82aa2be4..6ed66d726c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingConfig.java @@ -2,19 +2,14 @@ import java.util.ArrayList; import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; -import com.bakdata.conquery.apiv1.query.concept.specific.external.DateColumn; -import com.bakdata.conquery.apiv1.query.concept.specific.external.DateFormat; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.config.ColumnConfig; import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.worker.Namespace; -import com.fasterxml.jackson.annotation.JsonIgnore; import com.univocity.parsers.common.record.Record; import com.univocity.parsers.csv.CsvParser; +import lombok.AccessLevel; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; @@ -22,94 +17,17 @@ @Slf4j -@NoArgsConstructor +@NoArgsConstructor(access = AccessLevel.NONE) @Setter @Getter public class IdMappingConfig { - public final Map dateFormats = Map.of( - DateColumn.DateSet.HANDLE, DateFormat.DATE_SET, - DateColumn.DateRange.HANDLE, DateFormat.DATE_RANGE, - DateColumn.EventDate.HANDLE, DateFormat.EVENT_DATE, - DateColumn.StartDate.HANDLE, DateFormat.START_END_DATE, - DateColumn.EndDate.HANDLE, DateFormat.START_END_DATE - ); - - private List mappers = List.of( - ColumnConfig.builder() - .name("ID") - .mapping(ColumnConfig.Mapping.builder() - .field("id") - .resolvable(true) - .build()) - .build() - ); - private List idFieldsCached; - - /** - * Read incoming CSV-file extracting Id-Mappings for in and Output. - */ - public EntityIdMap generateIdMapping(CsvParser parser) { - - EntityIdMap mapping = new EntityIdMap(); - - Record record; - - - while ((record = parser.parseNextRecord()) != null) { - List idParts = new ArrayList<>(mappers.size()); - - final String id = record.getString("id"); - - for (ColumnConfig columnConfig : mappers) { - - final String otherId = record.getString(columnConfig.getMapping().getField()); - - idParts.add(otherId); - - if (otherId == null) { - continue; - } - - if (!columnConfig.getMapping().isResolvable()) { - continue; - } - - final EntityIdMap.ExternalId transformed = columnConfig.read(otherId); - - mapping.addInputMapping(id, transformed); - } - - mapping.addOutputMapping(id, new EntityPrintId(idParts.toArray(new String[0]))); - } - - return mapping; - } - - - - /** - * Headers for Output CSV. - */ - @JsonIgnore - public List getPrintIdFields() { - if(idFieldsCached == null) { - idFieldsCached = mappers.stream() - .map(ColumnConfig::getMapping) - .filter(Objects::nonNull) - .map(ColumnConfig.Mapping::getField) - .collect(Collectors.toList()); - } - - return idFieldsCached; - } - /** * Is called once before a mapping is used before a query result is created to * allow the mapping to have state information. */ - public IdMappingState initToExternal(User user, ManagedExecution execution) { + public static IdMappingState initToExternal(User user, ManagedExecution execution) { // This mapping does not need a per-query state, so we return an immutable empty map. return null; } @@ -117,7 +35,7 @@ public IdMappingState initToExternal(User user, ManagedExecution execution) { /** * Converts the internal ID to the an external. */ - public EntityPrintId toExternal(String csvEntityId, Namespace namespace, IdMappingState state, EntityIdMap mapping) { + public static EntityPrintId toExternal(String csvEntityId, Namespace namespace, IdMappingState state, EntityIdMap mapping) { // The state may be uses by implementations of this class if (mapping == null) { @@ -133,22 +51,5 @@ public EntityPrintId toExternal(String csvEntityId, Namespace namespace, IdMappi return externalEntityId; } - public ColumnConfig getIdMapper(String name) { - return mappers.stream() - .filter(mapper -> mapper.getName().equals(name)) //TODO use map - .findFirst() - .orElse(null); - } - - public int getIdIndex(List format) { - for (int index = 0; index < format.size(); index++) { - final String current = format.get(index); - - if (mappers.stream().map(ColumnConfig::getName).anyMatch(current::equals)) { - return index; - } - } - return -1; - } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java b/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java index e2c5a96dcf..c8b4c33517 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java @@ -95,7 +95,7 @@ protected void doInitExecutable(@NonNull DatasetRegistry namespaces, ConqueryCon involvedWorkers = namespace.getWorkers().size(); - query.resolve(new QueryResolveContext(getDataset(), namespaces, config,null)); + query.resolve(new QueryResolveContext(getDataset(), namespaces, config, null)); } @Override @@ -161,13 +161,15 @@ protected void setAdditionalFieldsForStatusWithColumnDescription(@NonNull MetaSt public List generateColumnDescriptions(DatasetRegistry datasetRegistry) { Preconditions.checkArgument(isInitialized(), "The execution must have been initialized first"); List columnDescriptions = new ArrayList<>(); + // First add the id columns to the descriptor list. The are the first columns - for (String header : config.getIdMapping().getPrintIdFields()) { + for (String header : config.getFrontend().getQueryUpload().getPrintIdFields()) { columnDescriptions.add(ColumnDescriptor.builder() .label(header) .type(ResultType.IdT.INSTANCE.typeInfo()) .build()); } + // Then all columns that originate from selects and static aggregators PrintSettings settings = new PrintSettings(true, I18n.LOCALE.get(), datasetRegistry, config, null); diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java index 1c075c5ad2..aafaf1dee9 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java @@ -260,7 +260,7 @@ public void setIdMapping(InputStream data, Namespace namespace) { parser.beginParsing(data); - EntityIdMap mapping = config.getIdMapping().generateIdMapping(parser); + EntityIdMap mapping = EntityIdMap.generateIdMapping(parser, config.getFrontend().getQueryUpload().getIds()); namespace.getStorage().updateIdMapping(mapping); } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ConfigResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ConfigResource.java index b75f7df718..a7655e84e3 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ConfigResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ConfigResource.java @@ -22,7 +22,6 @@ public class ConfigResource { public FrontendConfig getFrontendConfig() { - return config.getFrontend() - .withQueryUpload(config.getFrontend().getQueryUpload().withIds(config.getIdMapping().getMappers())); + return config.getFrontend(); } } \ No newline at end of file diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java b/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java index b350b108d4..f98a1bea0b 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java @@ -6,24 +6,19 @@ import java.io.IOException; import java.util.List; import java.util.Locale; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import com.bakdata.conquery.apiv1.query.Query; import com.bakdata.conquery.integration.common.IntegrationUtils; -import com.bakdata.conquery.integration.common.LoadingUtil; import com.bakdata.conquery.integration.common.ResourceFile; import com.bakdata.conquery.io.result.CsvLineStreamRenderer; -import com.bakdata.conquery.io.result.ResultTestUtil; import com.bakdata.conquery.io.result.ResultUtil; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.config.ConqueryConfig; -import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.execution.ExecutionState; -import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; +import com.bakdata.conquery.models.identifiable.mapping.IdMappingConfig; import com.bakdata.conquery.models.identifiable.mapping.IdMappingState; -import com.bakdata.conquery.models.query.ExecutionManager; import com.bakdata.conquery.models.query.ManagedQuery; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; @@ -74,7 +69,7 @@ public void executeTest(StandaloneSupport standaloneSupport) throws IOException .as("Should have same size as result infos") .allSatisfy(v -> assertThat(v).hasSameSizeAs(resultInfos)); - IdMappingState mappingState = config.getIdMapping().initToExternal(testUser, execution); + IdMappingState mappingState = IdMappingConfig.initToExternal(testUser, execution); PrintSettings PRINT_SETTINGS = new PrintSettings( @@ -82,14 +77,14 @@ public void executeTest(StandaloneSupport standaloneSupport) throws IOException Locale.ENGLISH, namespaces, config, - cer -> ResultUtil.createId(standaloneSupport.getNamespace(), cer, config.getIdMapping(), mappingState), + cer -> ResultUtil.createId(standaloneSupport.getNamespace(), cer,mappingState), (columnInfo) -> columnInfo.getSelect().getId().toStringWithoutDataset() ); CsvLineStreamRenderer renderer = new CsvLineStreamRenderer(config.getCsv().createWriter(), PRINT_SETTINGS); List actual = renderer.toStream( - config.getIdMapping().getPrintIdFields(), + config.getFrontend().getQueryUpload().getPrintIdFields(), resultInfos, execution.streamResults() ).collect(Collectors.toList()); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java b/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java index 9abeee7fee..bb86e4efd6 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java @@ -105,7 +105,6 @@ public void importRequiredData(StandaloneSupport support) throws Exception { log.info("{} PARSE JSON FORM DESCRIPTION", getLabel()); form = parseForm(support); - idMappingConfig = support.getConfig().getIdMapping(); } @Override @@ -137,7 +136,7 @@ public void executeTest(StandaloneSupport support) throws Exception { private void checkResults(StandaloneSupport standaloneSupport, ManagedForm managedForm, User user) throws IOException { Map> managedMapping = managedForm.getSubQueries(); - IdMappingState mappingState = idMappingConfig.initToExternal(user, managedForm); + IdMappingState mappingState = IdMappingConfig.initToExternal(user, managedForm); final ConqueryConfig config = standaloneSupport.getConfig(); PrintSettings PRINT_SETTINGS = @@ -146,7 +145,7 @@ private void checkResults(StandaloneSupport standaloneSupport, ManagedForm manag Locale.ENGLISH, standaloneSupport.getDatasetsProcessor().getDatasetRegistry(), config, - cer -> ResultUtil.createId(standaloneSupport.getNamespace(), cer, config.getIdMapping(), mappingState) + cer -> ResultUtil.createId(standaloneSupport.getNamespace(), cer, mappingState) ); CsvLineStreamRenderer renderer = new CsvLineStreamRenderer(config.getCsv().createWriter(), PRINT_SETTINGS); @@ -156,7 +155,7 @@ private void checkResults(StandaloneSupport standaloneSupport, ManagedForm manag log.info("{} CSV TESTING: {}", getLabel(), managed.getKey()); List actual = renderer.toStream( - config.getIdMapping().getPrintIdFields(), + config.getFrontend().getQueryUpload().getPrintIdFields(), resultInfos, managed.getValue().stream().flatMap(ManagedQuery::streamResults) ) From f832fe97d00991afb963fde273c316f2db688484 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Fri, 23 Jul 2021 14:48:01 +0200 Subject: [PATCH 15/82] also migrate Pseudomizer --- .../mapping/AutoIncrementingPseudomizer.java | 43 +++++++++++++++++++ .../mapping/PseudomizationTest.java | 28 ++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/AutoIncrementingPseudomizer.java create mode 100644 backend/src/test/java/com/bakdata/conquery/models/identifiable/mapping/PseudomizationTest.java diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/AutoIncrementingPseudomizer.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/AutoIncrementingPseudomizer.java new file mode 100644 index 0000000000..71c0981299 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/AutoIncrementingPseudomizer.java @@ -0,0 +1,43 @@ +package com.bakdata.conquery.models.identifiable.mapping; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + +import com.bakdata.conquery.models.identifiable.mapping.EntityPrintId; +import lombok.RequiredArgsConstructor; + +/** + * Used to generate pseudonymized ids for every query if the user does + * not have the rights to see the real data. + */ +@RequiredArgsConstructor +public class AutoIncrementingPseudomizer { + private static final String ANONYMOUS_ID_PREFIX = "anon_"; // Abbreviation for anonymous + + private final Map cachedIds = new ConcurrentHashMap<>(); + private final AtomicInteger pseudoIdPointer = new AtomicInteger(0); + + private final int size; + private final int position; + + /** + * In the pseudo format the actual id columns are preserved but empty. + * Only the Pid Column is written with a new generated id. + */ + public EntityPrintId getPseudoId(String csvEntityId) { + + EntityPrintId pseudonym = cachedIds.computeIfAbsent(csvEntityId, this::createPseudonym); + + return pseudonym; + } + + private EntityPrintId createPseudonym(String ignored) { + final String name = ANONYMOUS_ID_PREFIX + pseudoIdPointer.getAndIncrement(); + final String[] parts = new String[size]; + + parts[position] = name; + + return EntityPrintId.from(parts); + } +} diff --git a/backend/src/test/java/com/bakdata/conquery/models/identifiable/mapping/PseudomizationTest.java b/backend/src/test/java/com/bakdata/conquery/models/identifiable/mapping/PseudomizationTest.java new file mode 100644 index 0000000000..354172228b --- /dev/null +++ b/backend/src/test/java/com/bakdata/conquery/models/identifiable/mapping/PseudomizationTest.java @@ -0,0 +1,28 @@ +package com.bakdata.conquery.models.identifiable.mapping; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +public class PseudomizationTest { + + @Test + void pseudoIdGeneration() { + AutoIncrementingPseudomizer pseudomizer = new AutoIncrementingPseudomizer(4,2); + + String csvId1 = "123"; + String csvId1Copy = "123"; + String csvId2 = "234"; + + // Id changes from internal to external + assertThat(pseudomizer.getPseudoId(csvId1)).isEqualTo(EntityPrintId.from(null, null, "anon_0", null)); + + // Id mapping is constant + assertThat(pseudomizer.getPseudoId(csvId1)).isEqualTo(pseudomizer.getPseudoId(csvId1Copy)); + + // Mapping produces differing external ids + assertThat(pseudomizer.getPseudoId(csvId1)).isNotEqualTo(pseudomizer.getPseudoId(csvId2)); + + } + +} From a8bdcdaa8b6b642fff7d3fd4350c0f7d2c906ae1 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Fri, 23 Jul 2021 15:29:15 +0200 Subject: [PATCH 16/82] move Pseduomization to IdPrinter and Conquery --- .../concept/specific/external/CQExternal.java | 2 +- .../concept/specific/external/DateColumn.java | 59 ------------------- .../specific/external/DateSetColumn.java | 6 -- .../specific/external/FormatColumn.java | 10 ---- .../concept/specific/external/IdColumn.java | 14 ----- .../specific/external/IgnoreColumn.java | 12 ---- .../conquery/io/result/ResultUtil.java | 17 ------ .../io/result/arrow/ResultArrowProcessor.java | 11 ++-- .../io/result/csv/ResultCsvProcessor.java | 10 ++-- .../io/result/excel/ResultExcelProcessor.java | 11 ++-- .../conquery/models/config/ColumnConfig.java | 10 +--- .../models/config/ConqueryConfig.java | 2 +- .../models/config/FrontendConfig.java | 41 ++++++++++++- .../mapping/AutoIncrementingPseudomizer.java | 24 ++++---- .../identifiable/mapping/FullIdPrinter.java | 35 +++++++++++ .../identifiable/mapping/IdMappingConfig.java | 55 ----------------- .../identifiable/mapping/IdMappingState.java | 8 --- .../identifiable/mapping/IdPrinter.java | 7 +++ .../json/AbstractQueryEngineTest.java | 12 ++-- .../conquery/integration/json/FormTest.java | 14 ++--- .../mapping/PseudomizationTest.java | 12 ++-- 21 files changed, 128 insertions(+), 244 deletions(-) delete mode 100644 backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateColumn.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateSetColumn.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/FormatColumn.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/IdColumn.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/IgnoreColumn.java create mode 100644 backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/FullIdPrinter.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingConfig.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingState.java create mode 100644 backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdPrinter.java diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java index 9cee81794e..ba4994e0f2 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java @@ -17,7 +17,7 @@ import com.bakdata.conquery.models.config.FrontendConfig; import com.bakdata.conquery.models.error.ConqueryError; import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; -import com.bakdata.conquery.models.identifiable.mapping.IdMappingConfig; + import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; import com.bakdata.conquery.models.query.queryplan.ConceptQueryPlan; diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateColumn.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateColumn.java deleted file mode 100644 index da784920b5..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateColumn.java +++ /dev/null @@ -1,59 +0,0 @@ -package com.bakdata.conquery.apiv1.query.concept.specific.external; - -import com.bakdata.conquery.io.cps.CPSType; -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.With; - -@Getter -public abstract class DateColumn extends FormatColumn { - - private final DateFormat format; - - protected DateColumn(DateFormat format) { - this.format = format; - } - - - public static class DateSet extends DateColumn { - public static String HANDLE = "DATE_SET"; - - public DateSet() { - super(DateFormat.DATE_SET); - } - } - - public static class DateRange extends DateColumn { - public static String HANDLE = "DATE_RANGE"; - - public DateRange() { - super(DateFormat.DATE_RANGE); - } - } - - public static class EventDate extends DateColumn { - public static String HANDLE = "EVENT_DATE"; - - - public EventDate() { - super(DateFormat.EVENT_DATE); - } - } - - - public static class StartDate extends DateColumn { - public static String HANDLE = "START_DATE"; - - public StartDate() { - super(DateFormat.START_DATE); - } - } - - public static class EndDate extends DateColumn { - public static String HANDLE = "END_DATE"; - - public EndDate() { - super(DateFormat.END_DATE); - } - } -} diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateSetColumn.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateSetColumn.java deleted file mode 100644 index 80efa72de5..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateSetColumn.java +++ /dev/null @@ -1,6 +0,0 @@ -package com.bakdata.conquery.apiv1.query.concept.specific.external; - -import com.bakdata.conquery.apiv1.forms.Form; -import com.bakdata.conquery.io.cps.CPSType; - - diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/FormatColumn.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/FormatColumn.java deleted file mode 100644 index 27f769c8e8..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/FormatColumn.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.bakdata.conquery.apiv1.query.concept.specific.external; - -import lombok.Data; -import lombok.With; -import lombok.extern.slf4j.Slf4j; - - -public abstract class FormatColumn { - -} diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/IdColumn.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/IdColumn.java deleted file mode 100644 index 1877249878..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/IdColumn.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.bakdata.conquery.apiv1.query.concept.specific.external; - -import lombok.AllArgsConstructor; -import lombok.Value; -import lombok.With; - -public class IdColumn extends FormatColumn { - - public static String HANDLE = "ID"; - - public String[] read(String[] row, int position) { - return new String[]{row[position]}; - } -} diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/IgnoreColumn.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/IgnoreColumn.java deleted file mode 100644 index 4a1c314d84..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/IgnoreColumn.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.bakdata.conquery.apiv1.query.concept.specific.external; - -import com.bakdata.conquery.io.cps.CPSType; - -public class IgnoreColumn extends FormatColumn { - - public static final String HANDLE = "IGNORED"; - - public String read(String[] row) { - return null; - } -} diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/ResultUtil.java b/backend/src/main/java/com/bakdata/conquery/io/result/ResultUtil.java index 974fd57aec..0e8eb2e94d 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/ResultUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/ResultUtil.java @@ -7,15 +7,8 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.StreamingOutput; -import com.bakdata.conquery.models.dictionary.EncodedDictionary; import com.bakdata.conquery.models.execution.ManagedExecution; -import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; -import com.bakdata.conquery.models.identifiable.mapping.EntityPrintId; -import com.bakdata.conquery.models.identifiable.mapping.IdMappingConfig; -import com.bakdata.conquery.models.identifiable.mapping.IdMappingState; import com.bakdata.conquery.models.query.SingleTableResult; -import com.bakdata.conquery.models.query.results.EntityResult; -import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.util.io.FileUtil; import com.google.common.base.Strings; import lombok.extern.slf4j.Slf4j; @@ -24,16 +17,6 @@ public class ResultUtil { - public static EntityPrintId createId(Namespace namespace, EntityResult cer, IdMappingState mappingState) { - EncodedDictionary dict = namespace.getStorage().getPrimaryDictionary(); - final EntityIdMap idMapping = namespace.getStorage().getIdMapping(); - - return IdMappingConfig.toExternal( - dict.getElement(cer.getEntityId()), namespace, - mappingState, idMapping - ); - } - public static Response makeResponseWithFileName(StreamingOutput out, String label, String fileExtension) { Response.ResponseBuilder response = Response.ok(out); if (!(Strings.isNullOrEmpty(label) || label.isBlank())) { diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java index 237d56e4da..39a1d4cda6 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java @@ -10,7 +10,6 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.StreamingOutput; -import com.bakdata.conquery.io.result.ResultUtil; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.auth.permissions.Ability; import com.bakdata.conquery.models.config.ConqueryConfig; @@ -18,8 +17,8 @@ import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.forms.managed.ManagedForm; import com.bakdata.conquery.models.i18n.I18n; -import com.bakdata.conquery.models.identifiable.mapping.IdMappingConfig; -import com.bakdata.conquery.models.identifiable.mapping.IdMappingState; + +import com.bakdata.conquery.models.identifiable.mapping.IdPrinter; import com.bakdata.conquery.models.query.ManagedQuery; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.SingleTableResult; @@ -66,13 +65,15 @@ public static & SingleTableResult> Response getAr } // Get the locale extracted by the LocaleFilter - IdMappingState mappingState = IdMappingConfig.initToExternal(user, exec); + + + IdPrinter idPrinter = config.getFrontend().getQueryUpload().getIdPrinter(user,exec,namespace); PrintSettings settings = new PrintSettings( pretty, I18n.LOCALE.get(), datasetRegistry, config, - (EntityResult cer) -> ResultUtil.createId(namespace, cer, mappingState) + idPrinter::createId ); diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java index 3f6ac4b16b..b6d2b055fa 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java @@ -12,15 +12,14 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.StreamingOutput; -import com.bakdata.conquery.io.result.ResultUtil; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.auth.permissions.Ability; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.i18n.I18n; -import com.bakdata.conquery.models.identifiable.mapping.IdMappingConfig; -import com.bakdata.conquery.models.identifiable.mapping.IdMappingState; + +import com.bakdata.conquery.models.identifiable.mapping.IdPrinter; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.SingleTableResult; import com.bakdata.conquery.models.worker.DatasetRegistry; @@ -50,7 +49,8 @@ public & SingleTableResult> Response getResult(Us // Check if user is permitted to download on all datasets that were referenced by the query authorizeDownloadDatasets(user, exec); - IdMappingState mappingState = IdMappingConfig.initToExternal(user, exec); + IdPrinter idPrinter = config.getFrontend().getQueryUpload().getIdPrinter(user,exec,namespace); + // Get the locale extracted by the LocaleFilter PrintSettings settings = new PrintSettings( @@ -58,7 +58,7 @@ public & SingleTableResult> Response getResult(Us I18n.LOCALE.get(), datasetRegistry, config, - cer -> ResultUtil.createId(namespace, cer, mappingState) + idPrinter::createId ); Charset charset = determineCharset(userAgent, queryCharset); diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java index 86e08ea07d..76cb20600f 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java @@ -1,6 +1,5 @@ package com.bakdata.conquery.io.result.excel; -import com.bakdata.conquery.io.result.ResultUtil; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.auth.permissions.Ability; import com.bakdata.conquery.models.config.ConqueryConfig; @@ -8,8 +7,8 @@ import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.i18n.I18n; import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; -import com.bakdata.conquery.models.identifiable.mapping.IdMappingConfig; -import com.bakdata.conquery.models.identifiable.mapping.IdMappingState; + +import com.bakdata.conquery.models.identifiable.mapping.IdPrinter; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.SingleTableResult; import com.bakdata.conquery.models.query.results.EntityResult; @@ -39,13 +38,15 @@ public & SingleTableResult> Response getExcelResu user.authorize(dataset, Ability.DOWNLOAD); user.authorize(exec, Ability.READ); - IdMappingState mappingState = IdMappingConfig.initToExternal(user, exec); + IdPrinter idPrinter = config.getFrontend().getQueryUpload().getIdPrinter(user,exec,namespace); + PrintSettings settings = new PrintSettings( pretty, I18n.LOCALE.get(), datasetRegistry, config, - (EntityResult cer) -> ResultUtil.createId(namespace, cer, mappingState)); + idPrinter::createId + ); ExcelRenderer excelRenderer = new ExcelRenderer(config.getExcel()); diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java index 52b3abcdce..7f9658aea5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java @@ -5,13 +5,9 @@ import javax.validation.constraints.NotEmpty; -import com.bakdata.conquery.apiv1.query.concept.specific.external.FormatColumn; -import com.bakdata.conquery.io.cps.CPSTypeIdResolver; import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; -import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.base.Strings; import io.dropwizard.logback.shaded.checkerframework.checker.nullness.qual.Nullable; -import io.dropwizard.validation.ValidationMethod; import lombok.Builder; import lombok.Data; import org.apache.commons.lang3.StringUtils; @@ -33,6 +29,7 @@ public static class Mapping { @Builder.Default private final boolean resolvable = true; + @Builder.Default private final boolean fillAnon = false; } @@ -75,9 +72,4 @@ public EntityIdMap.ExternalId read(String value) { @Nullable private final Mapping mapping; - @JsonIgnore - @ValidationMethod - public boolean isExistingColumnFormat() { - return CPSTypeIdResolver.getImplementation(FormatColumn.class, name) != null; - } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java index 54e931acc4..0d62738b94 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java @@ -19,7 +19,7 @@ import com.bakdata.conquery.models.auth.develop.DevAuthConfig; import com.bakdata.conquery.models.config.auth.DevelopmentAuthorizationConfig; import com.bakdata.conquery.models.common.CDateSet; -import com.bakdata.conquery.models.identifiable.mapping.IdMappingConfig; + import com.bakdata.conquery.util.DateReader; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.module.SimpleModule; diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java index 134edcde2e..d1c873d0de 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java @@ -6,12 +6,20 @@ import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; +import java.util.stream.IntStream; import javax.validation.Valid; import javax.validation.constraints.NotEmpty; import javax.validation.constraints.NotNull; import com.bakdata.conquery.apiv1.query.concept.specific.external.DateFormat; +import com.bakdata.conquery.models.auth.entities.User; +import com.bakdata.conquery.models.auth.permissions.Ability; +import com.bakdata.conquery.models.execution.ManagedExecution; +import com.bakdata.conquery.models.identifiable.mapping.AutoIncrementingPseudomizer; +import com.bakdata.conquery.models.identifiable.mapping.FullIdPrinter; +import com.bakdata.conquery.models.identifiable.mapping.IdPrinter; +import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.util.VersionInfo; import com.fasterxml.jackson.annotation.JsonIgnore; import groovy.transform.ToString; @@ -54,6 +62,7 @@ public static class UploadConfig { .mapping(ColumnConfig.Mapping.builder() .field("id") .resolvable(true) + .fillAnon(true) .build()) .build() ); @@ -79,9 +88,9 @@ public List getPrintIdFields() { public ColumnConfig getIdMapper(String name) { return ids.stream() - .filter(mapper -> mapper.getName().equals(name)) //TODO use map - .findFirst() - .orElse(null); + .filter(mapper -> mapper.getName().equals(name)) //TODO use map + .findFirst() + .orElse(null); } public int getIdIndex(List format) { @@ -162,6 +171,17 @@ public boolean isIdColsHaveMapping() { return ids.stream().map(ColumnConfig::getMapping).allMatch(Objects::nonNull); } + @ValidationMethod(message = "Must have exactly one Column for Pseudomization.") + @JsonIgnore + public boolean isExactlyOnePseudo() { + return ids.stream() + .map(ColumnConfig::getMapping) + .filter(Objects::nonNull) + .filter(ColumnConfig.Mapping::isFillAnon) + .count() == 1; + } + + public DateFormat resolveDateFormat(String handle) { try { return DateFormat.valueOf(handle); @@ -172,6 +192,21 @@ public DateFormat resolveDateFormat(String handle) { } + public IdPrinter getIdPrinter(User owner, ManagedExecution execution, Namespace namespace) { + + if (owner.isPermitted(execution.getDataset(), Ability.PRESERVE_ID)) { + return new FullIdPrinter(namespace.getStorage().getPrimaryDictionary(), namespace.getStorage().getIdMapping()); + } + + final int size = getPrintIdFields().size(); + final int pos = IntStream.range(0, getIds().size()) + .filter(idx -> getIds().get(idx).getMapping() != null) + .filter(idx -> getIds().get(idx).getMapping().isFillAnon()) + .findFirst() + .orElseThrow(); + + return new AutoIncrementingPseudomizer(size,pos); + } } @Data diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/AutoIncrementingPseudomizer.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/AutoIncrementingPseudomizer.java index 71c0981299..efa7cc1e8e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/AutoIncrementingPseudomizer.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/AutoIncrementingPseudomizer.java @@ -1,10 +1,10 @@ package com.bakdata.conquery.models.identifiable.mapping; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; -import com.bakdata.conquery.models.identifiable.mapping.EntityPrintId; +import com.bakdata.conquery.models.query.results.EntityResult; +import it.unimi.dsi.fastutil.ints.Int2ObjectAVLTreeMap; +import it.unimi.dsi.fastutil.ints.Int2ObjectMap; import lombok.RequiredArgsConstructor; /** @@ -12,27 +12,29 @@ * not have the rights to see the real data. */ @RequiredArgsConstructor -public class AutoIncrementingPseudomizer { +public class AutoIncrementingPseudomizer implements IdPrinter { private static final String ANONYMOUS_ID_PREFIX = "anon_"; // Abbreviation for anonymous - private final Map cachedIds = new ConcurrentHashMap<>(); + private final Int2ObjectMap cachedIds = new Int2ObjectAVLTreeMap<>(); private final AtomicInteger pseudoIdPointer = new AtomicInteger(0); private final int size; private final int position; + @Override + public EntityPrintId createId(EntityResult entityResult) { + return getPseudoId(entityResult.getEntityId()); + } + /** * In the pseudo format the actual id columns are preserved but empty. * Only the Pid Column is written with a new generated id. */ - public EntityPrintId getPseudoId(String csvEntityId) { - - EntityPrintId pseudonym = cachedIds.computeIfAbsent(csvEntityId, this::createPseudonym); - - return pseudonym; + public EntityPrintId getPseudoId(int csvEntityId) { + return cachedIds.computeIfAbsent(csvEntityId, this::createPseudonym); } - private EntityPrintId createPseudonym(String ignored) { + private EntityPrintId createPseudonym(int ignored) { final String name = ANONYMOUS_ID_PREFIX + pseudoIdPointer.getAndIncrement(); final String[] parts = new String[size]; diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/FullIdPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/FullIdPrinter.java new file mode 100644 index 0000000000..db1c3ba8b5 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/FullIdPrinter.java @@ -0,0 +1,35 @@ +package com.bakdata.conquery.models.identifiable.mapping; + +import com.bakdata.conquery.models.dictionary.EncodedDictionary; +import com.bakdata.conquery.models.query.results.EntityResult; +import lombok.RequiredArgsConstructor; + +/** + * Maker interface for implementation specific state object during query result to csv rendering. + */ +@RequiredArgsConstructor +public class FullIdPrinter implements IdPrinter { + + private final EncodedDictionary dictionary; + private final EntityIdMap idMapping; + + @Override + public EntityPrintId createId(EntityResult entityResult){ + + String csvEntityId = dictionary.getElement(entityResult.getEntityId()); + // The state may be uses by implementations of this class + + if (idMapping == null) { + return EntityPrintId.from(csvEntityId); + } + + EntityPrintId externalEntityId = idMapping.toExternal(csvEntityId); + + if (externalEntityId == null) { + return EntityPrintId.from(csvEntityId); + } + + return externalEntityId; + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingConfig.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingConfig.java deleted file mode 100644 index 6ed66d726c..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingConfig.java +++ /dev/null @@ -1,55 +0,0 @@ -package com.bakdata.conquery.models.identifiable.mapping; - -import java.util.ArrayList; -import java.util.List; - -import com.bakdata.conquery.models.auth.entities.User; -import com.bakdata.conquery.models.config.ColumnConfig; -import com.bakdata.conquery.models.execution.ManagedExecution; -import com.bakdata.conquery.models.worker.Namespace; -import com.univocity.parsers.common.record.Record; -import com.univocity.parsers.csv.CsvParser; -import lombok.AccessLevel; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; -import lombok.extern.slf4j.Slf4j; - - -@Slf4j -@NoArgsConstructor(access = AccessLevel.NONE) -@Setter -@Getter -public class IdMappingConfig { - - - /** - * Is called once before a mapping is used before a query result is created to - * allow the mapping to have state information. - */ - public static IdMappingState initToExternal(User user, ManagedExecution execution) { - // This mapping does not need a per-query state, so we return an immutable empty map. - return null; - } - - /** - * Converts the internal ID to the an external. - */ - public static EntityPrintId toExternal(String csvEntityId, Namespace namespace, IdMappingState state, EntityIdMap mapping) { - // The state may be uses by implementations of this class - - if (mapping == null) { - return EntityPrintId.from(csvEntityId); - } - - EntityPrintId externalEntityId = mapping.toExternal(csvEntityId); - - if (externalEntityId == null) { - return EntityPrintId.from(csvEntityId); - } - - return externalEntityId; - } - - -} diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingState.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingState.java deleted file mode 100644 index 442db2db1a..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdMappingState.java +++ /dev/null @@ -1,8 +0,0 @@ -package com.bakdata.conquery.models.identifiable.mapping; - -/** - * Maker interface for implementation specific state object during query result to csv rendering. - */ -public interface IdMappingState { - -} diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdPrinter.java new file mode 100644 index 0000000000..25a746694e --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdPrinter.java @@ -0,0 +1,7 @@ +package com.bakdata.conquery.models.identifiable.mapping; + +import com.bakdata.conquery.models.query.results.EntityResult; + +public interface IdPrinter { + EntityPrintId createId(EntityResult entityResult); +} diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java b/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java index f98a1bea0b..118536ad7e 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java @@ -1,7 +1,6 @@ package com.bakdata.conquery.integration.json; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.fail; import java.io.IOException; import java.util.List; @@ -12,13 +11,11 @@ import com.bakdata.conquery.integration.common.IntegrationUtils; import com.bakdata.conquery.integration.common.ResourceFile; import com.bakdata.conquery.io.result.CsvLineStreamRenderer; -import com.bakdata.conquery.io.result.ResultUtil; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.execution.ExecutionState; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; -import com.bakdata.conquery.models.identifiable.mapping.IdMappingConfig; -import com.bakdata.conquery.models.identifiable.mapping.IdMappingState; +import com.bakdata.conquery.models.identifiable.mapping.IdPrinter; import com.bakdata.conquery.models.query.ManagedQuery; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; @@ -64,12 +61,13 @@ public void executeTest(StandaloneSupport standaloneSupport) throws IOException assertThat( execution.streamResults() - .flatMap(EntityResult::streamValues) + .flatMap(EntityResult::streamValues) ) .as("Should have same size as result infos") .allSatisfy(v -> assertThat(v).hasSameSizeAs(resultInfos)); - IdMappingState mappingState = IdMappingConfig.initToExternal(testUser, execution); + IdPrinter idPrinter = config.getFrontend().getQueryUpload().getIdPrinter(testUser, execution, execution.getNamespace()); + PrintSettings PRINT_SETTINGS = new PrintSettings( @@ -77,7 +75,7 @@ public void executeTest(StandaloneSupport standaloneSupport) throws IOException Locale.ENGLISH, namespaces, config, - cer -> ResultUtil.createId(standaloneSupport.getNamespace(), cer,mappingState), + idPrinter::createId, (columnInfo) -> columnInfo.getSelect().getId().toStringWithoutDataset() ); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java b/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java index bb86e4efd6..d35cc0f34b 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java @@ -21,7 +21,6 @@ import com.bakdata.conquery.integration.common.ResourceFile; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.result.CsvLineStreamRenderer; -import com.bakdata.conquery.io.result.ResultUtil; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.datasets.concepts.Concept; import com.bakdata.conquery.models.config.ConqueryConfig; @@ -30,8 +29,8 @@ import com.bakdata.conquery.models.execution.ExecutionState; import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.forms.managed.ManagedForm; -import com.bakdata.conquery.models.identifiable.mapping.IdMappingConfig; -import com.bakdata.conquery.models.identifiable.mapping.IdMappingState; + +import com.bakdata.conquery.models.identifiable.mapping.IdPrinter; import com.bakdata.conquery.models.query.ManagedQuery; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; @@ -74,9 +73,6 @@ public class FormTest extends ConqueryTestSpec { @JsonIgnore private Form form; - @JsonIgnore - private IdMappingConfig idMappingConfig; - @Override public void overrideConfig(ConqueryConfig config) { config.setStorage(new NonPersistentStoreFactory()); @@ -136,7 +132,9 @@ public void executeTest(StandaloneSupport support) throws Exception { private void checkResults(StandaloneSupport standaloneSupport, ManagedForm managedForm, User user) throws IOException { Map> managedMapping = managedForm.getSubQueries(); - IdMappingState mappingState = IdMappingConfig.initToExternal(user, managedForm); + + IdPrinter idPrinter = standaloneSupport.getConfig().getFrontend().getQueryUpload().getIdPrinter(user, managedForm, standaloneSupport.getNamespace()); + final ConqueryConfig config = standaloneSupport.getConfig(); PrintSettings PRINT_SETTINGS = @@ -145,7 +143,7 @@ private void checkResults(StandaloneSupport standaloneSupport, ManagedForm manag Locale.ENGLISH, standaloneSupport.getDatasetsProcessor().getDatasetRegistry(), config, - cer -> ResultUtil.createId(standaloneSupport.getNamespace(), cer, mappingState) + idPrinter::createId ); CsvLineStreamRenderer renderer = new CsvLineStreamRenderer(config.getCsv().createWriter(), PRINT_SETTINGS); diff --git a/backend/src/test/java/com/bakdata/conquery/models/identifiable/mapping/PseudomizationTest.java b/backend/src/test/java/com/bakdata/conquery/models/identifiable/mapping/PseudomizationTest.java index 354172228b..470024c659 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/identifiable/mapping/PseudomizationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/identifiable/mapping/PseudomizationTest.java @@ -9,19 +9,15 @@ public class PseudomizationTest { @Test void pseudoIdGeneration() { AutoIncrementingPseudomizer pseudomizer = new AutoIncrementingPseudomizer(4,2); - - String csvId1 = "123"; - String csvId1Copy = "123"; - String csvId2 = "234"; - + // Id changes from internal to external - assertThat(pseudomizer.getPseudoId(csvId1)).isEqualTo(EntityPrintId.from(null, null, "anon_0", null)); + assertThat(pseudomizer.getPseudoId(0)).isEqualTo(EntityPrintId.from(null, null, "anon_0", null)); // Id mapping is constant - assertThat(pseudomizer.getPseudoId(csvId1)).isEqualTo(pseudomizer.getPseudoId(csvId1Copy)); + assertThat(pseudomizer.getPseudoId(0)).isEqualTo(pseudomizer.getPseudoId(0)); // Mapping produces differing external ids - assertThat(pseudomizer.getPseudoId(csvId1)).isNotEqualTo(pseudomizer.getPseudoId(csvId2)); + assertThat(pseudomizer.getPseudoId(1)).isNotEqualTo(pseudomizer.getPseudoId(0)); } From 7a4ae6f97b8f86c9e39e909d99d96b206059d461 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Fri, 23 Jul 2021 15:32:20 +0200 Subject: [PATCH 17/82] remove unused class --- autodoc/src/main/java/com/bakdata/conquery/Constants.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/autodoc/src/main/java/com/bakdata/conquery/Constants.java b/autodoc/src/main/java/com/bakdata/conquery/Constants.java index d5267a1368..e37a36016b 100644 --- a/autodoc/src/main/java/com/bakdata/conquery/Constants.java +++ b/autodoc/src/main/java/com/bakdata/conquery/Constants.java @@ -62,7 +62,6 @@ import com.bakdata.conquery.models.forms.configs.FormConfig; import com.bakdata.conquery.models.forms.configs.FormConfig.FormConfigFullRepresentation; import com.bakdata.conquery.models.forms.configs.FormConfig.FormConfigOverviewRepresentation; -import com.bakdata.conquery.models.identifiable.mapping.IdMappingConfig; import com.bakdata.conquery.models.preproc.TableImportDescriptor; import com.bakdata.conquery.models.preproc.TableInputDescriptor; import com.bakdata.conquery.models.preproc.outputs.OutputDescription; @@ -118,7 +117,6 @@ public class Constants { .base(new Base(AuthorizationConfig.class, "An `AuthorizationConfig` defines the initial users that are created on application startup and other permission related options.")) .base(new Base(AuthenticationConfig.class, "An `AuthenticationConfig` is used to define how specific realms for authentication are configured.")) .base(new Base(PluginConfig.class, "A `PluginConfig` is used to define settings for Conquery plugins.")) - .base(new Base(IdMappingConfig.class, "An `IdMappingConfig` is used to define how multi column entity IDs are printed and parsed")) .otherClass(APIConfig.class) .otherClass(ConqueryConfig.class) .otherClass(ClusterConfig.class) From f2963f3a595d44304374f21397d099edebaf82fd Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 23 Jul 2021 13:34:17 +0000 Subject: [PATCH 18/82] Update AutoDoc --- docs/Config JSON.md | 63 ++++++++++-------------------------------- docs/REST API JSONs.md | 20 +++++++------- 2 files changed, 25 insertions(+), 58 deletions(-) diff --git a/docs/Config JSON.md b/docs/Config JSON.md index b82a65e60c..97d541c2ee 100644 --- a/docs/Config JSON.md +++ b/docs/Config JSON.md @@ -133,38 +133,6 @@ Different types of PluginConfig can be used by setting `type` to one of the foll ---- - -## Base IdMappingConfig -An `IdMappingConfig` is used to define how multi column entity IDs are printed and parsed - -Different types of IdMappingConfig can be used by setting `type` to one of the following values: - - -### NO_ID_MAPPING [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/NoIdMapping.java#L11-L13) -Disables Id-Mapping completely. - -

Details

- -Java Type: `com.bakdata.conquery.models.config.NoIdMapping` - -No fields can be set for this type. - -

- -### SIMPLE [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/SimpleIdMapping.java#L14-L16) -Maps input to itself for upload. - -
Details

- -Java Type: `com.bakdata.conquery.models.config.SimpleIdMapping` - -No fields can be set for this type. - -

- - - --- ## Other Types @@ -237,18 +205,17 @@ Supported Fields: | --- | --- | --- | --- | --- | --- | | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L74) | api | [APIConfig](#Type-APIConfig) | | | | | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L50) | arrow | `@Valid @NotNull ArrowConfig` | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L86) | authentication | list of [AuthenticationConfig](#Base-AuthenticationConfig) | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L90) | authorization | [@Valid @NotNull AuthorizationConfig](#Base-AuthorizationConfig) | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L83) | authentication | list of [AuthenticationConfig](#Base-AuthenticationConfig) | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L87) | authorization | [@Valid @NotNull AuthorizationConfig](#Base-AuthorizationConfig) | | | | | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L41) | cluster | [ClusterConfig](#Type-ClusterConfig) | | | | | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L47) | csv | [CSVConfig](#Type-CSVConfig) | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L99-L101) | debugMode | `boolean` or `null` | `null` | | null means here that we try to deduce from an attached agent | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L93) | excel | `@Valid @NotNull ExcelConfig` | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L104) | failOnError | `boolean` | `false` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L96-L98) | debugMode | `boolean` or `null` | `null` | | null means here that we try to deduce from an attached agent | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L90) | excel | `@Valid @NotNull ExcelConfig` | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L101) | failOnError | `boolean` | `false` | | | | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L77) | frontend | [FrontendConfig](#Type-FrontendConfig) | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L83) | idMapping | [@NotNull @Valid IdMappingConfig](#Base-IdMappingConfig) | | | | | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L63) | locale | [LocaleConfig](#Type-LocaleConfig) | | | | | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L81) | metricsConfig | `ConqueryMetricsConfig` | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L97) | plugins | list of `PluginConfig` | `[]` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L94) | plugins | list of `PluginConfig` | `[]` | | | | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L44) | preprocessor | [PreprocessingConfig](#Type-PreprocessingConfig) | | | | | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L71) | queries | [QueryConfig](#Type-QueryConfig) | | | | | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L53-L55) | resultProviders | list of `ResultRendererProvider` | | | The order of this lists determines the ordner of the generated result urls in a query status. | @@ -256,7 +223,7 @@ Supported Fields: | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L68) | storage | `@Valid @NotNull StoreFactory` | | | |

-### Type CurrencyConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L92) +### Type CurrencyConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L212)
Details

@@ -267,13 +234,13 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L97) | decimalScale | `int` | `2` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L96) | decimalSeparator | `String` | `","` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L94) | prefix | `String` | `"€"` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L95) | thousandSeparator | `String` | `"."` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L217) | decimalScale | `int` | `2` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L216) | decimalSeparator | `String` | `","` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L214) | prefix | `String` | `"€"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L215) | thousandSeparator | `String` | `"."` | | |

-### Type FrontendConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L27) +### Type FrontendConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L35)
Details

@@ -284,9 +251,9 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L33) | currency | [CurrencyConfig](#Type-CurrencyConfig) | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L37) | queryUpload | `UploadConfig` | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L32) | version | `String` | `"0.0.0-SNAPSHOT"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L45) | currency | [CurrencyConfig](#Type-CurrencyConfig) | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L49) | queryUpload | `UploadConfig` | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L44) | version | `String` | `"0.0.0-SNAPSHOT"` | | |

### Type LocaleConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L11) diff --git a/docs/REST API JSONs.md b/docs/REST API JSONs.md index 536882392a..34eecedd76 100644 --- a/docs/REST API JSONs.md +++ b/docs/REST API JSONs.md @@ -509,7 +509,7 @@ Supported Fields: | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQElement.java#L31-L33) | label | `String` | ? | | Allows the user to define labels. |

-### EXTERNAL [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java#L32-L34) +### EXTERNAL [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java#L38-L40) Allows uploading lists of entities.
Details

@@ -755,7 +755,7 @@ Supported Fields: | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptResource.java#L68) | concepts | list of `String` | `null` | | |

-### Type CurrencyConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L92) +### Type CurrencyConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L212)
Details

@@ -766,10 +766,10 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L97) | decimalScale | `int` | `2` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L96) | decimalSeparator | `String` | `","` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L94) | prefix | `String` | `"€"` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L95) | thousandSeparator | `String` | `"."` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L217) | decimalScale | `int` | `2` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L216) | decimalSeparator | `String` | `","` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L214) | prefix | `String` | `"€"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L215) | thousandSeparator | `String` | `"."` | | |

### Type ExecutionStatus [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/ExecutionStatus.java#L18) @@ -885,7 +885,7 @@ No fields can be set for this type.

-### Type FrontendConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L27) +### Type FrontendConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L35)
Details

@@ -896,9 +896,9 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L33) | currency | [CurrencyConfig](#Type-CurrencyConfig) | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L37) | queryUpload | `UploadConfig` | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L32) | version | `String` | `"0.0.0-SNAPSHOT"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L45) | currency | [CurrencyConfig](#Type-CurrencyConfig) | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L49) | queryUpload | `UploadConfig` | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L44) | version | `String` | `"0.0.0-SNAPSHOT"` | | |

### Type FullExecutionStatus [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/FullExecutionStatus.java#L20-L24) From 4d7c6652916146a9a5d57db00c08f78668155ce3 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Mon, 26 Jul 2021 09:37:54 +0200 Subject: [PATCH 19/82] adds json defaults --- .../com/bakdata/conquery/models/config/ColumnConfig.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java index 7f9658aea5..d1d04b79a9 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java @@ -6,6 +6,7 @@ import javax.validation.constraints.NotEmpty; import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; +import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Strings; import io.dropwizard.logback.shaded.checkerframework.checker.nullness.qual.Nullable; import lombok.Builder; @@ -23,13 +24,17 @@ public static class Mapping { @Builder.Default private final String pad = null; + @Builder.Default + @JsonProperty(defaultValue = "-1") private final int length = -1; @Builder.Default + @JsonProperty(defaultValue = "true") private final boolean resolvable = true; @Builder.Default + @JsonProperty(defaultValue = "false") private final boolean fillAnon = false; } From ac3cf5a5330ac4e82ec474eed9fb89badbc8a6d5 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Mon, 26 Jul 2021 09:41:19 +0200 Subject: [PATCH 20/82] use noArgs+Setter instead --- .../conquery/models/config/ColumnConfig.java | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java index d1d04b79a9..a972be5a62 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java @@ -9,33 +9,36 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Strings; import io.dropwizard.logback.shaded.checkerframework.checker.nullness.qual.Nullable; +import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; import org.apache.commons.lang3.StringUtils; @Data @Builder public class ColumnConfig { - @Data @Builder + @NoArgsConstructor + @Getter + @Setter public static class Mapping { - private final String field; + private String field; @Builder.Default - private final String pad = null; + private String pad = null; @Builder.Default - @JsonProperty(defaultValue = "-1") - private final int length = -1; + private int length = -1; @Builder.Default - @JsonProperty(defaultValue = "true") - private final boolean resolvable = true; + private boolean resolvable = true; @Builder.Default - @JsonProperty(defaultValue = "false") - private final boolean fillAnon = false; + private boolean fillAnon = false; } From c7e7bfc6cadc57146a019b410213beba93ca4bfa Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Mon, 26 Jul 2021 09:43:57 +0200 Subject: [PATCH 21/82] adds missing Allargs for builder --- .../java/com/bakdata/conquery/models/config/ColumnConfig.java | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java index a972be5a62..91f774b001 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java @@ -22,6 +22,7 @@ public class ColumnConfig { @Builder + @AllArgsConstructor @NoArgsConstructor @Getter @Setter From eb2c7434cc05b599b395f968754c414bbbe2058d Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Mon, 26 Jul 2021 12:05:03 +0200 Subject: [PATCH 22/82] fix missing negation for resolving --- .../conquery/models/config/ColumnConfig.java | 58 +++++++------------ .../models/config/FrontendConfig.java | 23 +++----- .../identifiable/mapping/EntityIdMap.java | 4 +- 3 files changed, 32 insertions(+), 53 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java index 91f774b001..2ba551ccd2 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java @@ -6,57 +6,32 @@ import javax.validation.constraints.NotEmpty; import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; -import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Strings; -import io.dropwizard.logback.shaded.checkerframework.checker.nullness.qual.Nullable; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; import org.apache.commons.lang3.StringUtils; @Data @Builder +@AllArgsConstructor public class ColumnConfig { - @Builder - @AllArgsConstructor - @NoArgsConstructor - @Getter - @Setter - public static class Mapping { - private String field; - - @Builder.Default - private String pad = null; - - @Builder.Default - private int length = -1; - - @Builder.Default - private boolean resolvable = true; - - @Builder.Default - private boolean fillAnon = false; - - } public EntityIdMap.ExternalId read(String value) { - if(!mapping.isResolvable()){ - return null; //TODO throw Exception? + if (!isResolvable()) { + return null; } if (Strings.isNullOrEmpty(value)) { return null; } - if (getMapping().getLength() == -1) { + if (getLength() == -1) { return new EntityIdMap.ExternalId(new String[]{getName(), value}); } - String padded = StringUtils.leftPad(value, getMapping().getLength(), getMapping().getPad()); + String padded = StringUtils.leftPad(value, getLength(), getPad()); return new EntityIdMap.ExternalId(new String[]{getName(), padded}); @@ -64,21 +39,30 @@ public EntityIdMap.ExternalId read(String value) { @NotEmpty - private final String name; + private String name; /** * Map of Localized labels. */ - @Builder.Default - private final Map label = Collections.emptyMap(); + + private Map label = Collections.emptyMap(); /** * Map of Localized description. */ - @Builder.Default - private final Map description = Collections.emptyMap(); + private Map description = Collections.emptyMap(); + + + private String field; + + private String pad = null; + + + private int length = -1; + + private boolean resolvable = false; + - @Nullable - private final Mapping mapping; + private boolean fillAnon = false; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java index d1c873d0de..366d050e39 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java @@ -59,11 +59,9 @@ public static class UploadConfig { private List ids = List.of( ColumnConfig.builder() .name("ID") - .mapping(ColumnConfig.Mapping.builder() - .field("id") - .resolvable(true) - .fillAnon(true) - .build()) + .field("id") + .resolvable(true) + .fillAnon(true) .build() ); @@ -77,9 +75,8 @@ public static class UploadConfig { public List getPrintIdFields() { if (idFieldsCached == null) { idFieldsCached = ids.stream() - .map(ColumnConfig::getMapping) + .map(ColumnConfig::getField) .filter(Objects::nonNull) - .map(ColumnConfig.Mapping::getField) .collect(Collectors.toList()); } @@ -168,16 +165,15 @@ public boolean isAllColsUnique() { @ValidationMethod(message = "Not all Id-Columns have mappings.") @JsonIgnore public boolean isIdColsHaveMapping() { - return ids.stream().map(ColumnConfig::getMapping).allMatch(Objects::nonNull); + return ids.stream().map(ColumnConfig::getField).allMatch(Objects::nonNull); } @ValidationMethod(message = "Must have exactly one Column for Pseudomization.") @JsonIgnore public boolean isExactlyOnePseudo() { return ids.stream() - .map(ColumnConfig::getMapping) - .filter(Objects::nonNull) - .filter(ColumnConfig.Mapping::isFillAnon) + .filter(conf -> conf.getField() != null) + .filter(ColumnConfig::isFillAnon) .count() == 1; } @@ -200,12 +196,11 @@ public IdPrinter getIdPrinter(User owner, ManagedExecution execution, Namespa final int size = getPrintIdFields().size(); final int pos = IntStream.range(0, getIds().size()) - .filter(idx -> getIds().get(idx).getMapping() != null) - .filter(idx -> getIds().get(idx).getMapping().isFillAnon()) + .filter(idx -> getIds().get(idx).isFillAnon()) .findFirst() .orElseThrow(); - return new AutoIncrementingPseudomizer(size,pos); + return new AutoIncrementingPseudomizer(size, pos); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java index 5cbbeee4df..77bcd6363d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java @@ -63,7 +63,7 @@ public static EntityIdMap generateIdMapping(CsvParser parser, List for (ColumnConfig columnConfig : mappers) { - final String otherId = record.getString(columnConfig.getMapping().getField()); + final String otherId = record.getString(columnConfig.getField()); idParts.add(otherId); @@ -71,7 +71,7 @@ public static EntityIdMap generateIdMapping(CsvParser parser, List continue; } - if (!columnConfig.getMapping().isResolvable()) { + if (!columnConfig.isResolvable()) { continue; } From 05e1a02aa5b73dea15677bf0a58d845ed9b7ab0b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 26 Jul 2021 10:07:07 +0000 Subject: [PATCH 23/82] Update AutoDoc --- docs/Config JSON.md | 10 +++++----- docs/REST API JSONs.md | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/Config JSON.md b/docs/Config JSON.md index 97d541c2ee..c9248a656f 100644 --- a/docs/Config JSON.md +++ b/docs/Config JSON.md @@ -223,7 +223,7 @@ Supported Fields: | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L68) | storage | `@Valid @NotNull StoreFactory` | | | |

-### Type CurrencyConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L212) +### Type CurrencyConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L207)
Details

@@ -234,10 +234,10 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L217) | decimalScale | `int` | `2` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L216) | decimalSeparator | `String` | `","` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L214) | prefix | `String` | `"€"` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L215) | thousandSeparator | `String` | `"."` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L212) | decimalScale | `int` | `2` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L211) | decimalSeparator | `String` | `","` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L209) | prefix | `String` | `"€"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L210) | thousandSeparator | `String` | `"."` | | |

### Type FrontendConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L35) diff --git a/docs/REST API JSONs.md b/docs/REST API JSONs.md index 34eecedd76..71e7db192b 100644 --- a/docs/REST API JSONs.md +++ b/docs/REST API JSONs.md @@ -755,7 +755,7 @@ Supported Fields: | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptResource.java#L68) | concepts | list of `String` | `null` | | |

-### Type CurrencyConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L212) +### Type CurrencyConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L207)
Details

@@ -766,10 +766,10 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L217) | decimalScale | `int` | `2` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L216) | decimalSeparator | `String` | `","` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L214) | prefix | `String` | `"€"` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L215) | thousandSeparator | `String` | `"."` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L212) | decimalScale | `int` | `2` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L211) | decimalSeparator | `String` | `","` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L209) | prefix | `String` | `"€"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L210) | thousandSeparator | `String` | `"."` | | |

### Type ExecutionStatus [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/ExecutionStatus.java#L18) From b1fb70a39bfa792f1d142baab37c4563f0d4e026 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Mon, 26 Jul 2021 12:08:46 +0200 Subject: [PATCH 24/82] allow optional fields --- .../com/bakdata/conquery/models/config/ColumnConfig.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java index 2ba551ccd2..6ffc37ccf1 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java @@ -10,11 +10,17 @@ import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; import org.apache.commons.lang3.StringUtils; -@Data @Builder @AllArgsConstructor +@ToString +@NoArgsConstructor @Setter +@Getter public class ColumnConfig { From 02a188845a4c2ba62468858746a0199197a8aba5 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Mon, 26 Jul 2021 12:19:01 +0200 Subject: [PATCH 25/82] add internalOnly anno --- .../bakdata/conquery/models/config/ColumnConfig.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java index 6ffc37ccf1..fc15fe7d5f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java @@ -5,11 +5,11 @@ import javax.validation.constraints.NotEmpty; +import com.bakdata.conquery.io.jackson.InternalOnly; import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; import com.google.common.base.Strings; import lombok.AllArgsConstructor; import lombok.Builder; -import lombok.Data; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; @@ -19,7 +19,8 @@ @Builder @AllArgsConstructor @ToString -@NoArgsConstructor @Setter +@NoArgsConstructor +@Setter @Getter public class ColumnConfig { @@ -58,17 +59,18 @@ public EntityIdMap.ExternalId read(String value) { */ private Map description = Collections.emptyMap(); - + @InternalOnly private String field; + @InternalOnly private String pad = null; - + @InternalOnly private int length = -1; private boolean resolvable = false; - + @InternalOnly private boolean fillAnon = false; } From cb2c03917c2503ff41adfa74e5137241ba0f2388 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Mon, 26 Jul 2021 16:33:41 +0200 Subject: [PATCH 26/82] also adds endpoint for uploading external result lists --- .../conquery/apiv1/QueryProcessor.java | 38 ++++++++++ .../concept/specific/external/CQExternal.java | 73 +++++++++++++------ .../conquery/resources/api/QueryResource.java | 19 +++++ .../tests/endpoints/apiEndpointInfo.json | 5 ++ 4 files changed, 111 insertions(+), 24 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java index bda960ed8d..db81e07e5e 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java @@ -13,6 +13,7 @@ import java.util.stream.Stream; import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.core.Response; import javax.ws.rs.core.UriBuilder; import com.bakdata.conquery.apiv1.query.CQElement; @@ -44,10 +45,12 @@ import com.bakdata.conquery.models.query.visitor.QueryVisitor; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.Namespace; +import com.bakdata.conquery.resources.api.QueryResource; import com.bakdata.conquery.util.QueryUtils; import com.bakdata.conquery.util.QueryUtils.NamespacedIdentifiableCollector; import com.google.common.collect.ClassToInstanceMap; import com.google.common.collect.MutableClassToInstanceMap; +import lombok.Data; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -359,5 +362,40 @@ public FullExecutionStatus getQueryFullStatus(ManagedExecution query, User us return status; } + @Data + public static class UploadResponse { + private final ManagedExecutionId execution; + private final int resolved; + + private final List unresolvedId; + private final List unreadableDate; + } + + public Response uploadEntities(User user, Dataset dataset, QueryResource.ExternalUpload upload) { + final CQExternal.ResolveStatistic statistic = CQExternal.resolveEntities(upload.getValues(), upload.getFormat(), + datasetRegistry.get(dataset.getId()).getStorage().getIdMapping(), + config.getFrontend().getQueryUpload(), + config.getPreprocessor().getParsers().getDateReader() + ); + + // Resolving nothing is a problem and we fail. + if (statistic.getResolved().isEmpty()) { + return Response.status(Response.Status.BAD_REQUEST) + .entity(new UploadResponse(null, 0, statistic.getUnresolvedId(), statistic.getUnreadableDate())) + .build(); + } + + final ConceptQuery query = new ConceptQuery(new CQExternal(upload.getFormat(), upload.getValues())); + final ManagedExecution execution = postQuery(dataset, query, user); + + return Response.ok() + .entity(new UploadResponse( + execution.getId(), + statistic.getResolved().size(), + statistic.getUnresolvedId(), + statistic.getUnreadableDate() + )) + .build(); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java index ba4994e0f2..c4031c3464 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java @@ -4,10 +4,10 @@ import java.util.Arrays; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.stream.Collectors; import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.NotNull; import com.bakdata.conquery.apiv1.query.CQElement; import com.bakdata.conquery.io.cps.CPSType; @@ -30,8 +30,7 @@ import it.unimi.dsi.fastutil.ints.Int2ObjectAVLTreeMap; import it.unimi.dsi.fastutil.ints.Int2ObjectMap; import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; -import it.unimi.dsi.fastutil.ints.IntArrayList; -import it.unimi.dsi.fastutil.ints.IntList; +import lombok.Data; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -71,7 +70,7 @@ public QPNode createQueryPlan(QueryPlanContext context, ConceptQueryPlan plan) { } - private Int2ObjectMap readDates(String[][] values, List format, DateReader dateReader, FrontendConfig.UploadConfig queryUpload) { + private static Int2ObjectMap readDates(String[][] values, List format, DateReader dateReader, FrontendConfig.UploadConfig queryUpload) { Int2ObjectMap out = new Int2ObjectAVLTreeMap<>(); @@ -103,14 +102,52 @@ private Int2ObjectMap readDates(String[][] values, List format @Override public void resolve(QueryResolveContext context) { - valuesResolved = new Int2ObjectOpenHashMap<>(); + final ResolveStatistic resolved = resolveEntities(values, format, context.getNamespace().getStorage().getIdMapping(), context.getConfig().getFrontend().getQueryUpload(), context.getConfig().getPreprocessor().getParsers().getDateReader()); - final EntityIdMap mapping = context.getNamespace().getStorage().getIdMapping(); + if (resolved.getResolved().isEmpty()) { + throw new ConqueryError.ExternalResolveEmptyError(); + } + + if (!resolved.getUnreadableDate().isEmpty()) { + log.warn( + "Could not read dates {} of the {} rows. Not resolved: {}", + resolved.getUnreadableDate().size(), + values.length - 1, + resolved.getUnreadableDate().subList(0, Math.min(resolved.getUnreadableDate().size(), 10)) + ); + } + + if (!resolved.getUnresolvedId().isEmpty()) { + log.warn( + "Could not read dates {} of the {} rows. Not resolved: {}", + resolved.getUnresolvedId().size(), + values.length - 1, + resolved.getUnresolvedId().subList(0, Math.min(resolved.getUnresolvedId().size(), 10)) + ); + } + + valuesResolved = resolved.getResolved(); + } - final DateReader dateReader = context.getConfig().getPreprocessor().getParsers().getDateReader(); + @Data + public static class ResolveStatistic { + + @JsonIgnore + private final Map resolved; + + private final List unreadableDate; + private final List unresolvedId; + + } + + public static ResolveStatistic resolveEntities(@NotEmpty String[][] values, @NotEmpty List format, EntityIdMap mapping, FrontendConfig.UploadConfig queryUpload, @NotNull DateReader dateReader) { + Map resolved = new Int2ObjectOpenHashMap<>(); + + List unresolvedDate = new ArrayList<>(); + List unresolvedId = new ArrayList<>(); // extract dates from rows - final FrontendConfig.UploadConfig uploadConfig = context.getConfig().getFrontend().getQueryUpload(); + final FrontendConfig.UploadConfig uploadConfig = queryUpload; final Int2ObjectMap rowDates = readDates(values, format, dateReader, uploadConfig); @@ -118,7 +155,6 @@ public void resolve(QueryResolveContext context) { final ColumnConfig reader = uploadConfig.getIdMapper(format.get(idIndex)); - final List unresolved = new ArrayList<>(); // ignore the first row, because this is the header for (int rowNum = 1; rowNum < values.length; rowNum++) { @@ -128,31 +164,20 @@ public void resolve(QueryResolveContext context) { final int resolvedId = mapping.resolve(externalId); if (resolvedId == -1) { - unresolved.add(row); + unresolvedId.add(row); continue; } if (!rowDates.containsKey(rowNum)) { - unresolved.add(row); + unresolvedDate.add(row); continue; } //read the dates from the row - valuesResolved.put(resolvedId, rowDates.get(rowNum)); + resolved.put(resolvedId, rowDates.get(rowNum)); } - if (!unresolved.isEmpty()) { - log.warn( - "Could not resolve {} of the {} rows. Not resolved: {}", - unresolved.size(), - values.length - 1, - unresolved.subList(0, Math.min(unresolved.size(), 10)) - ); - } - - if (valuesResolved.isEmpty()) { - throw new ConqueryError.ExternalResolveEmptyError(); - } + return new ResolveStatistic(resolved,unresolvedDate,unresolvedId); } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java index 1758f69536..cc1f8a4674 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java @@ -3,13 +3,17 @@ import com.bakdata.conquery.apiv1.*; import com.bakdata.conquery.apiv1.query.QueryDescription; +import com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.auth.permissions.Ability; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.exceptions.JSONException; import com.bakdata.conquery.models.execution.ManagedExecution; +import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import io.dropwizard.auth.Auth; import io.dropwizard.jersey.PATCH; +import lombok.Getter; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import javax.inject.Inject; @@ -120,4 +124,19 @@ public void cancel(@Auth User user, @PathParam(QUERY) ManagedExecution query) query ); } + + @RequiredArgsConstructor + @Getter + public static class ExternalUpload { + private final List format; + private final String[][] values; + + private final ManagedExecutionId executionId; + } + + @POST + @Path("/upload/") + public Response upload(@Auth User user, ExternalUpload upload) { + return processor.uploadEntities(user, dataset, upload); + } } diff --git a/backend/src/test/resources/tests/endpoints/apiEndpointInfo.json b/backend/src/test/resources/tests/endpoints/apiEndpointInfo.json index 65a2ebb17e..a55bb6bdf6 100644 --- a/backend/src/test/resources/tests/endpoints/apiEndpointInfo.json +++ b/backend/src/test/resources/tests/endpoints/apiEndpointInfo.json @@ -14,6 +14,11 @@ "path": "/datasets/{dataset}/queries", "clazz": "QueryResource" }, + { + "method": "POST", + "path": "/datasets/{dataset}/queries/upload", + "clazz": "QueryResource" + }, { "method": "GET", "path": "/datasets/{dataset}/queries/{query}", From 742493325ee1197f79444387b5c0217e5cb69743 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 26 Jul 2021 14:35:20 +0000 Subject: [PATCH 27/82] Update AutoDoc --- docs/REST API JSONs.md | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/docs/REST API JSONs.md b/docs/REST API JSONs.md index 310c70624e..884b863b2e 100644 --- a/docs/REST API JSONs.md +++ b/docs/REST API JSONs.md @@ -103,7 +103,7 @@ Returns: [ResolvedConceptsResult](#Type-ResolvedConceptsResult)

-### GET datasets/{dataset}/queries [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L46) +### GET datasets/{dataset}/queries [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L50)
Details

@@ -117,7 +117,7 @@ Returns: list of [ExecutionStatus](#Type-ExecutionStatus)

-### POST datasets/{dataset}/queries [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L55) +### POST datasets/{dataset}/queries [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L59)
Details

@@ -132,7 +132,21 @@ Returns: `Response`

-### GET datasets/{dataset}/queries/{query} [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L67) +### POST datasets/{dataset}/queries/upload/ [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L137) + + +
Details

+ +Java Type: `com.bakdata.conquery.resources.api.QueryResource` + +Method: `upload` + +Expects: `ExternalUpload` +Returns: `Response` + +

+ +### GET datasets/{dataset}/queries/{query} [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L71)
Details

@@ -146,7 +160,7 @@ Returns: [FullExecutionStatus](#Type-FullExecutionStatus)

-### PATCH datasets/{dataset}/queries/{query} [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L80) +### PATCH datasets/{dataset}/queries/{query} [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L84)
Details

@@ -161,7 +175,7 @@ Returns: [FullExecutionStatus](#Type-FullExecutionStatus)

-### DELETE datasets/{dataset}/queries/{query} [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L91) +### DELETE datasets/{dataset}/queries/{query} [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L95)
Details

@@ -174,7 +188,7 @@ Returns: `void`

-### POST datasets/{dataset}/queries/{query}/cancel [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L110) +### POST datasets/{dataset}/queries/{query}/cancel [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L114)
Details

@@ -187,7 +201,7 @@ Returns: `void`

-### POST datasets/{dataset}/queries/{query}/reexecute [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L100) +### POST datasets/{dataset}/queries/{query}/reexecute [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L104)
Details

@@ -495,7 +509,7 @@ Supported Fields: | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQElement.java#L31-L33) | label | `String` | ? | | Allows the user to define labels. |

-### EXTERNAL [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java#L38-L40) +### EXTERNAL [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java#L37-L39) Allows uploading lists of entities.
Details

From d261a24797a4cad8696e9ab8c35fde7dc8f5d542 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Tue, 27 Jul 2021 09:36:21 +0200 Subject: [PATCH 28/82] adds missing initialization of Table#getColumns --- .../conquery/resources/admin/rest/AdminDatasetProcessor.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java index 391b8af522..6515969224 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java @@ -195,10 +195,13 @@ else if (!table.getDataset().equals(dataset)) { throw new IllegalArgumentException(); } + if (namespace.getStorage().getTable(table.getId()) != null) { throw new WebApplicationException("Table already exists", Response.Status.CONFLICT); } + table.init(); + ValidatorHelper.failOnError(log, validator.validate(table)); namespace.getStorage().addTable(table); From 34d058d1d9e300fa9d33450e717610587549b634 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Tue, 27 Jul 2021 09:40:52 +0200 Subject: [PATCH 29/82] add exclude to DictionaryMapping --- .../bakdata/conquery/models/dictionary/DictionaryMapping.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/backend/src/main/java/com/bakdata/conquery/models/dictionary/DictionaryMapping.java b/backend/src/main/java/com/bakdata/conquery/models/dictionary/DictionaryMapping.java index 707c3ac287..00ad342eef 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/dictionary/DictionaryMapping.java +++ b/backend/src/main/java/com/bakdata/conquery/models/dictionary/DictionaryMapping.java @@ -22,9 +22,12 @@ public class DictionaryMapping { private final Dictionary sourceDictionary; + private final Dictionary targetDictionary; + @ToString.Exclude private final Int2IntMap source2Target; + @ToString.Exclude private final Int2IntMap target2Source; private final int numberOfNewIds; From 1f066650e80e41ae5a89abe23c4f356475410a9c Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Tue, 27 Jul 2021 09:52:39 +0200 Subject: [PATCH 30/82] undo InternalOnly annos in conf as they will not be read --- .../java/com/bakdata/conquery/models/config/ColumnConfig.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java index fc15fe7d5f..3d7772b1a4 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java @@ -59,18 +59,14 @@ public EntityIdMap.ExternalId read(String value) { */ private Map description = Collections.emptyMap(); - @InternalOnly private String field; - @InternalOnly private String pad = null; - @InternalOnly private int length = -1; private boolean resolvable = false; - @InternalOnly private boolean fillAnon = false; } From ce394c4f33498e57c2bbb21c985c9d17489bed37 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Tue, 27 Jul 2021 11:02:57 +0200 Subject: [PATCH 31/82] only create Query, don't run it --- .../java/com/bakdata/conquery/apiv1/QueryProcessor.java | 5 ++++- .../bakdata/conquery/models/query/ExecutionManager.java | 9 --------- .../bakdata/conquery/resources/api/QueryResource.java | 2 +- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java index db81e07e5e..8e8258fe17 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java @@ -387,7 +387,10 @@ public Response uploadEntities(User user, Dataset dataset, QueryResource.Externa } final ConceptQuery query = new ConceptQuery(new CQExternal(upload.getFormat(), upload.getValues())); - final ManagedExecution execution = postQuery(dataset, query, user); + + // We only create the Query, really no need to execute it as it's only useful for composition. + final ManagedExecution execution = + datasetRegistry.get(dataset.getId()).getExecutionManager().createExecution(datasetRegistry, query, user, dataset); return Response.ok() .entity(new UploadResponse( diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/ExecutionManager.java b/backend/src/main/java/com/bakdata/conquery/models/query/ExecutionManager.java index 6ac7b0735f..c62b789163 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/ExecutionManager.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/ExecutionManager.java @@ -95,15 +95,6 @@ public ManagedExecution createExecution(DatasetRegistry datasets, QueryDescri return createQuery(datasets, query, UUID.randomUUID(), user, submittedDataset); } - /** - * Send message for query execution to all workers. - */ - private ManagedExecution executeQueryInNamespace(ManagedExecution query) { - final WorkerMessage executionMessage = query.createExecutionMessage(); - - namespace.sendToAll(executionMessage); - return query; - } public ManagedExecution createQuery(DatasetRegistry datasets, QueryDescription query, UUID queryId, User user, Dataset submittedDataset) { // Transform the submitted query into an initialized execution diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java index cc1f8a4674..d5825aa3ca 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java @@ -135,7 +135,7 @@ public static class ExternalUpload { } @POST - @Path("/upload/") + @Path("/upload") public Response upload(@Auth User user, ExternalUpload upload) { return processor.uploadEntities(user, dataset, upload); } From 4f55967bf179d2cf1c6f31871a4aac601317b5ee Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Tue, 27 Jul 2021 11:18:42 +0200 Subject: [PATCH 32/82] check for validity of ColumnConfig and locale keys --- .../conquery/models/config/ColumnConfig.java | 19 ++++++++++++++++++- .../models/config/FrontendConfig.java | 8 +++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java index 3d7772b1a4..a2233799a5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java @@ -1,13 +1,18 @@ package com.bakdata.conquery.models.config; import java.util.Collections; +import java.util.Locale; import java.util.Map; +import java.util.Objects; import javax.validation.constraints.NotEmpty; +import c10n.C10N; import com.bakdata.conquery.io.jackson.InternalOnly; import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.base.Strings; +import io.dropwizard.validation.ValidationMethod; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Getter; @@ -51,7 +56,7 @@ public EntityIdMap.ExternalId read(String value) { /** * Map of Localized labels. */ - + @NotEmpty private Map label = Collections.emptyMap(); /** @@ -69,4 +74,16 @@ public EntityIdMap.ExternalId read(String value) { private boolean fillAnon = false; + @JsonIgnore + @ValidationMethod(message = "Keys must be valid Locales.") + public boolean isLabelKeysLocale() { + return getLabel().keySet().stream().map(Locale::forLanguageTag).noneMatch(Objects::isNull); + } + + @JsonIgnore + @ValidationMethod(message = "Keys must be valid Locales.") + public boolean isDescriptionKeysLocale() { + return getDescription().keySet().stream().map(Locale::forLanguageTag).noneMatch(Objects::isNull); + } + } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java index 366d050e39..cfed108cb4 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java @@ -56,6 +56,7 @@ public class FrontendConfig { public static class UploadConfig { @NotEmpty + @Valid private List ids = List.of( ColumnConfig.builder() .name("ID") @@ -94,7 +95,7 @@ public int getIdIndex(List format) { for (int index = 0; index < format.size(); index++) { final String current = format.get(index); - if (ids.stream().map(ColumnConfig::getName).anyMatch(current::equals)) { + if (getIdMapper(current) != null) { return index; } } @@ -103,6 +104,7 @@ public int getIdIndex(List format) { } @NotNull + @Valid private ColumnConfig dateStart = ColumnConfig.builder() .name(DateFormat.START_DATE.name()) .label(Map.of("en", "Begin")) @@ -110,6 +112,7 @@ public int getIdIndex(List format) { .build(); @NotNull + @Valid private ColumnConfig dateEnd = ColumnConfig.builder() .name(DateFormat.END_DATE.name()) .label(Map.of("en", "End")) @@ -118,6 +121,7 @@ public int getIdIndex(List format) { @NotNull + @Valid private ColumnConfig dateRange = ColumnConfig.builder() .name(DateFormat.DATE_RANGE.name()) .label(Map.of("en", "Date Range")) @@ -126,6 +130,7 @@ public int getIdIndex(List format) { @NotNull + @Valid private ColumnConfig dateSet = ColumnConfig.builder() .name(DateFormat.DATE_SET.name()) .label(Map.of("en", "Dateset")) @@ -134,6 +139,7 @@ public int getIdIndex(List format) { @NotNull + @Valid private ColumnConfig eventDate = ColumnConfig.builder() .name(DateFormat.EVENT_DATE.name()) .label(Map.of("en", "Event Date")) From f620dd63f80fccdb5e98bf2e234c50634143341a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 27 Jul 2021 09:20:37 +0000 Subject: [PATCH 33/82] Update AutoDoc --- docs/Config JSON.md | 10 +++++----- docs/REST API JSONs.md | 12 ++++++------ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/Config JSON.md b/docs/Config JSON.md index c9248a656f..adb95b4ea6 100644 --- a/docs/Config JSON.md +++ b/docs/Config JSON.md @@ -223,7 +223,7 @@ Supported Fields: | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L68) | storage | `@Valid @NotNull StoreFactory` | | | |

-### Type CurrencyConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L207) +### Type CurrencyConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L213)
Details

@@ -234,10 +234,10 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L212) | decimalScale | `int` | `2` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L211) | decimalSeparator | `String` | `","` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L209) | prefix | `String` | `"€"` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L210) | thousandSeparator | `String` | `"."` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L218) | decimalScale | `int` | `2` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L217) | decimalSeparator | `String` | `","` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L215) | prefix | `String` | `"€"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L216) | thousandSeparator | `String` | `"."` | | |

### Type FrontendConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L35) diff --git a/docs/REST API JSONs.md b/docs/REST API JSONs.md index 884b863b2e..4862d0891d 100644 --- a/docs/REST API JSONs.md +++ b/docs/REST API JSONs.md @@ -132,7 +132,7 @@ Returns: `Response`

-### POST datasets/{dataset}/queries/upload/ [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L137) +### POST datasets/{dataset}/queries/upload [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L137)
Details

@@ -755,7 +755,7 @@ Supported Fields: | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptResource.java#L68) | concepts | list of `String` | `null` | | |

-### Type CurrencyConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L207) +### Type CurrencyConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L213)
Details

@@ -766,10 +766,10 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L212) | decimalScale | `int` | `2` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L211) | decimalSeparator | `String` | `","` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L209) | prefix | `String` | `"€"` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L210) | thousandSeparator | `String` | `"."` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L218) | decimalScale | `int` | `2` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L217) | decimalSeparator | `String` | `","` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L215) | prefix | `String` | `"€"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L216) | thousandSeparator | `String` | `"."` | | |

### Type ExecutionStatus [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/ExecutionStatus.java#L18) From 70b5aeaad04591e3dce11d2f85fef31288a621c1 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Tue, 27 Jul 2021 11:59:03 +0200 Subject: [PATCH 34/82] use view to reduce API surface for QueryUpload --- .../java/com/bakdata/conquery/Conquery.java | 10 ++++++++-- .../conquery/models/config/ColumnConfig.java | 3 +++ .../conquery/models/config/FrontendConfig.java | 2 +- .../conquery/resources/api/ConfigResource.java | 17 ++++++++++++++--- 4 files changed, 26 insertions(+), 6 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/Conquery.java b/backend/src/main/java/com/bakdata/conquery/Conquery.java index af84bae5d4..4c54818fa9 100644 --- a/backend/src/main/java/com/bakdata/conquery/Conquery.java +++ b/backend/src/main/java/com/bakdata/conquery/Conquery.java @@ -12,10 +12,12 @@ import com.bakdata.conquery.commands.RecodeStoreCommand; import com.bakdata.conquery.commands.ShardNode; import com.bakdata.conquery.commands.StandaloneCommand; +import com.bakdata.conquery.io.jackson.InternalOnly; import com.bakdata.conquery.io.jackson.Jackson; import com.bakdata.conquery.io.jackson.MutableInjectableValues; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.util.UrlRewriteBundle; +import com.fasterxml.jackson.databind.ObjectMapper; import io.dropwizard.Application; import io.dropwizard.ConfiguredBundle; import io.dropwizard.configuration.JsonConfigurationFactory; @@ -44,7 +46,11 @@ public Conquery() { @Override public void initialize(Bootstrap bootstrap) { - Jackson.configure(bootstrap.getObjectMapper()); + final ObjectMapper confMapper = bootstrap.getObjectMapper(); + Jackson.configure(confMapper); + + confMapper.setConfig(confMapper.getDeserializationConfig().withView(InternalOnly.class)); + // check for java compiler, needed for the class generation if (ToolProvider.getSystemJavaCompiler() == null) { throw new IllegalStateException("Conquery requires to be run on either a JDK or a ServerJRE"); @@ -59,7 +65,7 @@ public void initialize(Bootstrap bootstrap) { bootstrap.addCommand(new StandaloneCommand(this)); bootstrap.addCommand(new RecodeStoreCommand()); - ((MutableInjectableValues)bootstrap.getObjectMapper().getInjectableValues()).add(Validator.class, bootstrap.getValidatorFactory().getValidator()); + ((MutableInjectableValues) confMapper.getInjectableValues()).add(Validator.class, bootstrap.getValidatorFactory().getValidator()); // do some setup in other classes after initialization but before running a // command diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java index a2233799a5..6158277ca5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java @@ -66,12 +66,15 @@ public EntityIdMap.ExternalId read(String value) { private String field; + @InternalOnly private String pad = null; + @InternalOnly private int length = -1; private boolean resolvable = false; + @InternalOnly private boolean fillAnon = false; @JsonIgnore diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java index cfed108cb4..9406bbdedd 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java @@ -35,10 +35,10 @@ @ToString @Getter @Setter -@With @AllArgsConstructor @NoArgsConstructor @Slf4j +@With public class FrontendConfig { private String version = VersionInfo.INSTANCE.getProjectVersion(); diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ConfigResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ConfigResource.java index a7655e84e3..2c43f41df2 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ConfigResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ConfigResource.java @@ -1,10 +1,13 @@ package com.bakdata.conquery.resources.api; +import java.util.stream.Collectors; + import javax.ws.rs.GET; import javax.ws.rs.Path; import javax.ws.rs.Produces; import com.bakdata.conquery.apiv1.AdditionalMediaTypes; +import com.bakdata.conquery.models.config.ColumnConfig; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.config.FrontendConfig; import lombok.RequiredArgsConstructor; @@ -20,8 +23,16 @@ public class ConfigResource { @GET @Path("frontend") public FrontendConfig getFrontendConfig() { - - - return config.getFrontend(); + // Filter Ids that are not resolvable + return config.getFrontend() + .withQueryUpload(config.getFrontend() + .getQueryUpload() + .withIds(config.getFrontend() + .getQueryUpload() + .getIds() + .stream() + .filter(ColumnConfig::isResolvable) + .collect(Collectors.toList()))); } + } \ No newline at end of file From 5c6e72f94923b32d9ee04c3bed1da6bc90b4974d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 27 Jul 2021 10:01:19 +0000 Subject: [PATCH 35/82] Update AutoDoc --- docs/REST API JSONs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/REST API JSONs.md b/docs/REST API JSONs.md index 4862d0891d..159501ca89 100644 --- a/docs/REST API JSONs.md +++ b/docs/REST API JSONs.md @@ -20,7 +20,7 @@ Returns: list of [IdLabel](#Type-IdLabel)

-### GET config/frontend [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConfigResource.java#L20) +### GET config/frontend [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConfigResource.java#L23)
Details

From 0dbdcb2e848e3abc091b20b10c20d4c50b06fd00 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Tue, 27 Jul 2021 12:01:47 +0200 Subject: [PATCH 36/82] remove more fields from API --- .../java/com/bakdata/conquery/models/config/ColumnConfig.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java index 6158277ca5..707d267446 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java @@ -64,6 +64,7 @@ public EntityIdMap.ExternalId read(String value) { */ private Map description = Collections.emptyMap(); + @InternalOnly private String field; @InternalOnly @@ -72,6 +73,7 @@ public EntityIdMap.ExternalId read(String value) { @InternalOnly private int length = -1; + @InternalOnly private boolean resolvable = false; @InternalOnly From 12e9e417c752a6ee02c2a1f01835649a0bae5c2b Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Tue, 27 Jul 2021 15:07:48 +0200 Subject: [PATCH 37/82] use the csv output directly from the endpoint --- .../concept/specific/external/CQExternal.java | 24 ++-- .../concept/specific/external/DateFormat.java | 53 +++------ .../io/result/csv/ResultCsvProcessor.java | 3 +- .../models/config/FrontendConfig.java | 6 +- .../com/bakdata/conquery/util/io/Cloner.java | 32 +++--- .../integration/IntegrationTests.java | 107 +++++++++++------- .../json/AbstractQueryEngineTest.java | 51 +++++---- .../conquery/util/support/TestConquery.java | 4 +- 8 files changed, 143 insertions(+), 137 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java index c4031c3464..12c4d06913 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java @@ -78,19 +78,25 @@ private static Int2ObjectMap readDates(String[][] values, List //validate structures - final int[] datePositions = DateFormat.select(dateFormats); - - DateFormat dateFormat = datePositions.length > 0 ? dateFormats.get(datePositions[0]) : DateFormat.ALL; - for (int row = 1; row < values.length; row++) { try { - final CDateSet dates = dateFormat.readDates(datePositions, values[row], dateReader); + for (int col = 0; col < dateFormats.size(); col++) { + final DateFormat dateFormat = dateFormats.get(col); - if (dates == null) { - continue; - } + if(dateFormat == null){ + continue; + } + + CDateSet dates = CDateSet.create(); - out.put(row, dates); + dateFormat.readDates(values[row][col], dateReader, dates); + + if (dates.isEmpty()) { + continue; + } + + out.put(row, dates); + } } catch (Exception e) { log.warn("Failed to parse Date from {}", row, e); diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java index d172a177c3..e4c83a7308 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java @@ -1,7 +1,5 @@ package com.bakdata.conquery.apiv1.query.concept.specific.external; -import java.time.LocalDate; -import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; @@ -14,68 +12,43 @@ public enum DateFormat { EVENT_DATE { @Override - public CDateSet readDates(int[] formats, String[] row, DateReader dateReader) { - final int index = formats[0]; - - return CDateSet.create(Collections.singleton(CDateRange.exactly(dateReader.parseToLocalDate(row[index])))); + public void readDates(String value, DateReader dateReader, CDateSet out) { + out.add(CDateRange.exactly(dateReader.parseToLocalDate(value))); } }, END_DATE { @Override - public CDateSet readDates(int[] formats, String[] row, DateReader dateReader) { - - if (formats.length == 1) { - return CDateSet.create(CDateRange.atMost(dateReader.parseToLocalDate(row[0]))); - } - - return END_DATE.readDates(formats, row, dateReader); + public void readDates(String value, DateReader dateReader, CDateSet out) { + out.add(CDateRange.atMost(dateReader.parseToLocalDate(value))); } }, START_DATE { @Override - public CDateSet readDates(int[] formats, String[] row, DateReader dateReader) { - - if (formats.length == 1) { - return CDateSet.create(CDateRange.atLeast(dateReader.parseToLocalDate(row[0]))); - } - - final int startIndex = formats[0]; - final int endIndex = formats[1]; - - LocalDate start = dateReader.parseToLocalDate(row[startIndex]); - LocalDate end = dateReader.parseToLocalDate(row[endIndex]); - - if (start == null && end == null) { - return null; - } - - return CDateSet.create(Collections.singleton(CDateRange.of(start, end))); + public void readDates(String value, DateReader dateReader, CDateSet out) { + out.add(CDateRange.atLeast(dateReader.parseToLocalDate(value))); } }, DATE_RANGE { @Override - public CDateSet readDates(int[] formats, String[] row, DateReader dateReader) { - final int index = formats[0]; - - return CDateSet.create(Collections.singleton(DateRangeParser.parseISORange(row[index], dateReader))); + public void readDates(String value, DateReader dateReader, CDateSet out) { + out.add(DateRangeParser.parseISORange(value, dateReader)); } }, DATE_SET { @Override - public CDateSet readDates(int[] formats, String[] row, DateReader dateReader) { - final int index = formats[0]; + public void readDates(String value, DateReader dateReader, CDateSet out) { - return CDateSet.parse(row[index], dateReader); + out.addAll(CDateSet.parse(value, dateReader)); } }, ALL { @Override - public CDateSet readDates(int[] formats, String[] row, DateReader dateReader) { - return CDateSet.createFull(); + public void readDates(String value, DateReader dateReader, CDateSet out) { + out.add(CDateRange.all()); } }; - public abstract CDateSet readDates(int[] formats, String[] row, DateReader dateReader); + public abstract void readDates(String value, DateReader dateReader, CDateSet out); public static int[] select(List dateFormat) { final List distinct = dateFormat.stream().filter(Objects::nonNull).distinct().sorted().collect(Collectors.toList()); diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java index b6d2b055fa..cac11892ec 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java @@ -18,7 +18,6 @@ import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.i18n.I18n; - import com.bakdata.conquery.models.identifiable.mapping.IdPrinter; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.SingleTableResult; @@ -63,7 +62,7 @@ public & SingleTableResult> Response getResult(Us Charset charset = determineCharset(userAgent, queryCharset); - StreamingOutput out = os -> { + StreamingOutput out = os -> { try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(os, charset))) { CsvRenderer renderer = new CsvRenderer(config.getCsv().createWriter(writer), settings); renderer.toCSV(config.getFrontend().getQueryUpload().getPrintIdFields(), exec.getResultInfo(), exec.streamResults()); diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java index 9406bbdedd..fa053e326d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java @@ -1,6 +1,7 @@ package com.bakdata.conquery.models.config; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -24,6 +25,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import groovy.transform.ToString; import io.dropwizard.validation.ValidationMethod; +import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Data; import lombok.Getter; @@ -67,6 +69,8 @@ public static class UploadConfig { ); @JsonIgnore + @Setter(AccessLevel.NONE) + @Getter(AccessLevel.NONE) private List idFieldsCached; /** @@ -78,7 +82,7 @@ public List getPrintIdFields() { idFieldsCached = ids.stream() .map(ColumnConfig::getField) .filter(Objects::nonNull) - .collect(Collectors.toList()); + .collect(Collectors.toUnmodifiableList()); } return idFieldsCached; diff --git a/backend/src/main/java/com/bakdata/conquery/util/io/Cloner.java b/backend/src/main/java/com/bakdata/conquery/util/io/Cloner.java index 7dface2db9..f972e6a516 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/io/Cloner.java +++ b/backend/src/main/java/com/bakdata/conquery/util/io/Cloner.java @@ -12,35 +12,37 @@ public class Cloner { - public static ConqueryConfig clone(ConqueryConfig config, Map , T > injectables) { + public static ConqueryConfig clone(ConqueryConfig config, Map, T> injectables, ObjectMapper mapper) { try { - ObjectMapper mapper = Jackson.BINARY_MAPPER.copy(); - MutableInjectableValues injectableHolder = ((MutableInjectableValues)Jackson.BINARY_MAPPER.getInjectableValues()); - for(Entry, T> injectable : injectables.entrySet()) { + + MutableInjectableValues injectableHolder = ((MutableInjectableValues) mapper.getInjectableValues()); + for (Entry, T> injectable : injectables.entrySet()) { injectableHolder.add(injectable.getKey(), injectable.getValue()); } ConqueryConfig clone = mapper.readValue( - Jackson.BINARY_MAPPER.writeValueAsBytes(config), - ConqueryConfig.class + mapper.writeValueAsBytes(config), + ConqueryConfig.class ); clone.setLoggingFactory(config.getLoggingFactory()); return clone; - } catch (IOException e) { - throw new IllegalStateException("Failed to clone a conquery config "+config, e); + } + catch (IOException e) { + throw new IllegalStateException("Failed to clone a conquery config " + config, e); } } public static T clone(T element, Injectable injectable, Class valueType) { try { return injectable - .injectInto(Jackson.BINARY_MAPPER) - .readValue( - Jackson.BINARY_MAPPER.writeValueAsBytes(element), - valueType - ); - } catch (IOException e) { - throw new IllegalStateException("Failed to clone the CQElement "+element, e); + .injectInto(Jackson.BINARY_MAPPER) + .readValue( + Jackson.BINARY_MAPPER.writeValueAsBytes(element), + valueType + ); + } + catch (IOException e) { + throw new IllegalStateException("Failed to clone the CQElement " + element, e); } } } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java b/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java index 99f562ae62..f767152b97 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java @@ -7,7 +7,13 @@ import java.io.InputStream; import java.net.URI; import java.nio.file.Files; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -15,16 +21,16 @@ import com.bakdata.conquery.TestTags; import com.bakdata.conquery.integration.json.JsonIntegrationTest; import com.bakdata.conquery.integration.tests.ProgrammaticIntegrationTest; -import com.bakdata.conquery.integration.tests.RestartTest; import com.bakdata.conquery.io.cps.CPSTypeIdResolver; +import com.bakdata.conquery.io.jackson.InternalOnly; import com.bakdata.conquery.io.jackson.Jackson; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.util.io.Cloner; import com.bakdata.conquery.util.support.ConfigOverride; import com.bakdata.conquery.util.support.TestConquery; +import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import io.github.classgraph.Resource; -import it.unimi.dsi.fastutil.ints.Int2ObjectArrayMap; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.SneakyThrows; @@ -39,14 +45,27 @@ @Slf4j public class IntegrationTests { - private static final ObjectWriter CONFIG_WRITER = Jackson.MAPPER.writerFor(ConqueryConfig.class); + private static final ObjectMapper MAPPER; + private static final ObjectWriter CONFIG_WRITER; + + static { + final ObjectMapper mapper = Jackson.MAPPER.copy(); + + MAPPER = mapper.setConfig(mapper.getDeserializationConfig().withView(InternalOnly.class)) + .setConfig(mapper.getSerializationConfig().withView(InternalOnly.class)); + + CONFIG_WRITER = MAPPER.writerFor(ConqueryConfig.class); + } + + private static final Map reusedInstances = new HashMap<>(); - + private final String defaultTestRoot; private final String defaultTestRootPackage; @Getter private final File workDir; - @Getter @RegisterExtension + @Getter + @RegisterExtension public static TestConqueryConfig DEFAULT_CONFIG = new TestConqueryConfig(); @SneakyThrows(IOException.class) @@ -62,10 +81,10 @@ public List jsonTests() { ResourceTree tree = new ResourceTree(null, null); tree.addAll( - CPSTypeIdResolver.SCAN_RESULT - .getResourcesMatchingPattern(Pattern.compile("^" + testRoot + ".*\\.test\\.json$")) + CPSTypeIdResolver.SCAN_RESULT + .getResourcesMatchingPattern(Pattern.compile("^" + testRoot + ".*\\.test\\.json$")) ); - + // collect tests from directory if (tree.getChildren().isEmpty()) { log.warn("Could not find tests in {}", testRoot); @@ -77,64 +96,65 @@ public List jsonTests() { return Collections.singletonList(collectTests(reduced)); } return reduced.getChildren().values().stream() - .map(this::collectTests) - .collect(Collectors.toList()); + .map(this::collectTests) + .collect(Collectors.toList()); } @SneakyThrows public Stream programmaticTests() { List> programmatic = CPSTypeIdResolver - .SCAN_RESULT - .getClassesImplementing(ProgrammaticIntegrationTest.class.getName()) - .filter(info -> info.getPackageName().startsWith(defaultTestRootPackage)) - .loadClasses(); + .SCAN_RESULT + .getClassesImplementing(ProgrammaticIntegrationTest.class.getName()) + .filter(info -> info.getPackageName().startsWith(defaultTestRootPackage)) + .loadClasses(); return programmatic - .stream() - .map(c-> { - try { - return c.asSubclass(ProgrammaticIntegrationTest.class).getDeclaredConstructor().newInstance(); - } - catch(Exception e) { - throw new RuntimeException(e); - } - }) - .map(this::createDynamicProgrammaticTestNode); + .stream() + .map(c -> { + try { + return c.asSubclass(ProgrammaticIntegrationTest.class).getDeclaredConstructor().newInstance(); + } + catch (Exception e) { + throw new RuntimeException(e); + } + }) + .map(this::createDynamicProgrammaticTestNode); } private DynamicTest createDynamicProgrammaticTestNode(ProgrammaticIntegrationTest test) { TestConquery conquery = getCachedConqueryInstance(workDir, getConfigOverride(test)); + return DynamicTest.dynamicTest( test.getClass().getSimpleName(), //classpath URI - URI.create("classpath:/"+test.getClass().getName().replace('.', '/')+".java"), + URI.create("classpath:/" + test.getClass().getName().replace('.', '/') + ".java"), new IntegrationTest.Wrapper(test.getClass().getSimpleName(), conquery, test) ); } private DynamicNode collectTests(ResourceTree currentDir) { - if(currentDir.getValue() != null) { + if (currentDir.getValue() != null) { return readTest(currentDir.getValue(), currentDir.getName(), this); } List list = new ArrayList<>(); - for(ResourceTree child : currentDir.getChildren().values()) { + for (ResourceTree child : currentDir.getChildren().values()) { list.add(collectTests(child)); } list.sort(Comparator.comparing(DynamicNode::getDisplayName)); - + return dynamicContainer( - currentDir.getName(), - URI.create("classpath:/"+currentDir.getFullName()+"/"), - list.stream() + currentDir.getName(), + URI.create("classpath:/" + currentDir.getFullName() + "/"), + list.stream() ); } private static DynamicTest readTest(Resource resource, String name, IntegrationTests integrationTests) { - try(InputStream in = resource.open()) { + try (InputStream in = resource.open()) { JsonIntegrationTest test = new JsonIntegrationTest(in); ConqueryConfig conf = getConfigOverride(test); @@ -148,23 +168,24 @@ private static DynamicTest readTest(Resource resource, String name, IntegrationT new IntegrationTest.Wrapper( name, conquery, - test) + test + ) ); } - catch(Exception e) { + catch (Exception e) { return DynamicTest.dynamicTest( - name, - resource.getURI(), - () -> { - throw e; - } + name, + resource.getURI(), + () -> { + throw e; + } ); } } @NotNull private static ConqueryConfig getConfigOverride(IntegrationTest test) { - ConqueryConfig conf = Cloner.clone(DEFAULT_CONFIG, Map.of()); + ConqueryConfig conf = Cloner.clone(DEFAULT_CONFIG, Map.of(), MAPPER); test.overrideConfig(conf); return conf; } @@ -173,7 +194,7 @@ private static ConqueryConfig getConfigOverride(IntegrationTest test) { private static synchronized TestConquery getCachedConqueryInstance(File workDir, ConqueryConfig conf) { // This should be fast enough and a stable comparison String confString = CONFIG_WRITER.writeValueAsString(conf); - if(!reusedInstances.containsKey(confString)){ + if (!reusedInstances.containsKey(confString)) { // For the overriden config we must override the ports so there are no clashes // We do it here so the config "hash" is not influenced by the port settings TestConquery.configureRandomPorts(conf); @@ -187,7 +208,7 @@ private static synchronized TestConquery getCachedConqueryInstance(File workDir, } @EqualsAndHashCode(callSuper = true) - public static class TestConqueryConfig extends ConqueryConfig implements Extension, BeforeAllCallback { + public static class TestConqueryConfig extends ConqueryConfig implements Extension, BeforeAllCallback { @Override public void beforeAll(ExtensionContext context) throws Exception { diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java b/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java index 118536ad7e..1709155450 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java @@ -1,27 +1,32 @@ package com.bakdata.conquery.integration.json; +import static com.bakdata.conquery.resources.ResourceConstants.DATASET; +import static com.bakdata.conquery.resources.ResourceConstants.QUERY; import static org.assertj.core.api.Assertions.assertThat; import java.io.IOException; +import java.io.InputStream; import java.util.List; import java.util.Locale; -import java.util.stream.Collectors; +import java.util.Map; +import javax.ws.rs.core.Response; + +import com.bakdata.conquery.apiv1.AdditionalMediaTypes; import com.bakdata.conquery.apiv1.query.Query; import com.bakdata.conquery.integration.common.IntegrationUtils; import com.bakdata.conquery.integration.common.ResourceFile; -import com.bakdata.conquery.io.result.CsvLineStreamRenderer; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.execution.ExecutionState; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; -import com.bakdata.conquery.models.identifiable.mapping.IdPrinter; import com.bakdata.conquery.models.query.ManagedQuery; -import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.query.results.MultilineEntityResult; import com.bakdata.conquery.models.worker.DatasetRegistry; +import com.bakdata.conquery.resources.api.ResultCsvResource; +import com.bakdata.conquery.resources.hierarchies.HierarchyHelper; import com.bakdata.conquery.util.NonPersistentStoreFactory; import com.bakdata.conquery.util.support.StandaloneSupport; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -66,32 +71,28 @@ public void executeTest(StandaloneSupport standaloneSupport) throws IOException .as("Should have same size as result infos") .allSatisfy(v -> assertThat(v).hasSameSizeAs(resultInfos)); - IdPrinter idPrinter = config.getFrontend().getQueryUpload().getIdPrinter(testUser, execution, execution.getNamespace()); - - PrintSettings - PRINT_SETTINGS = - new PrintSettings( - false, - Locale.ENGLISH, - namespaces, - config, - idPrinter::createId, - (columnInfo) -> columnInfo.getSelect().getId().toStringWithoutDataset() - ); - - CsvLineStreamRenderer renderer = new CsvLineStreamRenderer(config.getCsv().createWriter(), PRINT_SETTINGS); - - List actual = renderer.toStream( - config.getFrontend().getQueryUpload().getPrintIdFields(), - resultInfos, - execution.streamResults() - ).collect(Collectors.toList()); + // Get the actual response and compare with expected result. + final Response csvResponse = + standaloneSupport.getClient() + .target(HierarchyHelper.hierarchicalPath(standaloneSupport.defaultApiURIBuilder(), ResultCsvResource.class, "getAsCsv") + .buildFromMap( + Map.of(DATASET, standaloneSupport.getDataset().getName(), + QUERY, execution.getId().toString() + ) + )) + .request(AdditionalMediaTypes.CSV) + .acceptLanguage(Locale.ENGLISH) + .get(); + + List actual = In.stream(((InputStream) csvResponse.getEntity())).readLines(); ResourceFile expectedCsv = getExpectedCsv(); List expected = In.stream(expectedCsv.stream()).readLines(); - assertThat(actual).as("Results for %s are not as expected.", this).containsExactlyInAnyOrderElementsOf(expected); + assertThat(actual).as("Results for %s are not as expected.", this) + .containsExactlyInAnyOrderElementsOf(expected); + // check that getLastResultCount returns the correct size if (execution.streamResults().noneMatch(MultilineEntityResult.class::isInstance)) { assertThat(execution.getLastResultCount()).as("Result count for %s is not as expected.", this).isEqualTo(expected.size() - 1); diff --git a/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java b/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java index e7f54ec527..2ec0325784 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java +++ b/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java @@ -19,13 +19,13 @@ import com.bakdata.conquery.Conquery; import com.bakdata.conquery.commands.ShardNode; import com.bakdata.conquery.commands.StandaloneCommand; +import com.bakdata.conquery.io.jackson.Jackson; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.config.XodusStoreFactory; import com.bakdata.conquery.models.execution.ExecutionState; import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; -import com.bakdata.conquery.models.messages.network.specific.RemoveWorker; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.util.Wait; @@ -137,7 +137,7 @@ private synchronized StandaloneSupport createSupport(DatasetId datasetId, String log.info("Reusing existing folder {} for Support", localTmpDir.getPath()); } - ConqueryConfig localCfg = Cloner.clone(config, Map.of(Validator.class, standaloneCommand.getManager().getEnvironment().getValidator())); + ConqueryConfig localCfg = Cloner.clone(config, Map.of(Validator.class, standaloneCommand.getManager().getEnvironment().getValidator()), Jackson.BINARY_MAPPER.copy()); StandaloneSupport support = new StandaloneSupport( From 6e362a81ae4bdcfa0da94e12128b395deeabde99 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 27 Jul 2021 13:10:04 +0000 Subject: [PATCH 38/82] Update AutoDoc --- docs/Config JSON.md | 18 +++++++++--------- docs/REST API JSONs.md | 18 +++++++++--------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/docs/Config JSON.md b/docs/Config JSON.md index adb95b4ea6..00b1cc6f43 100644 --- a/docs/Config JSON.md +++ b/docs/Config JSON.md @@ -223,7 +223,7 @@ Supported Fields: | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L68) | storage | `@Valid @NotNull StoreFactory` | | | |

-### Type CurrencyConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L213) +### Type CurrencyConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L217)
Details

@@ -234,13 +234,13 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L218) | decimalScale | `int` | `2` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L217) | decimalSeparator | `String` | `","` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L215) | prefix | `String` | `"€"` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L216) | thousandSeparator | `String` | `"."` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L222) | decimalScale | `int` | `2` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L221) | decimalSeparator | `String` | `","` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L219) | prefix | `String` | `"€"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L220) | thousandSeparator | `String` | `"."` | | |

-### Type FrontendConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L35) +### Type FrontendConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L37)
Details

@@ -251,9 +251,9 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L45) | currency | [CurrencyConfig](#Type-CurrencyConfig) | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L49) | queryUpload | `UploadConfig` | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L44) | version | `String` | `"0.0.0-SNAPSHOT"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L47) | currency | [CurrencyConfig](#Type-CurrencyConfig) | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L51) | queryUpload | `UploadConfig` | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L46) | version | `String` | `"0.0.0-SNAPSHOT"` | | |

### Type LocaleConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L11) diff --git a/docs/REST API JSONs.md b/docs/REST API JSONs.md index 159501ca89..fdf4537bc1 100644 --- a/docs/REST API JSONs.md +++ b/docs/REST API JSONs.md @@ -755,7 +755,7 @@ Supported Fields: | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptResource.java#L68) | concepts | list of `String` | `null` | | |

-### Type CurrencyConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L213) +### Type CurrencyConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L217)
Details

@@ -766,10 +766,10 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L218) | decimalScale | `int` | `2` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L217) | decimalSeparator | `String` | `","` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L215) | prefix | `String` | `"€"` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L216) | thousandSeparator | `String` | `"."` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L222) | decimalScale | `int` | `2` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L221) | decimalSeparator | `String` | `","` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L219) | prefix | `String` | `"€"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L220) | thousandSeparator | `String` | `"."` | | |

### Type ExecutionStatus [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/ExecutionStatus.java#L18) @@ -885,7 +885,7 @@ No fields can be set for this type.

-### Type FrontendConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L35) +### Type FrontendConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L37)
Details

@@ -896,9 +896,9 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L45) | currency | [CurrencyConfig](#Type-CurrencyConfig) | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L49) | queryUpload | `UploadConfig` | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L44) | version | `String` | `"0.0.0-SNAPSHOT"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L47) | currency | [CurrencyConfig](#Type-CurrencyConfig) | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L51) | queryUpload | `UploadConfig` | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L46) | version | `String` | `"0.0.0-SNAPSHOT"` | | |

### Type FullExecutionStatus [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/FullExecutionStatus.java#L20-L24) From 1b0fbafaa0f4c145d5a238c4b556584ccdbf6c4e Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Tue, 27 Jul 2021 15:10:37 +0200 Subject: [PATCH 39/82] cleanup --- .../concept/specific/external/CQExternal.java | 8 +++---- .../concept/specific/external/DateFormat.java | 21 ------------------- .../conquery/resources/api/QueryResource.java | 13 +++++++++++- 3 files changed, 15 insertions(+), 27 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java index 12c4d06913..6a8bf31859 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java @@ -73,7 +73,6 @@ public QPNode createQueryPlan(QueryPlanContext context, ConceptQueryPlan plan) { private static Int2ObjectMap readDates(String[][] values, List format, DateReader dateReader, FrontendConfig.UploadConfig queryUpload) { Int2ObjectMap out = new Int2ObjectAVLTreeMap<>(); - List dateFormats = format.stream().map(queryUpload::resolveDateFormat).collect(Collectors.toList()); //validate structures @@ -153,13 +152,12 @@ public static ResolveStatistic resolveEntities(@NotEmpty String[][] values, @Not List unresolvedId = new ArrayList<>(); // extract dates from rows - final FrontendConfig.UploadConfig uploadConfig = queryUpload; - final Int2ObjectMap rowDates = readDates(values, format, dateReader, uploadConfig); + final Int2ObjectMap rowDates = readDates(values, format, dateReader, queryUpload); - final int idIndex = uploadConfig.getIdIndex(format); + final int idIndex = queryUpload.getIdIndex(format); - final ColumnConfig reader = uploadConfig.getIdMapper(format.get(idIndex)); + final ColumnConfig reader = queryUpload.getIdMapper(format.get(idIndex)); // ignore the first row, because this is the header diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java index e4c83a7308..d1c2ed33c1 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java @@ -37,7 +37,6 @@ public void readDates(String value, DateReader dateReader, CDateSet out) { DATE_SET { @Override public void readDates(String value, DateReader dateReader, CDateSet out) { - out.addAll(CDateSet.parse(value, dateReader)); } }, @@ -50,24 +49,4 @@ public void readDates(String value, DateReader dateReader, CDateSet out) { public abstract void readDates(String value, DateReader dateReader, CDateSet out); - public static int[] select(List dateFormat) { - final List distinct = dateFormat.stream().filter(Objects::nonNull).distinct().sorted().collect(Collectors.toList()); - - // => ALL - if (distinct.isEmpty()) { - return new int[0]; - } - - - if (distinct.size() == 1) { - return new int[]{dateFormat.indexOf(distinct.get(0))}; - } - - if (distinct.size() == 2 && distinct.get(0).equals(START_DATE) && distinct.get(1).equals(END_DATE)) { - return new int[]{dateFormat.indexOf(START_DATE), dateFormat.indexOf(END_DATE)}; - } - - throw new IllegalStateException("can only handle 1 or 2 format columns"); //TODO map error - - } } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java index d5825aa3ca..a1f42e0200 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java @@ -10,8 +10,10 @@ import com.bakdata.conquery.models.exceptions.JSONException; import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; +import com.fasterxml.jackson.annotation.JsonIgnore; import io.dropwizard.auth.Auth; import io.dropwizard.jersey.PATCH; +import io.dropwizard.validation.ValidationMethod; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -24,6 +26,8 @@ import javax.ws.rs.core.Context; import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; + +import java.util.Arrays; import java.util.List; import java.util.Optional; import java.util.concurrent.TimeUnit; @@ -132,11 +136,18 @@ public static class ExternalUpload { private final String[][] values; private final ManagedExecutionId executionId; + + @JsonIgnore + @ValidationMethod(message = "Values and Format are not of same width.") + public boolean isAllSameLength() { + final int expected = format.size(); + return Arrays.stream(values).mapToInt(a -> a.length).allMatch(v -> expected == v); + } } @POST @Path("/upload") - public Response upload(@Auth User user, ExternalUpload upload) { + public Response upload(@Auth User user, @Valid ExternalUpload upload) { return processor.uploadEntities(user, dataset, upload); } } From 453ae3ae0a949bf31ecca3016e582791c36b27d3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 27 Jul 2021 13:12:43 +0000 Subject: [PATCH 40/82] Update AutoDoc --- docs/REST API JSONs.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/REST API JSONs.md b/docs/REST API JSONs.md index fdf4537bc1..03b364470b 100644 --- a/docs/REST API JSONs.md +++ b/docs/REST API JSONs.md @@ -103,7 +103,7 @@ Returns: [ResolvedConceptsResult](#Type-ResolvedConceptsResult)

-### GET datasets/{dataset}/queries [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L50) +### GET datasets/{dataset}/queries [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L54)
Details

@@ -117,7 +117,7 @@ Returns: list of [ExecutionStatus](#Type-ExecutionStatus)

-### POST datasets/{dataset}/queries [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L59) +### POST datasets/{dataset}/queries [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L63)
Details

@@ -132,7 +132,7 @@ Returns: `Response`

-### POST datasets/{dataset}/queries/upload [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L137) +### POST datasets/{dataset}/queries/upload [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L148)
Details

@@ -141,12 +141,12 @@ Java Type: `com.bakdata.conquery.resources.api.QueryResource` Method: `upload` -Expects: `ExternalUpload` +Expects: `@Valid ExternalUpload` Returns: `Response`

-### GET datasets/{dataset}/queries/{query} [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L71) +### GET datasets/{dataset}/queries/{query} [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L75)
Details

@@ -160,7 +160,7 @@ Returns: [FullExecutionStatus](#Type-FullExecutionStatus)

-### PATCH datasets/{dataset}/queries/{query} [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L84) +### PATCH datasets/{dataset}/queries/{query} [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L88)
Details

@@ -175,7 +175,7 @@ Returns: [FullExecutionStatus](#Type-FullExecutionStatus)

-### DELETE datasets/{dataset}/queries/{query} [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L95) +### DELETE datasets/{dataset}/queries/{query} [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L99)
Details

@@ -188,7 +188,7 @@ Returns: `void`

-### POST datasets/{dataset}/queries/{query}/cancel [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L114) +### POST datasets/{dataset}/queries/{query}/cancel [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L118)
Details

@@ -201,7 +201,7 @@ Returns: `void`

-### POST datasets/{dataset}/queries/{query}/reexecute [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L104) +### POST datasets/{dataset}/queries/{query}/reexecute [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L108)
Details

From 8b0d745e4f335318ae7a787a67949134cff0ced9 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Tue, 27 Jul 2021 15:20:02 +0200 Subject: [PATCH 41/82] add check for all-Case --- .../concept/specific/external/CQExternal.java | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java index 6a8bf31859..9d7b73ef15 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java @@ -4,6 +4,7 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; import javax.validation.constraints.NotEmpty; @@ -17,7 +18,6 @@ import com.bakdata.conquery.models.config.FrontendConfig; import com.bakdata.conquery.models.error.ConqueryError; import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; - import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; import com.bakdata.conquery.models.query.queryplan.ConceptQueryPlan; @@ -75,14 +75,19 @@ private static Int2ObjectMap readDates(String[][] values, List List dateFormats = format.stream().map(queryUpload::resolveDateFormat).collect(Collectors.toList()); - //validate structures + final boolean isAll = dateFormats.stream().allMatch(Objects::isNull); for (int row = 1; row < values.length; row++) { try { + if (isAll) { + out.put(row, CDateSet.createFull()); + continue; + } + for (int col = 0; col < dateFormats.size(); col++) { final DateFormat dateFormat = dateFormats.get(col); - if(dateFormat == null){ + if (dateFormat == null) { continue; } @@ -107,7 +112,14 @@ private static Int2ObjectMap readDates(String[][] values, List @Override public void resolve(QueryResolveContext context) { - final ResolveStatistic resolved = resolveEntities(values, format, context.getNamespace().getStorage().getIdMapping(), context.getConfig().getFrontend().getQueryUpload(), context.getConfig().getPreprocessor().getParsers().getDateReader()); + final ResolveStatistic + resolved = + resolveEntities(values, format, context.getNamespace().getStorage().getIdMapping(), context.getConfig() + .getFrontend() + .getQueryUpload(), context.getConfig() + .getPreprocessor() + .getParsers() + .getDateReader()); if (resolved.getResolved().isEmpty()) { throw new ConqueryError.ExternalResolveEmptyError(); @@ -181,7 +193,7 @@ public static ResolveStatistic resolveEntities(@NotEmpty String[][] values, @Not resolved.put(resolvedId, rowDates.get(rowNum)); } - return new ResolveStatistic(resolved,unresolvedDate,unresolvedId); + return new ResolveStatistic(resolved, unresolvedDate, unresolvedId); } From 5e8970ddcea3c6e652afca725812739cb4503026 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Tue, 27 Jul 2021 16:03:13 +0200 Subject: [PATCH 42/82] adds documentation --- .../conquery/apiv1/QueryProcessor.java | 20 ++++--- .../concept/specific/external/CQExternal.java | 58 ++++++++++--------- .../concept/specific/external/DateFormat.java | 3 + .../models/config/FrontendConfig.java | 18 ++++-- 4 files changed, 60 insertions(+), 39 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java index 8e8258fe17..e82fd97885 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java @@ -372,14 +372,19 @@ public static class UploadResponse { private final List unreadableDate; } + /** + * Try to resolve the upload, if successful, create query for the user and return id and statistics for that. + */ public Response uploadEntities(User user, Dataset dataset, QueryResource.ExternalUpload upload) { - final CQExternal.ResolveStatistic statistic = CQExternal.resolveEntities(upload.getValues(), upload.getFormat(), - datasetRegistry.get(dataset.getId()).getStorage().getIdMapping(), - config.getFrontend().getQueryUpload(), - config.getPreprocessor().getParsers().getDateReader() - ); - // Resolving nothing is a problem and we fail. + final CQExternal.ResolveStatistic statistic = + CQExternal.resolveEntities(upload.getValues(), upload.getFormat(), + datasetRegistry.get(dataset.getId()).getStorage().getIdMapping(), + config.getFrontend().getQueryUpload(), + config.getPreprocessor().getParsers().getDateReader() + ); + + // Resolving nothing is a problem thus we fail. if (statistic.getResolved().isEmpty()) { return Response.status(Response.Status.BAD_REQUEST) .entity(new UploadResponse(null, 0, statistic.getUnresolvedId(), statistic.getUnreadableDate())) @@ -390,7 +395,8 @@ public Response uploadEntities(User user, Dataset dataset, QueryResource.Externa // We only create the Query, really no need to execute it as it's only useful for composition. final ManagedExecution execution = - datasetRegistry.get(dataset.getId()).getExecutionManager().createExecution(datasetRegistry, query, user, dataset); + datasetRegistry.get(dataset.getId()).getExecutionManager() + .createExecution(datasetRegistry, query, user, dataset); return Response.ok() .entity(new UploadResponse( diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java index 9d7b73ef15..e76f6d2390 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java @@ -70,37 +70,43 @@ public QPNode createQueryPlan(QueryPlanContext context, ConceptQueryPlan plan) { } + /** + * For each row try and collect all dates. + * + * @return Row -> Dates + */ private static Int2ObjectMap readDates(String[][] values, List format, DateReader dateReader, FrontendConfig.UploadConfig queryUpload) { Int2ObjectMap out = new Int2ObjectAVLTreeMap<>(); List dateFormats = format.stream().map(queryUpload::resolveDateFormat).collect(Collectors.toList()); - final boolean isAll = dateFormats.stream().allMatch(Objects::isNull); + // If no format provided, put all dates into output. + if (dateFormats.stream().allMatch(Objects::isNull)) { + for (int row = 0; row < values.length; row++) { + out.put(row, CDateSet.createFull()); + } + return out; + } for (int row = 1; row < values.length; row++) { try { - if (isAll) { - out.put(row, CDateSet.createFull()); - continue; - } + final CDateSet dates = CDateSet.create(); + // Collect all specified dates into a single set. for (int col = 0; col < dateFormats.size(); col++) { final DateFormat dateFormat = dateFormats.get(col); if (dateFormat == null) { continue; } - - CDateSet dates = CDateSet.create(); - dateFormat.readDates(values[row][col], dateReader, dates); + } - if (dates.isEmpty()) { - continue; - } - - out.put(row, dates); + if (dates.isEmpty()) { + continue; } + + out.put(row, dates); } catch (Exception e) { log.warn("Failed to parse Date from {}", row, e); @@ -112,14 +118,12 @@ private static Int2ObjectMap readDates(String[][] values, List @Override public void resolve(QueryResolveContext context) { - final ResolveStatistic - resolved = - resolveEntities(values, format, context.getNamespace().getStorage().getIdMapping(), context.getConfig() - .getFrontend() - .getQueryUpload(), context.getConfig() - .getPreprocessor() - .getParsers() - .getDateReader()); + final ResolveStatistic resolved = + resolveEntities(values, format, + context.getNamespace().getStorage().getIdMapping(), + context.getConfig().getFrontend().getQueryUpload(), + context.getConfig().getPreprocessor().getParsers().getDateReader() + ); if (resolved.getResolved().isEmpty()) { throw new ConqueryError.ExternalResolveEmptyError(); @@ -127,18 +131,16 @@ public void resolve(QueryResolveContext context) { if (!resolved.getUnreadableDate().isEmpty()) { log.warn( - "Could not read dates {} of the {} rows. Not resolved: {}", + "Could not read {} dates. Not resolved: {}", resolved.getUnreadableDate().size(), - values.length - 1, resolved.getUnreadableDate().subList(0, Math.min(resolved.getUnreadableDate().size(), 10)) ); } if (!resolved.getUnresolvedId().isEmpty()) { log.warn( - "Could not read dates {} of the {} rows. Not resolved: {}", + "Could not resolve {} ids. Not resolved: {}", resolved.getUnresolvedId().size(), - values.length - 1, resolved.getUnresolvedId().subList(0, Math.min(resolved.getUnresolvedId().size(), 10)) ); } @@ -157,6 +159,9 @@ public static class ResolveStatistic { } + /** + * Helper method to try and resolve entities in values using the specified format. + */ public static ResolveStatistic resolveEntities(@NotEmpty String[][] values, @NotEmpty List format, EntityIdMap mapping, FrontendConfig.UploadConfig queryUpload, @NotNull DateReader dateReader) { Map resolved = new Int2ObjectOpenHashMap<>(); @@ -164,14 +169,13 @@ public static ResolveStatistic resolveEntities(@NotEmpty String[][] values, @Not List unresolvedId = new ArrayList<>(); // extract dates from rows - final Int2ObjectMap rowDates = readDates(values, format, dateReader, queryUpload); + // TODO allow multiple ids final int idIndex = queryUpload.getIdIndex(format); final ColumnConfig reader = queryUpload.getIdMapper(format.get(idIndex)); - // ignore the first row, because this is the header for (int rowNum = 1; rowNum < values.length; rowNum++) { final String[] row = values[rowNum]; diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java index d1c2ed33c1..1ce4206ede 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java @@ -47,6 +47,9 @@ public void readDates(String value, DateReader dateReader, CDateSet out) { } }; + /** + * Try and parse value using dateReader, adding result into out set. + */ public abstract void readDates(String value, DateReader dateReader, CDateSet out); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java index fa053e326d..8a55a9b8ba 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java @@ -1,7 +1,6 @@ package com.bakdata.conquery.models.config; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -23,6 +22,7 @@ import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.util.VersionInfo; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.google.common.base.Functions; import groovy.transform.ToString; import io.dropwizard.validation.ValidationMethod; import lombok.AccessLevel; @@ -88,11 +88,16 @@ public List getPrintIdFields() { return idFieldsCached; } + @JsonIgnore + private Map idMappers; + public ColumnConfig getIdMapper(String name) { - return ids.stream() - .filter(mapper -> mapper.getName().equals(name)) //TODO use map - .findFirst() - .orElse(null); + if (idMappers == null) { + idMappers = ids.stream().filter(ColumnConfig::isResolvable) + .collect(Collectors.toMap(ColumnConfig::getName, Functions.identity())); + } + + return idMappers.get(name); } public int getIdIndex(List format) { @@ -198,6 +203,9 @@ public DateFormat resolveDateFormat(String handle) { } + /** + * Try to create a {@link FullIdPrinter} for user if they are allowed. If not allowed to read ids, they will receive ab pseudomized result instead. + */ public IdPrinter getIdPrinter(User owner, ManagedExecution execution, Namespace namespace) { if (owner.isPermitted(execution.getDataset(), Ability.PRESERVE_ID)) { From d80b2d1f0428dc60f6d8d81e155ee295fe552e8b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 27 Jul 2021 14:04:52 +0000 Subject: [PATCH 43/82] Update AutoDoc --- docs/Config JSON.md | 10 +++++----- docs/REST API JSONs.md | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/Config JSON.md b/docs/Config JSON.md index 00b1cc6f43..5ded2f198f 100644 --- a/docs/Config JSON.md +++ b/docs/Config JSON.md @@ -223,7 +223,7 @@ Supported Fields: | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L68) | storage | `@Valid @NotNull StoreFactory` | | | |

-### Type CurrencyConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L217) +### Type CurrencyConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L225)
Details

@@ -234,10 +234,10 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L222) | decimalScale | `int` | `2` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L221) | decimalSeparator | `String` | `","` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L219) | prefix | `String` | `"€"` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L220) | thousandSeparator | `String` | `"."` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L230) | decimalScale | `int` | `2` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L229) | decimalSeparator | `String` | `","` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L227) | prefix | `String` | `"€"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L228) | thousandSeparator | `String` | `"."` | | |

### Type FrontendConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L37) diff --git a/docs/REST API JSONs.md b/docs/REST API JSONs.md index 03b364470b..3620549724 100644 --- a/docs/REST API JSONs.md +++ b/docs/REST API JSONs.md @@ -755,7 +755,7 @@ Supported Fields: | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptResource.java#L68) | concepts | list of `String` | `null` | | |

-### Type CurrencyConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L217) +### Type CurrencyConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L225)
Details

@@ -766,10 +766,10 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L222) | decimalScale | `int` | `2` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L221) | decimalSeparator | `String` | `","` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L219) | prefix | `String` | `"€"` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L220) | thousandSeparator | `String` | `"."` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L230) | decimalScale | `int` | `2` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L229) | decimalSeparator | `String` | `","` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L227) | prefix | `String` | `"€"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L228) | thousandSeparator | `String` | `"."` | | |

### Type ExecutionStatus [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/ExecutionStatus.java#L18) From 580e5d3e1b27dadb80909f2154af5c96bc3f9534 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Wed, 28 Jul 2021 11:55:31 +0200 Subject: [PATCH 44/82] refactor PrimaryDictionary into it's own separate store #NeedsReimport --- .../bakdata/conquery/ConqueryConstants.java | 4 +- .../conquery/io/storage/NamespaceStorage.java | 36 +- .../io/storage/NamespacedStorage.java | 4 +- .../conquery/io/storage/StoreInfo.java | 125 +++--- .../xodus/stores/SerializingStore.java | 16 +- .../conquery/models/config/ColumnConfig.java | 16 +- .../models/config/FrontendConfig.java | 14 +- .../conquery/models/config/StoreFactory.java | 2 + .../models/config/XodusStoreFactory.java | 369 +++++++++--------- .../identifiable/mapping/EntityIdMap.java | 21 +- .../identifiable/mapping/FullIdPrinter.java | 8 +- .../conquery/models/jobs/ImportJob.java | 11 +- .../admin/rest/AdminDatasetProcessor.java | 1 + .../integration/IntegrationTests.java | 11 +- .../integration/common/LoadingUtil.java | 8 +- .../conquery/integration/json/FormTest.java | 6 +- .../conquery/integration/json/QueryTest.java | 2 +- .../identifiable/IdMapSerialisationTest.java | 2 +- .../util/NonPersistentStoreFactory.java | 213 +++++----- .../conquery/util/support/TestConquery.java | 4 +- 20 files changed, 468 insertions(+), 405 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/ConqueryConstants.java b/backend/src/main/java/com/bakdata/conquery/ConqueryConstants.java index cf44882817..c9d6c49611 100644 --- a/backend/src/main/java/com/bakdata/conquery/ConqueryConstants.java +++ b/backend/src/main/java/com/bakdata/conquery/ConqueryConstants.java @@ -30,12 +30,14 @@ public class ConqueryConstants { public static final ResultInfo FEATURE_DATE_RANGE_INFO = new LocalizedDefaultResultInfo((l) -> C10N.get(ResultHeadersC10n.class, l).featureDateRange(), ResultType.DateRangeT.INSTANCE); public static final ResultInfo OUTCOME_DATE_RANGE_INFO = new LocalizedDefaultResultInfo((l) -> C10N.get(ResultHeadersC10n.class, l).outcomeDateRange(), ResultType.DateRangeT.INSTANCE); + public static final String PRIMARY_DICTIONARY = "PRIMARY_DICTIONARY"; + public static class AuthenticationUtil { public static final String REALM_NAME = "CONQUERY"; } public static DictionaryId getPrimaryDictionary(Dataset dataset) { - return DictionaryId.Parser.INSTANCE.parse(Arrays.asList(dataset.getName(), "primary_dictionary")); + return DictionaryId.Parser.INSTANCE.parse(Arrays.asList(dataset.getName(), PRIMARY_DICTIONARY)); } } diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java index c85f6188dc..ee5bd5b426 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java @@ -5,9 +5,14 @@ import javax.validation.Validator; +import com.bakdata.conquery.ConqueryConstants; import com.bakdata.conquery.io.storage.xodus.stores.SingletonStore; import com.bakdata.conquery.models.datasets.concepts.StructureNode; import com.bakdata.conquery.models.config.StoreFactory; +import com.bakdata.conquery.models.dictionary.Dictionary; +import com.bakdata.conquery.models.dictionary.EncodedDictionary; +import com.bakdata.conquery.models.dictionary.MapDictionary; +import com.bakdata.conquery.models.events.stores.specific.string.StringTypeEncoded; import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; import com.bakdata.conquery.models.worker.SingletonNamespaceCollection; import com.bakdata.conquery.models.worker.WorkerToBucketsMap; @@ -27,6 +32,8 @@ public class NamespaceStorage extends NamespacedStorage { protected SingletonStore structure; protected SingletonStore workerToBuckets; + protected SingletonStore primaryDictionary; + @Getter private final boolean registerImports = true; @@ -36,13 +43,34 @@ public NamespaceStorage(Validator validator, StoreFactory storageFactory, String idMapping = storageFactory.createIdMappingStore(pathName); structure = storageFactory.createStructureStore(pathName, new SingletonNamespaceCollection(getCentralRegistry())); workerToBuckets = storageFactory.createWorkerToBucketsStore(pathName); - + primaryDictionary = storageFactory.createPrimaryDictionaryStore(pathName); decorateIdMapping(idMapping); } + public EncodedDictionary getPrimaryDictionary() { + return new EncodedDictionary(getPrimaryDictionaryRaw(), StringTypeEncoded.Encoding.UTF8); + } + + @NonNull + public Dictionary getPrimaryDictionaryRaw() { + final Dictionary dictionary = primaryDictionary.get(); + + if(dictionary == null){ + log.trace("No prior PrimaryDictionary, creating one"); + final MapDictionary newPrimary = new MapDictionary(getDataset(), ConqueryConstants.PRIMARY_DICTIONARY); + + primaryDictionary.update(newPrimary); + + return newPrimary; + } + + return dictionary; + } + + private void decorateIdMapping(SingletonStore idMapping) { - idMapping.onAdd(mapping -> mapping.setDictionary(getPrimaryDictionary())); + idMapping.onAdd(mapping -> mapping.setStorage(this)); } @@ -84,6 +112,10 @@ public EntityIdMap getIdMapping() { } + public void updatePrimaryDictionary(Dictionary dictionary){ + primaryDictionary.update(dictionary); + } + public void updateIdMapping(EntityIdMap idMapping) { this.idMapping.update(idMapping); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java index 0d7221b16f..6158a6791e 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java @@ -215,9 +215,7 @@ public void removeDictionary(DictionaryId id) { dictionaries.remove(id); } - public EncodedDictionary getPrimaryDictionary() { - return new EncodedDictionary(dictionaries.get(ConqueryConstants.getPrimaryDictionary(getDataset())), StringTypeEncoded.Encoding.UTF8); - } + public void addImport(Import imp) { imports.add(imp); diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/StoreInfo.java b/backend/src/main/java/com/bakdata/conquery/io/storage/StoreInfo.java index ea6440b9fa..1a97525fe1 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/StoreInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/StoreInfo.java @@ -6,12 +6,12 @@ import com.bakdata.conquery.models.auth.entities.Group; import com.bakdata.conquery.models.auth.entities.Role; import com.bakdata.conquery.models.auth.entities.User; -import com.bakdata.conquery.models.datasets.concepts.Concept; -import com.bakdata.conquery.models.datasets.concepts.StructureNode; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.datasets.Import; import com.bakdata.conquery.models.datasets.SecondaryIdDescription; import com.bakdata.conquery.models.datasets.Table; +import com.bakdata.conquery.models.datasets.concepts.Concept; +import com.bakdata.conquery.models.datasets.concepts.StructureNode; import com.bakdata.conquery.models.dictionary.Dictionary; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.events.CBlock; @@ -49,76 +49,77 @@ */ @RequiredArgsConstructor @Getter -@ToString(of={"name", "keyType", "valueType" }) +@ToString(of = {"name", "keyType", "valueType"}) public enum StoreInfo implements IStoreInfo { - DATASET(Dataset.class, Boolean.class), - ID_MAPPING(EntityIdMap.class, Boolean.class), - NAMESPACES(DatasetRegistry.class, Boolean.class), - SLAVE(ShardNodeInformation.class, Boolean.class), - DICTIONARIES(Dictionary.class, DictionaryId.class), - IMPORTS(Import.class, ImportId.class), - SECONDARY_IDS(SecondaryIdDescription.class, SecondaryIdDescriptionId.class), - TABLES(Table.class, TableId.class), - CONCEPTS(Concept.class, ConceptId.class), - BUCKETS(Bucket.class, BucketId.class), - C_BLOCKS(CBlock.class, CBlockId.class), - WORKER(WorkerInformation.class, Boolean.class), - EXECUTIONS(ManagedExecution.class, ManagedExecutionId.class), - AUTH_ROLE(Role.class, RoleId.class), - AUTH_USER(User.class, UserId.class), - AUTH_GROUP(Group.class, GroupId.class), - STRUCTURE(StructureNode[].class, Boolean.class), - FORM_CONFIG(FormConfig.class, FormConfigId.class), - WORKER_TO_BUCKETS(WorkerToBucketsMap.class, Boolean.class); + DATASET(Dataset.class, Boolean.class), + ID_MAPPING(EntityIdMap.class, Boolean.class), + NAMESPACES(DatasetRegistry.class, Boolean.class), + SLAVE(ShardNodeInformation.class, Boolean.class), + DICTIONARIES(Dictionary.class, DictionaryId.class), + IMPORTS(Import.class, ImportId.class), + SECONDARY_IDS(SecondaryIdDescription.class, SecondaryIdDescriptionId.class), + TABLES(Table.class, TableId.class), + CONCEPTS(Concept.class, ConceptId.class), + BUCKETS(Bucket.class, BucketId.class), + C_BLOCKS(CBlock.class, CBlockId.class), + WORKER(WorkerInformation.class, Boolean.class), + EXECUTIONS(ManagedExecution.class, ManagedExecutionId.class), + AUTH_ROLE(Role.class, RoleId.class), + AUTH_USER(User.class, UserId.class), + AUTH_GROUP(Group.class, GroupId.class), + STRUCTURE(StructureNode[].class, Boolean.class), + FORM_CONFIG(FormConfig.class, FormConfigId.class), + WORKER_TO_BUCKETS(WorkerToBucketsMap.class, Boolean.class), + PRIMARY_DICTIONARY(Dictionary.class, Boolean.class); - private final Class valueType; - private final Class keyType; + private final Class valueType; + private final Class keyType; - /** - * Store for identifiable values, with injectors. Store is also cached. - */ - public > DirectIdentifiableStore identifiable(Store, T> baseStore, CentralRegistry centralRegistry, Injectable... injectables) { + /** + * Store for identifiable values, with injectors. Store is also cached. + */ + public > DirectIdentifiableStore identifiable(Store, T> baseStore, CentralRegistry centralRegistry, Injectable... injectables) { - for (Injectable injectable : injectables) { - baseStore.inject(injectable); - } + for (Injectable injectable : injectables) { + baseStore.inject(injectable); + } - baseStore.inject(centralRegistry); + baseStore.inject(centralRegistry); - return new DirectIdentifiableStore<>(centralRegistry, baseStore); - } + return new DirectIdentifiableStore<>(centralRegistry, baseStore); + } - /** - * Store for identifiable values, without injectors. Store is also cached. - */ - public > DirectIdentifiableStore identifiable(Store, T> baseStore, CentralRegistry centralRegistry) { - return identifiable(baseStore, centralRegistry, new SingletonNamespaceCollection(centralRegistry)); - } + /** + * Store for identifiable values, without injectors. Store is also cached. + */ + public > DirectIdentifiableStore identifiable(Store, T> baseStore, CentralRegistry centralRegistry) { + return identifiable(baseStore, centralRegistry, new SingletonNamespaceCollection(centralRegistry)); + } - /** - * General Key-Value store with caching. - */ - public CachedStore cached(Store baseStore) { - return new CachedStore<>(baseStore); - } + /** + * General Key-Value store with caching. + */ + public CachedStore cached(Store baseStore) { + return new CachedStore<>(baseStore); + } - /** - * Identifiable store, that lazy registers items in the central registry. - */ - public > IdentifiableCachedStore identifiableCachedStore(Store baseStore, CentralRegistry centralRegistry) { - return new IdentifiableCachedStore(centralRegistry, baseStore); - } + /** + * Identifiable store, that lazy registers items in the central registry. + */ + public > IdentifiableCachedStore identifiableCachedStore(Store baseStore, CentralRegistry centralRegistry) { + return new IdentifiableCachedStore(centralRegistry, baseStore); + } - /** - * Store holding a single value. - */ - public SingletonStore singleton(Store baseStore, Injectable... injectables) { - return new SingletonStore<>(baseStore, injectables); - } + /** + * Store holding a single value. + */ + public SingletonStore singleton(Store baseStore, Injectable... injectables) { + return new SingletonStore<>(baseStore, injectables); + } - @Override - public String getName() { - return name(); - } + @Override + public String getName() { + return name(); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java index 2469a5d444..ebc9f4f560 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java @@ -114,21 +114,13 @@ public SerializingStore(XodusStoreFactory config, XodusStore store, Validator va this.objectMapper = objectMapper; - valueWriter = objectMapper - .writerFor(valueType) - .withView(InternalOnly.class); + valueWriter = objectMapper.writerFor(valueType); - valueReader = objectMapper - .readerFor(valueType) - .withView(InternalOnly.class); + valueReader = objectMapper.readerFor(valueType); - keyWriter = objectMapper - .writerFor(storeInfo.getKeyType()) - .withView(InternalOnly.class); + keyWriter = objectMapper.writerFor(storeInfo.getKeyType()); - keyReader = objectMapper - .readerFor(storeInfo.getKeyType()) - .withView(InternalOnly.class); + keyReader = objectMapper.readerFor(storeInfo.getKeyType()); removeUnreadablesFromUnderlyingStore = config.isRemoveUnreadableFromStore(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java index 707d267446..18c70fe764 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java @@ -29,7 +29,6 @@ @Getter public class ColumnConfig { - public EntityIdMap.ExternalId read(String value) { if (!isResolvable()) { return null; @@ -39,13 +38,13 @@ public EntityIdMap.ExternalId read(String value) { return null; } - if (getLength() == -1) { - return new EntityIdMap.ExternalId(new String[]{getName(), value}); + if (getLength() == -1 || getPad() == null) { + return new EntityIdMap.ExternalId(getName(), value); } String padded = StringUtils.leftPad(value, getLength(), getPad()); - return new EntityIdMap.ExternalId(new String[]{getName(), padded}); + return new EntityIdMap.ExternalId(getName(), padded); } @@ -57,11 +56,13 @@ public EntityIdMap.ExternalId read(String value) { * Map of Localized labels. */ @NotEmpty + @Builder.Default private Map label = Collections.emptyMap(); /** * Map of Localized description. */ + @Builder.Default private Map description = Collections.emptyMap(); @InternalOnly @@ -71,12 +72,19 @@ public EntityIdMap.ExternalId read(String value) { private String pad = null; @InternalOnly + @Builder.Default private int length = -1; @InternalOnly + @Builder.Default private boolean resolvable = false; @InternalOnly + @Builder.Default + private boolean print = true; + + @InternalOnly + @Builder.Default private boolean fillAnon = false; @JsonIgnore diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java index 8a55a9b8ba..157f0008aa 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java @@ -80,8 +80,8 @@ public static class UploadConfig { public List getPrintIdFields() { if (idFieldsCached == null) { idFieldsCached = ids.stream() + .filter(ColumnConfig::isPrint) .map(ColumnConfig::getField) - .filter(Objects::nonNull) .collect(Collectors.toUnmodifiableList()); } @@ -89,6 +89,7 @@ public List getPrintIdFields() { } @JsonIgnore + @Setter(AccessLevel.NONE) private Map idMappers; public ColumnConfig getIdMapper(String name) { @@ -207,17 +208,18 @@ public DateFormat resolveDateFormat(String handle) { * Try to create a {@link FullIdPrinter} for user if they are allowed. If not allowed to read ids, they will receive ab pseudomized result instead. */ public IdPrinter getIdPrinter(User owner, ManagedExecution execution, Namespace namespace) { - - if (owner.isPermitted(execution.getDataset(), Ability.PRESERVE_ID)) { - return new FullIdPrinter(namespace.getStorage().getPrimaryDictionary(), namespace.getStorage().getIdMapping()); - } - final int size = getPrintIdFields().size(); final int pos = IntStream.range(0, getIds().size()) .filter(idx -> getIds().get(idx).isFillAnon()) .findFirst() .orElseThrow(); + if (owner.isPermitted(execution.getDataset(), Ability.PRESERVE_ID)) { + return new FullIdPrinter(namespace.getStorage().getPrimaryDictionary(), namespace.getStorage().getIdMapping(), size, pos); + } + + + return new AutoIncrementingPseudomizer(size, pos); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java index 8170c77fdc..0c642243ab 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java @@ -66,4 +66,6 @@ public interface StoreFactory { IdentifiableStore createUserStore(CentralRegistry centralRegistry, String pathName); IdentifiableStore createRoleStore(CentralRegistry centralRegistry, String pathName); IdentifiableStore createGroupStore(CentralRegistry centralRegistry, String pathName); + + SingletonStore createPrimaryDictionaryStore(String pathName); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java index 8d823fdc62..5e9a3b7afa 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java @@ -24,6 +24,7 @@ import com.bakdata.conquery.commands.ManagerNode; import com.bakdata.conquery.commands.ShardNode; import com.bakdata.conquery.io.cps.CPSType; +import com.bakdata.conquery.io.jackson.InternalOnly; import com.bakdata.conquery.io.jackson.Jackson; import com.bakdata.conquery.io.storage.IdentifiableStore; import com.bakdata.conquery.io.storage.NamespaceStorage; @@ -40,12 +41,12 @@ import com.bakdata.conquery.models.auth.entities.Group; import com.bakdata.conquery.models.auth.entities.Role; import com.bakdata.conquery.models.auth.entities.User; -import com.bakdata.conquery.models.datasets.concepts.Concept; -import com.bakdata.conquery.models.datasets.concepts.StructureNode; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.datasets.Import; import com.bakdata.conquery.models.datasets.SecondaryIdDescription; import com.bakdata.conquery.models.datasets.Table; +import com.bakdata.conquery.models.datasets.concepts.Concept; +import com.bakdata.conquery.models.datasets.concepts.StructureNode; import com.bakdata.conquery.models.dictionary.Dictionary; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.events.CBlock; @@ -90,69 +91,73 @@ @CPSType(id = "XODUS", base = StoreFactory.class) public class XodusStoreFactory implements StoreFactory { - private Path directory = Path.of("storage"); + private Path directory = Path.of("storage"); - private boolean validateOnWrite = false; - @NotNull - @Valid - private XodusConfig xodus = new XodusConfig(); + private boolean validateOnWrite = false; + @NotNull + @Valid + private XodusConfig xodus = new XodusConfig(); - private boolean useWeakDictionaryCaching = true; - @NotNull - private Duration weakCacheDuration = Duration.hours(48); + private boolean useWeakDictionaryCaching = true; + @NotNull + private Duration weakCacheDuration = Duration.hours(48); - @Min(1) - private int nThreads = Runtime.getRuntime().availableProcessors(); + @Min(1) + private int nThreads = Runtime.getRuntime().availableProcessors(); - /** - * Flag for the {@link SerializingStore} whether to delete values from the underlying store, that cannot be mapped to an object anymore. - */ - private boolean removeUnreadableFromStore = false; + /** + * Flag for the {@link SerializingStore} whether to delete values from the underlying store, that cannot be mapped to an object anymore. + */ + private boolean removeUnreadableFromStore = false; - /** - * When set, all values that could not be deserialized from the persistent store, are dump into individual files. - */ - private Optional unreadableDataDumpDirectory = Optional.empty(); + /** + * When set, all values that could not be deserialized from the persistent store, are dump into individual files. + */ + private Optional unreadableDataDumpDirectory = Optional.empty(); - @JsonIgnore - private transient Validator validator; + @JsonIgnore + private transient Validator validator; - @JsonIgnore - private transient ObjectMapper objectMapper = Jackson.BINARY_MAPPER.copy(); + @JsonIgnore + private transient ObjectMapper objectMapper = Jackson.BINARY_MAPPER.copy(); - @JsonIgnore - private BiMap activeEnvironments = HashBiMap.create(); + @JsonIgnore + private BiMap activeEnvironments = HashBiMap.create(); - @JsonIgnore - private final transient Multimap openStoresInEnv = Multimaps.synchronizedSetMultimap(MultimapBuilder.hashKeys().hashSetValues().build()); + @JsonIgnore + private final transient Multimap + openStoresInEnv = + Multimaps.synchronizedSetMultimap(MultimapBuilder.hashKeys().hashSetValues().build()); - @Override - public void init(ManagerNode managerNode) { - validator = managerNode.getValidator(); - configureMapper(managerNode.getConfig()); - } + @Override + public void init(ManagerNode managerNode) { + validator = managerNode.getValidator(); + configureMapper(managerNode.getConfig()); + } - @Override - public void init(ShardNode shardNode) { - validator = shardNode.getValidator(); - configureMapper(shardNode.getConfig()); - } + @Override + public void init(ShardNode shardNode) { + validator = shardNode.getValidator(); + configureMapper(shardNode.getConfig()); + } - private void configureMapper(ConqueryConfig config) { - config.configureObjectMapper(objectMapper); - } + private void configureMapper(ConqueryConfig config) { + config.configureObjectMapper(objectMapper); + objectMapper.setConfig(objectMapper.getDeserializationConfig().withView(InternalOnly.class)); + objectMapper.setConfig(objectMapper.getSerializationConfig().withView(InternalOnly.class)); + } - @Override - @SneakyThrows - public Collection loadNamespaceStorages() { + @Override + @SneakyThrows + public Collection loadNamespaceStorages() { return loadNamespacedStores("dataset_", (elements) -> new NamespaceStorage(validator, this, elements)); - } + } - @Override - @SneakyThrows - public Collection loadWorkerStorages() { + @Override + @SneakyThrows + public Collection loadWorkerStorages() { return loadNamespacedStores("worker_", (elements) -> new WorkerStorage(validator, this, elements)); - } + } private Queue loadNamespacedStores(String prefix, Function creator) @@ -167,7 +172,7 @@ private Queue loadNamespacedStores(String prefi ExecutorService loaders = Executors.newFixedThreadPool(getNThreads()); - for (File directory : baseDir.listFiles((file, name) -> file.isDirectory() && name.startsWith(prefix))) { + for (File directory : baseDir.listFiles((file, name) -> file.isDirectory() && name.startsWith(prefix))) { final String name = directory.getName(); @@ -209,38 +214,38 @@ private Queue loadNamespacedStores(String prefi } private List getRelativePathElements(Path path) { - ArrayList list = new ArrayList<>(); - Path relative = getDirectory().relativize(path); - for (int i = 0; i < relative.getNameCount(); i++) { - list.add(relative.getName(i).toString()); - } - return list; - } - - private boolean environmentHasStores(File pathName) { - Environment env = findEnvironment(pathName); - boolean exists = env.computeInTransaction(t -> env.storeExists(StoreInfo.DATASET.getName(), t)); - env.computeInTransaction(t -> env.getAllStoreNames(t)); - if (!exists) { - closeEnvironment(env); - } - return exists; - } - - @Override - public SingletonStore createDatasetStore(String pathName) { - return DATASET.singleton(createStore(findEnvironment(pathName), validator, DATASET)); - } - - @Override - public IdentifiableStore createSecondaryIdDescriptionStore(CentralRegistry centralRegistry, String pathName) { - return SECONDARY_IDS.identifiable(createStore(findEnvironment(pathName), validator, SECONDARY_IDS), centralRegistry); - } - - @Override - public IdentifiableStore createTableStore(CentralRegistry centralRegistry, String pathName) { - return TABLES.identifiable(createStore(findEnvironment(pathName), validator, TABLES), centralRegistry); - } + ArrayList list = new ArrayList<>(); + Path relative = getDirectory().relativize(path); + for (int i = 0; i < relative.getNameCount(); i++) { + list.add(relative.getName(i).toString()); + } + return list; + } + + private boolean environmentHasStores(File pathName) { + Environment env = findEnvironment(pathName); + boolean exists = env.computeInTransaction(t -> env.storeExists(StoreInfo.DATASET.getName(), t)); + env.computeInTransaction(t -> env.getAllStoreNames(t)); + if (!exists) { + closeEnvironment(env); + } + return exists; + } + + @Override + public SingletonStore createDatasetStore(String pathName) { + return DATASET.singleton(createStore(findEnvironment(pathName), validator, DATASET)); + } + + @Override + public IdentifiableStore createSecondaryIdDescriptionStore(CentralRegistry centralRegistry, String pathName) { + return SECONDARY_IDS.identifiable(createStore(findEnvironment(pathName), validator, SECONDARY_IDS), centralRegistry); + } + + @Override + public IdentifiableStore
createTableStore(CentralRegistry centralRegistry, String pathName) { + return TABLES.identifiable(createStore(findEnvironment(pathName), validator, TABLES), centralRegistry); + } @Override public IdentifiableStore createDictionaryStore(CentralRegistry centralRegistry, String pathName) { @@ -251,7 +256,9 @@ public IdentifiableStore createDictionaryStore(CentralRegistry centr final BigStore, Dictionary> bigStore; synchronized (openStoresInEnv) { - bigStore = new BigStore<>(this, validator, environment, DICTIONARIES, openStoresInEnv.get(environment), this::closeEnvironment, this::removeEnvironment, namespaceCollection.injectInto(objectMapper)); + bigStore = + new BigStore<>(this, validator, environment, DICTIONARIES, openStoresInEnv.get(environment), this::closeEnvironment, this::removeEnvironment, namespaceCollection + .injectInto(objectMapper)); } final Store, Dictionary> result; @@ -266,30 +273,30 @@ public IdentifiableStore createDictionaryStore(CentralRegistry centr return DICTIONARIES.identifiableCachedStore(result, centralRegistry); } - @Override - public IdentifiableStore> createConceptStore(CentralRegistry centralRegistry, String pathName) { - return CONCEPTS.identifiable(createStore(findEnvironment(pathName), validator, CONCEPTS), centralRegistry); - } + @Override + public IdentifiableStore> createConceptStore(CentralRegistry centralRegistry, String pathName) { + return CONCEPTS.identifiable(createStore(findEnvironment(pathName), validator, CONCEPTS), centralRegistry); + } - @Override - public IdentifiableStore createImportStore(CentralRegistry centralRegistry, String pathName) { - return IMPORTS.identifiable(createStore(findEnvironment(pathName), validator, IMPORTS), centralRegistry); - } + @Override + public IdentifiableStore createImportStore(CentralRegistry centralRegistry, String pathName) { + return IMPORTS.identifiable(createStore(findEnvironment(pathName), validator, IMPORTS), centralRegistry); + } - @Override - public IdentifiableStore createCBlockStore(CentralRegistry centralRegistry, String pathName) { - return C_BLOCKS.identifiable(createStore(findEnvironment(pathName), validator, C_BLOCKS), centralRegistry); - } + @Override + public IdentifiableStore createCBlockStore(CentralRegistry centralRegistry, String pathName) { + return C_BLOCKS.identifiable(createStore(findEnvironment(pathName), validator, C_BLOCKS), centralRegistry); + } - @Override - public IdentifiableStore createBucketStore(CentralRegistry centralRegistry, String pathName) { - return BUCKETS.identifiable(createStore(findEnvironment(pathName), validator, BUCKETS), centralRegistry); - } + @Override + public IdentifiableStore createBucketStore(CentralRegistry centralRegistry, String pathName) { + return BUCKETS.identifiable(createStore(findEnvironment(pathName), validator, BUCKETS), centralRegistry); + } - @Override - public SingletonStore createWorkerInformationStore(String pathName) { - return WORKER.singleton(createStore(findEnvironment(pathName), validator, WORKER)); - } + @Override + public SingletonStore createWorkerInformationStore(String pathName) { + return WORKER.singleton(createStore(findEnvironment(pathName), validator, WORKER)); + } @Override public SingletonStore createIdMappingStore(String pathName) { @@ -303,43 +310,48 @@ public SingletonStore createIdMappingStore(String pathName) { } } - @Override - public SingletonStore createWorkerToBucketsStore(String pathName) { - return WORKER_TO_BUCKETS.singleton(createStore(findEnvironment(pathName), validator, WORKER_TO_BUCKETS)); - } + @Override + public SingletonStore createWorkerToBucketsStore(String pathName) { + return WORKER_TO_BUCKETS.singleton(createStore(findEnvironment(pathName), validator, WORKER_TO_BUCKETS)); + } + + @Override + public SingletonStore createStructureStore(String pathName, SingletonNamespaceCollection centralRegistry) { + return STRUCTURE.singleton(createStore(findEnvironment(pathName), validator, STRUCTURE), centralRegistry); + } - @Override - public SingletonStore createStructureStore(String pathName, SingletonNamespaceCollection centralRegistry) { - return STRUCTURE.singleton(createStore(findEnvironment(pathName), validator, STRUCTURE), centralRegistry); - } + @Override + public IdentifiableStore> createExecutionsStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName) { + return EXECUTIONS.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "executions")), validator, EXECUTIONS), centralRegistry, datasetRegistry); + } - @Override - public IdentifiableStore> createExecutionsStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName) { - return EXECUTIONS.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "executions")), validator, EXECUTIONS), centralRegistry, datasetRegistry); - } + @Override + public IdentifiableStore createFormConfigStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName) { + return FORM_CONFIG.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "formConfigs")), validator, FORM_CONFIG), centralRegistry, datasetRegistry); + } - @Override - public IdentifiableStore createFormConfigStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName) { - return FORM_CONFIG.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "formConfigs")), validator, FORM_CONFIG), centralRegistry, datasetRegistry); - } + @Override + public IdentifiableStore createUserStore(CentralRegistry centralRegistry, String pathName) { + return AUTH_USER.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "users")), validator, AUTH_USER), centralRegistry); + } - @Override - public IdentifiableStore createUserStore(CentralRegistry centralRegistry, String pathName) { - return AUTH_USER.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "users")), validator, AUTH_USER), centralRegistry); - } + @Override + public IdentifiableStore createRoleStore(CentralRegistry centralRegistry, String pathName) { + return AUTH_ROLE.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "roles")), validator, AUTH_ROLE), centralRegistry); + } - @Override - public IdentifiableStore createRoleStore(CentralRegistry centralRegistry, String pathName) { - return AUTH_ROLE.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "roles")), validator, AUTH_ROLE), centralRegistry); - } + @Override + public IdentifiableStore createGroupStore(CentralRegistry centralRegistry, String pathName) { + return AUTH_GROUP.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "groups")), validator, AUTH_GROUP), centralRegistry); + } - @Override - public IdentifiableStore createGroupStore(CentralRegistry centralRegistry, String pathName) { - return AUTH_GROUP.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "groups")), validator, AUTH_GROUP), centralRegistry); - } + @Override + public SingletonStore createPrimaryDictionaryStore(String pathName) { + return PRIMARY_DICTIONARY.singleton(createStore(findEnvironment(pathName), validator, PRIMARY_DICTIONARY)); + } - private File resolveSubDir(String... subdirs) { + private File resolveSubDir(String... subdirs) { Path current = getDirectory(); for (String dir : subdirs) { @@ -347,55 +359,56 @@ private File resolveSubDir(String... subdirs) { } return current.toFile(); - } + } - /** - * Returns this.directory if the list is empty. - */ + /** + * Returns this.directory if the list is empty. + */ @NonNull @JsonIgnore private File getStorageDir(String pathName) { return getDirectory().resolve(pathName).toFile(); - } - - private Environment findEnvironment(@NonNull File path) { - synchronized (activeEnvironments) { - return activeEnvironments.computeIfAbsent(path, (p) -> Environments.newInstance(path, getXodus().createConfig())); - } - } - - private Environment findEnvironment(String pathName) { - synchronized (activeEnvironments) { - File path = getStorageDir(pathName); - return activeEnvironments.computeIfAbsent(path, (p) -> Environments.newInstance(p, getXodus().createConfig())); - } - } - - private void closeEnvironment(Environment env) { - synchronized (activeEnvironments) { - if (env == null) { - return; - } - - if(activeEnvironments.remove(activeEnvironments.inverse().get(env)) == null){ - return; - } - env.close(); - } - } - - private void removeEnvironment(Environment env) { - log.info("Deleting Environment[{}]", env.getLocation()); - try { - FileUtil.deleteRecursive(Path.of(env.getLocation())); - }catch (IOException e) { - log.error("Cannot delete directory of removed Environment[{}]", env.getLocation(), log.isDebugEnabled()? e : null); - } - } - - public Store createStore(Environment environment, Validator validator, StoreInfo storeId) { - synchronized (openStoresInEnv) { - return new CachedStore<>( + } + + private Environment findEnvironment(@NonNull File path) { + synchronized (activeEnvironments) { + return activeEnvironments.computeIfAbsent(path, (p) -> Environments.newInstance(path, getXodus().createConfig())); + } + } + + private Environment findEnvironment(String pathName) { + synchronized (activeEnvironments) { + File path = getStorageDir(pathName); + return activeEnvironments.computeIfAbsent(path, (p) -> Environments.newInstance(p, getXodus().createConfig())); + } + } + + private void closeEnvironment(Environment env) { + synchronized (activeEnvironments) { + if (env == null) { + return; + } + + if (activeEnvironments.remove(activeEnvironments.inverse().get(env)) == null) { + return; + } + env.close(); + } + } + + private void removeEnvironment(Environment env) { + log.info("Deleting Environment[{}]", env.getLocation()); + try { + FileUtil.deleteRecursive(Path.of(env.getLocation())); + } + catch (IOException e) { + log.error("Cannot delete directory of removed Environment[{}]", env.getLocation(), log.isDebugEnabled() ? e : null); + } + } + + public Store createStore(Environment environment, Validator validator, StoreInfo storeId) { + synchronized (openStoresInEnv) { + return new CachedStore<>( new SerializingStore<>( this, new XodusStore(environment, storeId, openStoresInEnv.get(environment), this::closeEnvironment, this::removeEnvironment), @@ -403,7 +416,7 @@ public Store createStore(Environment environment, Valid storeId, objectMapper )); - } - } + } + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java index 77bcd6363d..6979661322 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java @@ -5,8 +5,10 @@ import java.util.List; import java.util.Map; +import com.bakdata.conquery.io.storage.NamespaceStorage; import com.bakdata.conquery.models.config.ColumnConfig; import com.bakdata.conquery.models.dictionary.EncodedDictionary; +import com.bakdata.conquery.models.worker.Namespace; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonValue; @@ -32,7 +34,7 @@ public class EntityIdMap { @Setter @JsonIgnore - private EncodedDictionary dictionary; + private NamespaceStorage storage; /** * The map from csv entity ids to external entity ids. @@ -65,12 +67,14 @@ public static EntityIdMap generateIdMapping(CsvParser parser, List final String otherId = record.getString(columnConfig.getField()); - idParts.add(otherId); - if (otherId == null) { continue; } + if(columnConfig.isPrint()) { + idParts.add(otherId); + } + if (!columnConfig.isResolvable()) { continue; } @@ -144,10 +148,13 @@ public int resolve(ExternalId key) { String value = external2Internal.get(key); if (value != null) { - return dictionary.getId(value); + return getStorage().getPrimaryDictionary().getId(value); } - return -1; + // Maybe we can find them directly in the dictionary? + final int id = getStorage().getPrimaryDictionary().getId(key.getId()); + + return id; } public void addOutputMapping(String csvEntityId, EntityPrintId externalEntityId) { @@ -171,8 +178,8 @@ public void addInputMapping(String csvEntityId, ExternalId externalEntityId) { @Data @RequiredArgsConstructor(onConstructor_ = @JsonCreator) public static class ExternalId { - @Getter - private final String[] parts; + private final String type; + private final String id; } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/FullIdPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/FullIdPrinter.java index db1c3ba8b5..81711a7fbf 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/FullIdPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/FullIdPrinter.java @@ -13,6 +13,9 @@ public class FullIdPrinter implements IdPrinter { private final EncodedDictionary dictionary; private final EntityIdMap idMapping; + private final int size; + private final int idPos; + @Override public EntityPrintId createId(EntityResult entityResult){ @@ -26,7 +29,10 @@ public EntityPrintId createId(EntityResult entityResult){ EntityPrintId externalEntityId = idMapping.toExternal(csvEntityId); if (externalEntityId == null) { - return EntityPrintId.from(csvEntityId); + final String[] parts = new String[size]; + parts[idPos] = csvEntityId; + + return EntityPrintId.from(parts); } return externalEntityId; diff --git a/backend/src/main/java/com/bakdata/conquery/models/jobs/ImportJob.java b/backend/src/main/java/com/bakdata/conquery/models/jobs/ImportJob.java index 4facb3c0ce..1f5f8fea75 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/jobs/ImportJob.java +++ b/backend/src/main/java/com/bakdata/conquery/models/jobs/ImportJob.java @@ -420,15 +420,8 @@ private Bucket selectBucket(Map localStarts, Map co.override(this)); + context.getTestInstance() + .filter(ConfigOverride.class::isInstance) + .map(ConfigOverride.class::cast) + .ifPresent(co -> co.override(this)); } } } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java b/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java index 4550f3fd44..964f470b14 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java @@ -57,10 +57,6 @@ @UtilityClass public class LoadingUtil { - public static void importPreviousQueries(StandaloneSupport support, RequiredData content) throws IOException { - importPreviousQueries(support, content, support.getTestUser()); - } - public static void importPreviousQueries(StandaloneSupport support, RequiredData content, User user) throws IOException { // Load previous query results if available int id = 1; @@ -72,7 +68,9 @@ public static void importPreviousQueries(StandaloneSupport support, RequiredData ConceptQuery q = new ConceptQuery(new CQExternal(Arrays.asList("ID", "DATE_SET"), data)); - ManagedExecution managed = support.getNamespace().getExecutionManager().createQuery(support.getNamespace().getNamespaces(),q, queryId, user, support.getNamespace().getDataset()); + ManagedExecution managed = support.getNamespace().getExecutionManager() + .createQuery(support.getNamespace().getNamespaces(),q, queryId, user, support.getNamespace().getDataset()); + user.addPermission(support.getMetaStorage(), ExecutionPermission.onInstance(AbilitySets.QUERY_CREATOR, managed.getId())); if (managed.getState() == ExecutionState.FAILED) { diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java b/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java index d35cc0f34b..80f1075c77 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.integration.json; +import static com.bakdata.conquery.integration.common.LoadingUtil.importIdMapping; import static com.bakdata.conquery.integration.common.LoadingUtil.importSecondaryIds; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.fail; @@ -93,6 +94,10 @@ public void importRequiredData(StandaloneSupport support) throws Exception { LoadingUtil.importTableContents(support, content.getTables()); support.waitUntilWorkDone(); + + importIdMapping(support, content); + support.waitUntilWorkDone(); + log.info("{} IMPORT TABLE CONTENTS", getLabel()); LoadingUtil.importPreviousQueries(support, content, support.getTestUser()); @@ -112,7 +117,6 @@ public void executeTest(StandaloneSupport support) throws Exception { .isEmpty(); - ManagedExecution managedForm = support.getNamespace().getExecutionManager().runQuery(namespaces, form, support.getTestUser(), support.getDataset(), support.getConfig()); managedForm.awaitDone(10, TimeUnit.MINUTES); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/QueryTest.java b/backend/src/test/java/com/bakdata/conquery/integration/json/QueryTest.java index f797d3324f..04167c3c6f 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/QueryTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/QueryTest.java @@ -63,7 +63,7 @@ public void importRequiredData(StandaloneSupport support) throws Exception { importIdMapping(support, content); support.waitUntilWorkDone(); - importPreviousQueries(support, content); + importPreviousQueries(support, content, support.getTestUser()); support.waitUntilWorkDone(); query = IntegrationUtils.parseQuery(support, rawQuery); diff --git a/backend/src/test/java/com/bakdata/conquery/models/identifiable/IdMapSerialisationTest.java b/backend/src/test/java/com/bakdata/conquery/models/identifiable/IdMapSerialisationTest.java index a92ce8f261..42f895727c 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/identifiable/IdMapSerialisationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/identifiable/IdMapSerialisationTest.java @@ -8,7 +8,7 @@ public class IdMapSerialisationTest { public static EntityIdMap createTestPersistentMap() { EntityIdMap entityIdMap = new EntityIdMap(); - entityIdMap.addInputMapping("test1", new EntityIdMap.ExternalId(new String[]{"a"})); + entityIdMap.addInputMapping("test1", new EntityIdMap.ExternalId("id", "a")); entityIdMap.addOutputMapping("test2", EntityPrintId.from("c")); diff --git a/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java b/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java index 4af353f08e..d2f06bee60 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java +++ b/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java @@ -14,13 +14,13 @@ import com.bakdata.conquery.models.auth.entities.Group; import com.bakdata.conquery.models.auth.entities.Role; import com.bakdata.conquery.models.auth.entities.User; -import com.bakdata.conquery.models.datasets.concepts.Concept; -import com.bakdata.conquery.models.datasets.concepts.StructureNode; import com.bakdata.conquery.models.config.StoreFactory; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.datasets.Import; import com.bakdata.conquery.models.datasets.SecondaryIdDescription; import com.bakdata.conquery.models.datasets.Table; +import com.bakdata.conquery.models.datasets.concepts.Concept; +import com.bakdata.conquery.models.datasets.concepts.StructureNode; import com.bakdata.conquery.models.dictionary.Dictionary; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.events.CBlock; @@ -36,107 +36,110 @@ @CPSType(id = "NON_PERSISTENT", base = StoreFactory.class) public class NonPersistentStoreFactory implements StoreFactory { - @Override - public Collection loadNamespaceStorages() { - - return Collections.emptyList(); - } - - @Override - public Collection loadWorkerStorages() { - - return Collections.emptyList(); - } - - @Override - public SingletonStore createDatasetStore(String pathName) { - return DATASET.singleton(new NonPersistentStore()); - } - - @Override - public IdentifiableStore createSecondaryIdDescriptionStore(CentralRegistry centralRegistry, String pathName) { - return SECONDARY_IDS.identifiable(new NonPersistentStore(), centralRegistry); - } - - @Override - public IdentifiableStore
createTableStore(CentralRegistry centralRegistry, String pathName) { - return TABLES.identifiable(new NonPersistentStore(), centralRegistry); - } - - @Override - public IdentifiableStore createDictionaryStore(CentralRegistry centralRegistry, String pathName) { - return DICTIONARIES.identifiable(new NonPersistentStore(), centralRegistry); - } - - @Override - public IdentifiableStore> createConceptStore(CentralRegistry centralRegistry, String pathName) { - return CONCEPTS.identifiable(new NonPersistentStore(), centralRegistry); - } - - @Override - public IdentifiableStore createImportStore(CentralRegistry centralRegistry, String pathName) { - return IMPORTS.identifiable(new NonPersistentStore(), centralRegistry); - } - - @Override - public IdentifiableStore createCBlockStore(CentralRegistry centralRegistry, String pathName) { - return C_BLOCKS.identifiable(new NonPersistentStore(), centralRegistry); - } - - @Override - public IdentifiableStore createBucketStore(CentralRegistry centralRegistry, String pathName) { - return BUCKETS.identifiable(new NonPersistentStore(), centralRegistry); - } - - @Override - public SingletonStore createWorkerInformationStore(String pathName) { - return WORKER.singleton(new NonPersistentStore()); - } - - @Override - public SingletonStore createIdMappingStore(String pathName) { - return ID_MAPPING.singleton(new NonPersistentStore()); - } - - @Override - public SingletonStore createWorkerToBucketsStore(String pathName) { - return WORKER_TO_BUCKETS.singleton(new NonPersistentStore()); - } - - @Override - public SingletonStore createStructureStore(String pathName, SingletonNamespaceCollection centralRegistry) { - return STRUCTURE.singleton(new NonPersistentStore(), centralRegistry); - } - - @Override - public IdentifiableStore> createExecutionsStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName) { - return EXECUTIONS.identifiable(new NonPersistentStore(), centralRegistry, datasetRegistry); - } - - @Override - public IdentifiableStore createFormConfigStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName) { - return FORM_CONFIG.identifiable(new NonPersistentStore(), centralRegistry, datasetRegistry); - } - - @Override - public IdentifiableStore createUserStore(CentralRegistry centralRegistry, String pathName) { - return AUTH_USER.identifiable(new NonPersistentStore(), centralRegistry); - } - - @Override - public IdentifiableStore createRoleStore(CentralRegistry centralRegistry, String pathName) { - return AUTH_ROLE.identifiable(new NonPersistentStore(), centralRegistry); - } - - @Override - public IdentifiableStore createGroupStore(CentralRegistry centralRegistry, String pathName) { - return AUTH_GROUP.identifiable(new NonPersistentStore(), centralRegistry); - } - - /** - * @implNote intended for Unit-tests - */ - public MetaStorage createMetaStorage(){ - return new MetaStorage(null, this, null); - } + @Override + public Collection loadNamespaceStorages() { + return Collections.emptyList(); + } + + @Override + public Collection loadWorkerStorages() { + return Collections.emptyList(); + } + + @Override + public SingletonStore createDatasetStore(String pathName) { + return DATASET.singleton(new NonPersistentStore()); + } + + @Override + public IdentifiableStore createSecondaryIdDescriptionStore(CentralRegistry centralRegistry, String pathName) { + return SECONDARY_IDS.identifiable(new NonPersistentStore(), centralRegistry); + } + + @Override + public IdentifiableStore
createTableStore(CentralRegistry centralRegistry, String pathName) { + return TABLES.identifiable(new NonPersistentStore(), centralRegistry); + } + + @Override + public IdentifiableStore createDictionaryStore(CentralRegistry centralRegistry, String pathName) { + return DICTIONARIES.identifiable(new NonPersistentStore(), centralRegistry); + } + + @Override + public IdentifiableStore> createConceptStore(CentralRegistry centralRegistry, String pathName) { + return CONCEPTS.identifiable(new NonPersistentStore(), centralRegistry); + } + + @Override + public IdentifiableStore createImportStore(CentralRegistry centralRegistry, String pathName) { + return IMPORTS.identifiable(new NonPersistentStore(), centralRegistry); + } + + @Override + public IdentifiableStore createCBlockStore(CentralRegistry centralRegistry, String pathName) { + return C_BLOCKS.identifiable(new NonPersistentStore(), centralRegistry); + } + + @Override + public IdentifiableStore createBucketStore(CentralRegistry centralRegistry, String pathName) { + return BUCKETS.identifiable(new NonPersistentStore(), centralRegistry); + } + + @Override + public SingletonStore createWorkerInformationStore(String pathName) { + return WORKER.singleton(new NonPersistentStore()); + } + + @Override + public SingletonStore createIdMappingStore(String pathName) { + return ID_MAPPING.singleton(new NonPersistentStore()); + } + + @Override + public SingletonStore createWorkerToBucketsStore(String pathName) { + return WORKER_TO_BUCKETS.singleton(new NonPersistentStore()); + } + + @Override + public SingletonStore createStructureStore(String pathName, SingletonNamespaceCollection centralRegistry) { + return STRUCTURE.singleton(new NonPersistentStore(), centralRegistry); + } + + @Override + public IdentifiableStore> createExecutionsStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName) { + return EXECUTIONS.identifiable(new NonPersistentStore(), centralRegistry, datasetRegistry); + } + + @Override + public IdentifiableStore createFormConfigStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName) { + return FORM_CONFIG.identifiable(new NonPersistentStore(), centralRegistry, datasetRegistry); + } + + @Override + public IdentifiableStore createUserStore(CentralRegistry centralRegistry, String pathName) { + return AUTH_USER.identifiable(new NonPersistentStore(), centralRegistry); + } + + @Override + public IdentifiableStore createRoleStore(CentralRegistry centralRegistry, String pathName) { + return AUTH_ROLE.identifiable(new NonPersistentStore(), centralRegistry); + } + + @Override + public IdentifiableStore createGroupStore(CentralRegistry centralRegistry, String pathName) { + return AUTH_GROUP.identifiable(new NonPersistentStore(), centralRegistry); + } + + @Override + public SingletonStore createPrimaryDictionaryStore(String pathName) { + return DICTIONARIES.singleton(new NonPersistentStore<>()); + } + + /** + * @implNote intended for Unit-tests + */ + public MetaStorage createMetaStorage() { + return new MetaStorage(null, this, null); + } } diff --git a/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java b/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java index 2ec0325784..0992a66660 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java +++ b/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java @@ -19,6 +19,8 @@ import com.bakdata.conquery.Conquery; import com.bakdata.conquery.commands.ShardNode; import com.bakdata.conquery.commands.StandaloneCommand; +import com.bakdata.conquery.integration.IntegrationTests; +import com.bakdata.conquery.integration.json.JsonIntegrationTest; import com.bakdata.conquery.io.jackson.Jackson; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.config.ConqueryConfig; @@ -137,7 +139,7 @@ private synchronized StandaloneSupport createSupport(DatasetId datasetId, String log.info("Reusing existing folder {} for Support", localTmpDir.getPath()); } - ConqueryConfig localCfg = Cloner.clone(config, Map.of(Validator.class, standaloneCommand.getManager().getEnvironment().getValidator()), Jackson.BINARY_MAPPER.copy()); + ConqueryConfig localCfg = Cloner.clone(config, Map.of(Validator.class, standaloneCommand.getManager().getEnvironment().getValidator()), IntegrationTests.MAPPER); StandaloneSupport support = new StandaloneSupport( From 7dae4c94155b4691f2d73b437e276d790221c790 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 28 Jul 2021 09:57:50 +0000 Subject: [PATCH 45/82] Update AutoDoc --- docs/Config JSON.md | 28 ++++++++++++++-------------- docs/REST API JSONs.md | 10 +++++----- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/docs/Config JSON.md b/docs/Config JSON.md index 5ded2f198f..9bc446139e 100644 --- a/docs/Config JSON.md +++ b/docs/Config JSON.md @@ -223,7 +223,7 @@ Supported Fields: | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L68) | storage | `@Valid @NotNull StoreFactory` | | | |

-### Type CurrencyConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L225) +### Type CurrencyConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L227)
Details

@@ -234,10 +234,10 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L230) | decimalScale | `int` | `2` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L229) | decimalSeparator | `String` | `","` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L227) | prefix | `String` | `"€"` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L228) | thousandSeparator | `String` | `"."` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L232) | decimalScale | `int` | `2` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L231) | decimalSeparator | `String` | `","` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L229) | prefix | `String` | `"€"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L230) | thousandSeparator | `String` | `"."` | | |

### Type FrontendConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L37) @@ -398,7 +398,7 @@ Supported Fields: | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusConfig.java#L51) | treeMaxPageSize | `int` or `null` | `null` | | |

-### Type XodusStoreFactory [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L83) +### Type XodusStoreFactory [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L84)
Details

@@ -409,12 +409,12 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L93) | directory | `Path` | `"file://./storage"` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L104) | nThreads | `@javax.validation.constraints.Min(1) int` | ␀ | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L107-L109) | removeUnreadableFromStore | `boolean` | `false` | | Flag for the {@link SerializingStore} whether to delete values from the underlying store, that cannot be mapped to an object anymore. | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L112-L114) | unreadableDataDumpDirectory | `Optional` | `null` | | When set, all values that could not be deserialized from the persistent store, are dump into individual files. | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L100) | useWeakDictionaryCaching | `boolean` | `true` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L95) | validateOnWrite | `boolean` | `false` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L101) | weakCacheDuration | `@NotNull Duration` | `"48 hours"` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L96) | xodus | [XodusConfig](#Type-XodusConfig) | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L94) | directory | `Path` | `"file://./storage"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L105) | nThreads | `@javax.validation.constraints.Min(1) int` | ␀ | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L108-L110) | removeUnreadableFromStore | `boolean` | `false` | | Flag for the {@link SerializingStore} whether to delete values from the underlying store, that cannot be mapped to an object anymore. | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L113-L115) | unreadableDataDumpDirectory | `Optional` | `null` | | When set, all values that could not be deserialized from the persistent store, are dump into individual files. | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L101) | useWeakDictionaryCaching | `boolean` | `true` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L96) | validateOnWrite | `boolean` | `false` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L102) | weakCacheDuration | `@NotNull Duration` | `"48 hours"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L97) | xodus | [XodusConfig](#Type-XodusConfig) | | | |

diff --git a/docs/REST API JSONs.md b/docs/REST API JSONs.md index 3620549724..f4c8e5be55 100644 --- a/docs/REST API JSONs.md +++ b/docs/REST API JSONs.md @@ -755,7 +755,7 @@ Supported Fields: | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptResource.java#L68) | concepts | list of `String` | `null` | | |

-### Type CurrencyConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L225) +### Type CurrencyConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L227)
Details

@@ -766,10 +766,10 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L230) | decimalScale | `int` | `2` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L229) | decimalSeparator | `String` | `","` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L227) | prefix | `String` | `"€"` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L228) | thousandSeparator | `String` | `"."` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L232) | decimalScale | `int` | `2` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L231) | decimalSeparator | `String` | `","` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L229) | prefix | `String` | `"€"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L230) | thousandSeparator | `String` | `"."` | | |

### Type ExecutionStatus [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/ExecutionStatus.java#L18) From b1a64448fddd36daa1ccef753e08d920b8b8586b Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Wed, 28 Jul 2021 12:22:46 +0200 Subject: [PATCH 46/82] don't use pretty print --- .../com/bakdata/conquery/models/config/FrontendConfig.java | 2 +- .../conquery/integration/json/AbstractQueryEngineTest.java | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java index 157f0008aa..0d5a0ff920 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java @@ -62,7 +62,7 @@ public static class UploadConfig { private List ids = List.of( ColumnConfig.builder() .name("ID") - .field("id") + .field("result") .resolvable(true) .fillAnon(true) .build() diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java b/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java index 1709155450..c8c29c8fbe 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java @@ -76,10 +76,12 @@ public void executeTest(StandaloneSupport standaloneSupport) throws IOException standaloneSupport.getClient() .target(HierarchyHelper.hierarchicalPath(standaloneSupport.defaultApiURIBuilder(), ResultCsvResource.class, "getAsCsv") .buildFromMap( - Map.of(DATASET, standaloneSupport.getDataset().getName(), + Map.of( + DATASET, standaloneSupport.getDataset().getName(), QUERY, execution.getId().toString() ) )) + .queryParam("pretty", false) .request(AdditionalMediaTypes.CSV) .acceptLanguage(Locale.ENGLISH) .get(); From cfe8eada0da0ece31b95791bdfe289ebd6692d1e Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Wed, 28 Jul 2021 12:55:55 +0200 Subject: [PATCH 47/82] cleanup Aggregator tests --- .../SIMPLE_VIRTUAL_CONCEPT_Query.test.json | 23 ++- .../aggregator/COUNT_AGGREGATOR/content.csv | 2 +- .../SIMPLE_VIRTUAL_CONCEPT_Query.test.json | 23 ++- .../COUNT_DISTINCT_AGGREGATOR/content.csv | 2 +- .../SIMPLE_VIRTUAL_CONCEPT_Query.test.json | 19 ++- .../DATE_DISTANCE.test.json | 8 +- .../DATE_DISTANCE2.test.json | 8 +- .../DURATION_SUM.test.json | 17 +-- ...T_DATE_AGGREGATOR_NO_RESTRICTION.test.json | 124 ++++++++-------- ...VENT_DATE_AGGREGATOR_RESTRICTION.test.json | 140 +++++++++--------- .../EXISTS_AGGREGATOR/NUMBER.test.json | 23 ++- .../AND.test.json | 38 ++--- .../EXISTS_AGGREGATOR_OR/NUMBER.test.json | 28 ++-- .../SIMPLE_VIRTUAL_CONCEPT_Query.test.json | 19 ++- .../aggregator/FIRST_AGGREGATOR/content.csv | 2 +- .../SIMPLE_VIRTUAL_CONCEPT_Query.test.json | 21 ++- .../aggregator/LAST_AGGREGATOR/content.csv | 2 +- .../SIMPLE_VIRTUAL_CONCEPT_Query.test.json | 19 ++- .../MULTI_SELECT_AGGREGATOR/content.csv | 2 +- .../SIMPLE_VIRTUAL_CONCEPT_Query.test.json | 23 ++- .../aggregator/PREFIX_AGGREGATOR/content.csv | 2 +- .../SIMPLE_VIRTUAL_CONCEPT_Query.test.json | 8 +- .../SIMPLE_VIRTUAL_CONCEPT_Query.test.json | 19 ++- .../aggregator/RANDOM_AGGREGATOR/content.csv | 2 +- .../SIMPLE_VIRTUAL_CONCEPT_Query.test.json | 21 ++- .../aggregator/SELECT_AGGREGATOR/content.csv | 2 +- .../SIMPLE_VIRTUAL_CONCEPT_Query.test.json | 21 ++- .../aggregator/SUM_AGGREGATOR/content.csv | 2 +- .../SIMPLE_VIRTUAL_CONCEPT_Query.test.json | 21 ++- .../SIMPLE_VIRTUAL_CONCEPT_Query.test.json | 19 ++- .../aggregator/VALUES_AGGREGATOR/content.csv | 2 +- 31 files changed, 322 insertions(+), 340 deletions(-) diff --git a/backend/src/test/resources/tests/aggregator/COUNT_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json b/backend/src/test/resources/tests/aggregator/COUNT_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json index 9d0260ffd1..dfb097e137 100644 --- a/backend/src/test/resources/tests/aggregator/COUNT_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json +++ b/backend/src/test/resources/tests/aggregator/COUNT_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json @@ -6,15 +6,14 @@ "type": "CONCEPT_QUERY", "root": { "ids": [ - "select" + "concept" ], "type": "CONCEPT", - "label": "select", "tables": [ { - "id": "select.connector", + "id": "concept.connector", "selects" : [ - "select.connector.count_genders" + "concept.connector.select" ] } ] @@ -22,21 +21,21 @@ }, "concepts": [ { - "label": "select", + "name": "concept", "type": "TREE", "connectors": [ { - "label": "connector", - "table": "table1", + "name": "connector", + "table": "table", "validityDates": { "label": "datum", - "column": "table1.datum" + "column": "table.datum" }, "selects": [ { - "name" : "count_genders", + "name" : "select", "type": "COUNT", - "column": "table1.geschlecht" + "column": "table.value" } ] } @@ -47,7 +46,7 @@ "tables": [ { "csv": "tests/aggregator/COUNT_AGGREGATOR/content.csv", - "name": "table1", + "name": "table", "primaryColumn": { "name": "pid", "type": "STRING" @@ -58,7 +57,7 @@ "type": "DATE" }, { - "name": "geschlecht", + "name": "value", "type": "STRING" } ] diff --git a/backend/src/test/resources/tests/aggregator/COUNT_AGGREGATOR/content.csv b/backend/src/test/resources/tests/aggregator/COUNT_AGGREGATOR/content.csv index 8175833be1..cadb60f0f0 100644 --- a/backend/src/test/resources/tests/aggregator/COUNT_AGGREGATOR/content.csv +++ b/backend/src/test/resources/tests/aggregator/COUNT_AGGREGATOR/content.csv @@ -1,4 +1,4 @@ -pid,datum,geschlecht +pid,datum,value 1,2012-01-01,"f" 1,2012-01-02,"f" 2,2010-07-15, diff --git a/backend/src/test/resources/tests/aggregator/COUNT_DISTINCT_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json b/backend/src/test/resources/tests/aggregator/COUNT_DISTINCT_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json index a773d677fa..65dc6cd7db 100644 --- a/backend/src/test/resources/tests/aggregator/COUNT_DISTINCT_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json +++ b/backend/src/test/resources/tests/aggregator/COUNT_DISTINCT_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json @@ -6,15 +6,14 @@ "type": "CONCEPT_QUERY", "root": { "ids": [ - "select" + "concept" ], "type": "CONCEPT", - "label": "select", "tables": [ { - "id": "select.connector", + "id": "concept.connector", "selects": [ - "select.connector.count_gender" + "concept.connector.select" ] } ] @@ -22,21 +21,21 @@ }, "concepts": [ { - "label": "select", + "name": "concept", "type": "TREE", "connectors": [ { - "label": "connector", - "table": "table1", + "name": "connector", + "table": "table", "validityDates": { "label": "datum", - "column": "table1.datum" + "column": "table.datum" }, "selects": { - "name": "count_gender", + "name": "select", "type": "COUNT", "distinct": true, - "column": "table1.geschlecht" + "column": "table.value" } } ] @@ -46,7 +45,7 @@ "tables": [ { "csv": "tests/aggregator/COUNT_DISTINCT_AGGREGATOR/content.csv", - "name": "table1", + "name": "table", "primaryColumn": { "name": "pid", "type": "STRING" @@ -57,7 +56,7 @@ "type": "DATE" }, { - "name": "geschlecht", + "name": "value", "type": "STRING" } ] diff --git a/backend/src/test/resources/tests/aggregator/COUNT_DISTINCT_AGGREGATOR/content.csv b/backend/src/test/resources/tests/aggregator/COUNT_DISTINCT_AGGREGATOR/content.csv index abe6693cf8..e1a6e6bd22 100644 --- a/backend/src/test/resources/tests/aggregator/COUNT_DISTINCT_AGGREGATOR/content.csv +++ b/backend/src/test/resources/tests/aggregator/COUNT_DISTINCT_AGGREGATOR/content.csv @@ -1,4 +1,4 @@ -pid,datum,geschlecht +pid,datum,value 1,2012-01-01,"f" 1,2012-01-02,"f" 2,2010-07-15, diff --git a/backend/src/test/resources/tests/aggregator/COUNT_QUARTERS_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json b/backend/src/test/resources/tests/aggregator/COUNT_QUARTERS_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json index 08b28f9a02..d0281c15a4 100644 --- a/backend/src/test/resources/tests/aggregator/COUNT_QUARTERS_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json +++ b/backend/src/test/resources/tests/aggregator/COUNT_QUARTERS_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json @@ -6,15 +6,14 @@ "type": "CONCEPT_QUERY", "root": { "ids": [ - "select" + "concept" ], "type": "CONCEPT", - "label": "select", "tables": [ { - "id": "select.connector", + "id": "concept.connector", "selects": [ - "select.connector.count_quarters" + "concept.connector.select" ] } ] @@ -23,20 +22,20 @@ }, "concepts": [ { - "label": "select", + "name": "concept", "type": "TREE", "connectors": [ { "label": "connector", - "table": "table1", + "table": "table", "validityDates": { "label": "datum", - "column": "table1.datum" + "column": "table.datum" }, "selects": { - "name" : "count_quarters", + "name" : "select", "type": "COUNT_QUARTERS", - "column": "table1.behandlungsdatum" + "column": "table.behandlungsdatum" } } ] @@ -46,7 +45,7 @@ "tables": [ { "csv": "tests/aggregator/COUNT_QUARTERS_AGGREGATOR/content.csv", - "name": "table1", + "name": "table", "primaryColumn": { "name": "pid", "type": "STRING" diff --git a/backend/src/test/resources/tests/aggregator/DATE_DISTANCE_AGGREGATOR/DATE_DISTANCE.test.json b/backend/src/test/resources/tests/aggregator/DATE_DISTANCE_AGGREGATOR/DATE_DISTANCE.test.json index 6a9d175bb9..9afdc03bd2 100644 --- a/backend/src/test/resources/tests/aggregator/DATE_DISTANCE_AGGREGATOR/DATE_DISTANCE.test.json +++ b/backend/src/test/resources/tests/aggregator/DATE_DISTANCE_AGGREGATOR/DATE_DISTANCE.test.json @@ -39,15 +39,15 @@ "connectors": [ { "label": "geschlecht_connector", - "table": "table1", + "table": "table", "validityDates": { "label": "indexdatum", - "column": "table1.indexdatum" + "column": "table.indexdatum" }, "selects": { "name": "geb", "type": "DATE_DISTANCE", - "column": "table1.geburtsdatum" + "column": "table.geburtsdatum" } } ] @@ -57,7 +57,7 @@ "tables": [ { "csv": "tests/aggregator/DATE_DISTANCE_AGGREGATOR/content.csv", - "name": "table1", + "name": "table", "primaryColumn": { "name": "pid", "type": "STRING" diff --git a/backend/src/test/resources/tests/aggregator/DATE_DISTANCE_AGGREGATOR/DATE_DISTANCE2.test.json b/backend/src/test/resources/tests/aggregator/DATE_DISTANCE_AGGREGATOR/DATE_DISTANCE2.test.json index dd6698195a..f92e2e31f4 100644 --- a/backend/src/test/resources/tests/aggregator/DATE_DISTANCE_AGGREGATOR/DATE_DISTANCE2.test.json +++ b/backend/src/test/resources/tests/aggregator/DATE_DISTANCE_AGGREGATOR/DATE_DISTANCE2.test.json @@ -39,15 +39,15 @@ "connectors": [ { "label": "geschlecht_connector", - "table": "table1", + "table": "table", "validityDates": { "label": "indexdatum", - "column": "table1.indexdatum" + "column": "table.indexdatum" }, "selects": { "name": "geb", "type": "DATE_DISTANCE", - "column": "table1.geburtsdatum" + "column": "table.geburtsdatum" } } ] @@ -57,7 +57,7 @@ "tables": [ { "csv": "tests/aggregator/DATE_DISTANCE_AGGREGATOR/content.csv", - "name": "table1", + "name": "table", "primaryColumn": { "name": "pid", "type": "STRING" diff --git a/backend/src/test/resources/tests/aggregator/DURATION_SUM_AGGREGATOR/DURATION_SUM.test.json b/backend/src/test/resources/tests/aggregator/DURATION_SUM_AGGREGATOR/DURATION_SUM.test.json index ca32df7104..18bcfe481e 100644 --- a/backend/src/test/resources/tests/aggregator/DURATION_SUM_AGGREGATOR/DURATION_SUM.test.json +++ b/backend/src/test/resources/tests/aggregator/DURATION_SUM_AGGREGATOR/DURATION_SUM.test.json @@ -6,15 +6,14 @@ "type": "CONCEPT_QUERY", "root": { "ids": [ - "select" + "concept" ], "type": "CONCEPT", - "label": "select", "tables": [ { - "id": "select.connector", + "id": "concept.connector", "selects": [ - "select.connector.select" + "concept.connector.select" ] } ] @@ -22,20 +21,20 @@ }, "concepts": [ { - "label": "select", + "name": "concept", "type": "TREE", "connectors": [ { "label": "connector", - "table": "table1", + "table": "table", "validityDates": { "label": "datum", - "column": "table1.datum" + "column": "table.datum" }, "selects": { "type": "DURATION_SUM", "name" : "select", - "column": "table1.datum" + "column": "table.datum" } } ] @@ -45,7 +44,7 @@ "tables": [ { "csv": "tests/aggregator/DURATION_SUM_AGGREGATOR/content.csv", - "name": "table1", + "name": "table", "primaryColumn": { "name": "pid", "type": "STRING" diff --git a/backend/src/test/resources/tests/aggregator/EVENT_DATE_AGGREGATOR/EVENT_DATE_AGGREGATOR_NO_RESTRICTION.test.json b/backend/src/test/resources/tests/aggregator/EVENT_DATE_AGGREGATOR/EVENT_DATE_AGGREGATOR_NO_RESTRICTION.test.json index 9c620d7d7d..d12d5cc0d5 100644 --- a/backend/src/test/resources/tests/aggregator/EVENT_DATE_AGGREGATOR/EVENT_DATE_AGGREGATOR_NO_RESTRICTION.test.json +++ b/backend/src/test/resources/tests/aggregator/EVENT_DATE_AGGREGATOR/EVENT_DATE_AGGREGATOR_NO_RESTRICTION.test.json @@ -1,69 +1,67 @@ { - "type": "QUERY_TEST", - "label": "EVENT_DATE_AGGREGATOR_NO_RESTRICTION Test", - "expectedCsv": "tests/aggregator/EVENT_DATE_AGGREGATOR/expected_no_restriction.csv", - "query": { - "type": "CONCEPT_QUERY", - "root": { - "ids": [ - "con" - ], - "type": "CONCEPT", - "label": "Geschlecht SELECT", - "tables": [ - { - "id": "con.geschlecht_connector", - "selects": "con.geschlecht_connector.event-date" - } - ], - "selects": "con.event-date" + "type": "QUERY_TEST", + "label": "EVENT_DATE_AGGREGATOR_NO_RESTRICTION Test", + "expectedCsv": "tests/aggregator/EVENT_DATE_AGGREGATOR/expected_no_restriction.csv", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "ids": [ + "concept" + ], + "type": "CONCEPT", + "tables": [ + { + "id": "concept.connector", + "selects": "concept.connector.event-date" } - }, - "concepts": [ + ], + "selects": "concept.event-date" + } + }, + "concepts": [ + { + "name": "concept", + "type": "TREE", + "connectors": [ { - "label": "Geschlecht SELECT", - "name": "con", - "type": "TREE", - "connectors": [ - { - "label": "geschlecht_connector", - "table": "table1", - "validityDates": { - "label": "indexdatum", - "column": "table1.indexdatum" - }, - "selects": { - "type": "EVENT_DATE_UNION", - "name": "event-date" - } - } - ], - "selects": { - "type": "EVENT_DATE_UNION", - "name": "event-date" - } + "label": "connector", + "table": "table", + "validityDates": { + "label": "indexdatum", + "column": "table.indexdatum" + }, + "selects": { + "type": "EVENT_DATE_UNION", + "name": "event-date" + } } - ], - "content": { - "tables": [ - { - "csv": "tests/aggregator/EVENT_DATE_AGGREGATOR/content.csv", - "name": "table1", - "primaryColumn": { - "name": "pid", - "type": "STRING" - }, - "columns": [ - { - "name": "indexdatum", - "type": "DATE_RANGE" - }, - { - "name": "geburtsdatum", - "type": "DATE" - } - ] - } - ] + ], + "selects": { + "type": "EVENT_DATE_UNION", + "name": "event-date" + } } + ], + "content": { + "tables": [ + { + "csv": "tests/aggregator/EVENT_DATE_AGGREGATOR/content.csv", + "name": "table", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "indexdatum", + "type": "DATE_RANGE" + }, + { + "name": "geburtsdatum", + "type": "DATE" + } + ] + } + ] + } } \ No newline at end of file diff --git a/backend/src/test/resources/tests/aggregator/EVENT_DATE_AGGREGATOR/EVENT_DATE_AGGREGATOR_RESTRICTION.test.json b/backend/src/test/resources/tests/aggregator/EVENT_DATE_AGGREGATOR/EVENT_DATE_AGGREGATOR_RESTRICTION.test.json index 02730a7965..9924999ef5 100644 --- a/backend/src/test/resources/tests/aggregator/EVENT_DATE_AGGREGATOR/EVENT_DATE_AGGREGATOR_RESTRICTION.test.json +++ b/backend/src/test/resources/tests/aggregator/EVENT_DATE_AGGREGATOR/EVENT_DATE_AGGREGATOR_RESTRICTION.test.json @@ -1,76 +1,74 @@ { - "type": "QUERY_TEST", - "label": "EVENT_DATE_AGGREGATOR_RESTRICTION Test", - "expectedCsv": "tests/aggregator/EVENT_DATE_AGGREGATOR/expected_restriction.csv", - "query": { - "type": "CONCEPT_QUERY", - "root": { - "type": "DATE_RESTRICTION", - "dateRange": { - "min": "2010-02-01", - "max": "2010-02-28" - }, - "child": { - "ids": [ - "con" - ], - "type": "CONCEPT", - "label": "Geschlecht SELECT", - "tables": [ - { - "id": "con.geschlecht_connector", - "selects": "con.geschlecht_connector.event-date" - } - ], - "selects": "con.event-date" - } - } - }, - "concepts": [ + "type": "QUERY_TEST", + "label": "EVENT_DATE_AGGREGATOR_RESTRICTION Test", + "expectedCsv": "tests/aggregator/EVENT_DATE_AGGREGATOR/expected_restriction.csv", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "type": "DATE_RESTRICTION", + "dateRange": { + "min": "2010-02-01", + "max": "2010-02-28" + }, + "child": { + "ids": [ + "concept" + ], + "type": "CONCEPT", + "tables": [ + { + "id": "concept.connector", + "selects": "concept.connector.event-date" + } + ], + "selects": "concept.event-date" + } + } + }, + "concepts": [ + { + "name": "concept", + "type": "TREE", + "connectors": [ { - "label": "Geschlecht SELECT", - "name": "con", - "type": "TREE", - "connectors": [ - { - "label": "geschlecht_connector", - "table": "table1", - "validityDates": { - "label": "indexdatum", - "column": "table1.indexdatum" - }, - "selects": { - "type": "EVENT_DATE_UNION", - "name": "event-date" - } - } - ], - "selects": { - "type": "EVENT_DATE_UNION", - "name": "event-date" - } + "label": "connector", + "table": "table", + "validityDates": { + "label": "indexdatum", + "column": "table.indexdatum" + }, + "selects": { + "type": "EVENT_DATE_UNION", + "name": "event-date" + } } - ], - "content": { - "tables": [ - { - "csv": "tests/aggregator/EVENT_DATE_AGGREGATOR/content.csv", - "name": "table1", - "primaryColumn": { - "name": "pid", - "type": "STRING" - }, - "columns": [ - { - "name": "indexdatum", - "type": "DATE_RANGE" - }, - { - "name": "geburtsdatum", - "type": "DATE" - } - ] - } - ] + ], + "selects": { + "type": "EVENT_DATE_UNION", + "name": "event-date" + } } + ], + "content": { + "tables": [ + { + "csv": "tests/aggregator/EVENT_DATE_AGGREGATOR/content.csv", + "name": "table", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "indexdatum", + "type": "DATE_RANGE" + }, + { + "name": "geburtsdatum", + "type": "DATE" + } + ] + } + ] + } } \ No newline at end of file diff --git a/backend/src/test/resources/tests/aggregator/EXISTS_AGGREGATOR/NUMBER.test.json b/backend/src/test/resources/tests/aggregator/EXISTS_AGGREGATOR/NUMBER.test.json index cc7e94680d..be8fc8abba 100644 --- a/backend/src/test/resources/tests/aggregator/EXISTS_AGGREGATOR/NUMBER.test.json +++ b/backend/src/test/resources/tests/aggregator/EXISTS_AGGREGATOR/NUMBER.test.json @@ -6,15 +6,15 @@ "type": "CONCEPT_QUERY", "root": { "ids": [ - "number" + "concept" ], "type": "CONCEPT", "tables": [ { - "id": "number.number_connector", + "id": "concept.connector", "filters": [ { - "filter": "number.number_connector.value", + "filter": "concept.connector.filter", "type": "REAL_RANGE", "value": { "min": 1, @@ -24,12 +24,12 @@ ] } ], - "selects": "number.exists" + "selects": "concept.exists" } }, "concepts": [ { - "label": "number", + "name": "concept", "type": "TREE", "selects": { "type": "EXISTS", @@ -37,16 +37,15 @@ }, "connectors": [ { - "label": "number_connector", - "table": "table1", + "label": "connector", + "table": "table", "validityDates": { "label": "datum", - "column": "table1.datum" + "column": "table.datum" }, "filters": { - "label": "value", - "description": "xy", - "column": "table1.value", + "label": "filter", + "column": "table.value", "type": "NUMBER" } } @@ -57,7 +56,7 @@ "tables": [ { "csv": "tests/aggregator/EXISTS_AGGREGATOR/content.csv", - "name": "table1", + "name": "table", "primaryColumn": { "name": "pid", "type": "STRING" diff --git a/backend/src/test/resources/tests/aggregator/EXISTS_AGGREGATOR_AGGREGATED/AND.test.json b/backend/src/test/resources/tests/aggregator/EXISTS_AGGREGATOR_AGGREGATED/AND.test.json index 08408751df..e64e06c75d 100644 --- a/backend/src/test/resources/tests/aggregator/EXISTS_AGGREGATOR_AGGREGATED/AND.test.json +++ b/backend/src/test/resources/tests/aggregator/EXISTS_AGGREGATOR_AGGREGATED/AND.test.json @@ -10,32 +10,32 @@ "type": "CONCEPT_QUERY", "root": { "type": "AND", - "label" : "AND", + "label": "AND", "createExists": true, "children": [ { "ids": [ - "number.1" + "concept.1" ], "type": "CONCEPT", "tables": [ { - "id": "number.number_connector" + "id": "concept.connector" } ], - "selects": "number.exists" + "selects": "concept.exists" }, { "ids": [ - "number.2" + "concept.2" ], "type": "CONCEPT", "tables": [ { - "id": "number.number_connector" + "id": "concept.connector" } ], - "selects": "number.exists" + "selects": "concept.exists" } ] } @@ -44,32 +44,32 @@ "type": "CONCEPT_QUERY", "root": { "type": "OR", - "label" : "OR", + "label": "OR", "createExists": true, "children": [ { "ids": [ - "number.1" + "concept.1" ], "type": "CONCEPT", "tables": [ { - "id": "number.number_connector" + "id": "concept.connector" } ], - "selects": "number.exists" + "selects": "concept.exists" }, { "ids": [ - "number.2" + "concept.2" ], "type": "CONCEPT", "tables": [ { - "id": "number.number_connector" + "id": "concept.connector" } ], - "selects": "number.exists" + "selects": "concept.exists" } ] } @@ -78,7 +78,7 @@ }, "concepts": [ { - "label": "number", + "label": "concept", "type": "TREE", "selects": { "type": "EXISTS", @@ -86,11 +86,11 @@ }, "connectors": [ { - "column": "table1.value", - "label": "number_connector", + "column": "table.value", + "label": "connector", "validityDates": { "label": "datum", - "column": "table1.datum" + "column": "table.datum" } } ], @@ -116,7 +116,7 @@ "tables": [ { "csv": "tests/aggregator/EXISTS_AGGREGATOR_AGGREGATED/content.csv", - "name": "table1", + "name": "table", "primaryColumn": { "name": "pid", "type": "STRING" diff --git a/backend/src/test/resources/tests/aggregator/EXISTS_AGGREGATOR_OR/NUMBER.test.json b/backend/src/test/resources/tests/aggregator/EXISTS_AGGREGATOR_OR/NUMBER.test.json index 2008269e54..9dbabdbbb3 100644 --- a/backend/src/test/resources/tests/aggregator/EXISTS_AGGREGATOR_OR/NUMBER.test.json +++ b/backend/src/test/resources/tests/aggregator/EXISTS_AGGREGATOR_OR/NUMBER.test.json @@ -9,15 +9,15 @@ "children": [ { "ids": [ - "number" + "concept" ], "type": "CONCEPT", "tables": [ { - "id": "number.number_connector", + "id": "concept.connector", "filters": [ { - "filter": "number.number_connector.value", + "filter": "concept.connector.value", "type": "REAL_RANGE", "value": { "min": 1, @@ -27,19 +27,19 @@ ] } ], - "selects": "number.exists" + "selects": "concept.exists" }, { "ids": [ - "number" + "concept" ], "type": "CONCEPT", "tables": [ { - "id": "number.number_connector", + "id": "concept.connector", "filters": [ { - "filter": "number.number_connector.value", + "filter": "concept.connector.value", "type": "REAL_RANGE", "value": { "min": 2, @@ -49,14 +49,14 @@ ] } ], - "selects": "number.exists" + "selects": "concept.exists" } ] } }, "concepts": [ { - "label": "number", + "label": "concept", "type": "TREE", "selects": { "type": "EXISTS", @@ -64,16 +64,16 @@ }, "connectors": [ { - "label": "number_connector", - "table": "table1", + "label": "connector", + "table": "table", "validityDates": { "label": "datum", - "column": "table1.datum" + "column": "table.datum" }, "filters": { "label": "value", "description": "xy", - "column": "table1.value", + "column": "table.value", "type": "NUMBER" } } @@ -84,7 +84,7 @@ "tables": [ { "csv": "tests/aggregator/EXISTS_AGGREGATOR_OR/content.csv", - "name": "table1", + "name": "table", "primaryColumn": { "name": "pid", "type": "STRING" diff --git a/backend/src/test/resources/tests/aggregator/FIRST_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json b/backend/src/test/resources/tests/aggregator/FIRST_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json index 9bbbf88030..f62636d356 100644 --- a/backend/src/test/resources/tests/aggregator/FIRST_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json +++ b/backend/src/test/resources/tests/aggregator/FIRST_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json @@ -6,15 +6,14 @@ "type": "CONCEPT_QUERY", "root": { "ids": [ - "select" + "concept" ], "type": "CONCEPT", - "label": "select", "tables": [ { - "id": "select.connector", + "id": "concept.connector", "selects": [ - "select.connector.select" + "concept.connector.select" ] } ] @@ -22,20 +21,20 @@ }, "concepts": [ { - "label": "SELECT", + "label": "concept", "type": "TREE", "connectors": [ { "label": "connector", - "table": "table1", + "table": "table", "validityDates": { "label": "datum", - "column": "table1.datum" + "column": "table.datum" }, "selects": { "name" : "select", "type": "FIRST", - "column": "table1.geschlecht" + "column": "table.value" } } ] @@ -45,7 +44,7 @@ "tables": [ { "csv": "tests/aggregator/FIRST_AGGREGATOR/content.csv", - "name": "table1", + "name": "table", "primaryColumn": { "name": "pid", "type": "STRING" @@ -56,7 +55,7 @@ "type": "DATE" }, { - "name": "geschlecht", + "name": "value", "type": "STRING" } ] diff --git a/backend/src/test/resources/tests/aggregator/FIRST_AGGREGATOR/content.csv b/backend/src/test/resources/tests/aggregator/FIRST_AGGREGATOR/content.csv index 079936318d..4ac80c5fe1 100644 --- a/backend/src/test/resources/tests/aggregator/FIRST_AGGREGATOR/content.csv +++ b/backend/src/test/resources/tests/aggregator/FIRST_AGGREGATOR/content.csv @@ -1,4 +1,4 @@ -pid,datum,geschlecht +pid,datum,value 1,2012-01-01,"f" 1,2012-01-02,"m" diff --git a/backend/src/test/resources/tests/aggregator/LAST_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json b/backend/src/test/resources/tests/aggregator/LAST_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json index 351d744ef3..30d31ac028 100644 --- a/backend/src/test/resources/tests/aggregator/LAST_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json +++ b/backend/src/test/resources/tests/aggregator/LAST_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json @@ -6,15 +6,14 @@ "type": "CONCEPT_QUERY", "root": { "ids": [ - "select" + "concept" ], "type": "CONCEPT", - "label": "select", "tables": [ { - "id": "select.connector", + "id": "concept.connector", "selects": [ - "select.connector.select" + "concept.connector.select" ] } ] @@ -22,20 +21,20 @@ }, "concepts": [ { - "label": "select", + "name": "concept", "type": "TREE", "connectors": [ { "label": "connector", - "table": "table1", + "table": "table", "validityDates": { "label": "datum", - "column": "table1.datum" + "column": "table.datum" }, "selects": { - "name" : "select", + "name": "select", "type": "LAST", - "column": "table1.geschlecht" + "column": "table.value" } } ] @@ -45,7 +44,7 @@ "tables": [ { "csv": "tests/aggregator/LAST_AGGREGATOR/content.csv", - "name": "table1", + "name": "table", "primaryColumn": { "name": "pid", "type": "STRING" @@ -56,7 +55,7 @@ "type": "DATE" }, { - "name": "geschlecht", + "name": "value", "type": "STRING" } ] diff --git a/backend/src/test/resources/tests/aggregator/LAST_AGGREGATOR/content.csv b/backend/src/test/resources/tests/aggregator/LAST_AGGREGATOR/content.csv index f79ff8063b..bb1adca9de 100644 --- a/backend/src/test/resources/tests/aggregator/LAST_AGGREGATOR/content.csv +++ b/backend/src/test/resources/tests/aggregator/LAST_AGGREGATOR/content.csv @@ -1,4 +1,4 @@ -pid,datum,geschlecht +pid,datum,value 1,2012-01-01,"f" 1,2012-01-02,"m" diff --git a/backend/src/test/resources/tests/aggregator/MULTI_SELECT_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json b/backend/src/test/resources/tests/aggregator/MULTI_SELECT_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json index 0995d9e239..8c624cf0e2 100644 --- a/backend/src/test/resources/tests/aggregator/MULTI_SELECT_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json +++ b/backend/src/test/resources/tests/aggregator/MULTI_SELECT_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json @@ -6,15 +6,14 @@ "type": "CONCEPT_QUERY", "root": { "ids": [ - "select" + "concept" ], "type": "CONCEPT", - "label": "select", "tables": [ { - "id": "select.connector", + "id": "concept.connector", "selects": [ - "select.connector.select" + "concept.connector.select" ] } ] @@ -22,20 +21,20 @@ }, "concepts": [ { - "label": "select", + "name": "concept", "type": "TREE", "connectors": [ { "label": "connector", - "table": "table1", + "table": "table", "validityDates": { "label": "datum", - "column": "table1.datum" + "column": "table.datum" }, "selects": { "type": "COUNT_OCCURENCES", "name" : "select", - "column": "table1.geschlecht", + "column": "table.value", "selection": [ "f", "m" @@ -49,7 +48,7 @@ "tables": [ { "csv": "tests/aggregator/MULTI_SELECT_AGGREGATOR/content.csv", - "name": "table1", + "name": "table", "primaryColumn": { "name": "pid", "type": "STRING" @@ -60,7 +59,7 @@ "type": "DATE" }, { - "name": "geschlecht", + "name": "value", "type": "STRING" } ] diff --git a/backend/src/test/resources/tests/aggregator/MULTI_SELECT_AGGREGATOR/content.csv b/backend/src/test/resources/tests/aggregator/MULTI_SELECT_AGGREGATOR/content.csv index c70835dffa..5358e7b61a 100644 --- a/backend/src/test/resources/tests/aggregator/MULTI_SELECT_AGGREGATOR/content.csv +++ b/backend/src/test/resources/tests/aggregator/MULTI_SELECT_AGGREGATOR/content.csv @@ -1,4 +1,4 @@ -pid,datum,geschlecht +pid,datum,value 1,2012-01-01,"f" 1,2012-01-02,"f" 2,2010-07-15, diff --git a/backend/src/test/resources/tests/aggregator/PREFIX_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json b/backend/src/test/resources/tests/aggregator/PREFIX_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json index 3cf3c20c08..1b06a5c362 100644 --- a/backend/src/test/resources/tests/aggregator/PREFIX_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json +++ b/backend/src/test/resources/tests/aggregator/PREFIX_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json @@ -6,15 +6,14 @@ "type": "CONCEPT_QUERY", "root": { "ids": [ - "select" + "concept" ], "type": "CONCEPT", - "label": "select", "tables": [ { - "id": "select.connector", + "id": "concept.connector", "selects": [ - "select.connector.select" + "concept.connector.select" ] } ] @@ -22,20 +21,20 @@ }, "concepts": [ { - "label": "select", + "name": "concept", "type": "TREE", "connectors": [ { "label": "connector", - "table": "table1", + "table": "table", "validityDates": { "label": "datum", - "column": "table1.datum" + "column": "table.datum" }, - "selects": { + "selects": { "type": "PREFIX", - "name" : "select", - "column": "table1.geschlecht", + "name": "select", + "column": "table.value", "prefix": "f" } } @@ -46,7 +45,7 @@ "tables": [ { "csv": "tests/aggregator/PREFIX_AGGREGATOR/content.csv", - "name": "table1", + "name": "table", "primaryColumn": { "name": "pid", "type": "STRING" @@ -57,7 +56,7 @@ "type": "DATE" }, { - "name": "geschlecht", + "name": "value", "type": "STRING" } ] diff --git a/backend/src/test/resources/tests/aggregator/PREFIX_AGGREGATOR/content.csv b/backend/src/test/resources/tests/aggregator/PREFIX_AGGREGATOR/content.csv index b67ebb2e50..c367b5ac24 100644 --- a/backend/src/test/resources/tests/aggregator/PREFIX_AGGREGATOR/content.csv +++ b/backend/src/test/resources/tests/aggregator/PREFIX_AGGREGATOR/content.csv @@ -1,4 +1,4 @@ -pid,datum,geschlecht +pid,datum,value 1,2012-01-01,"fa" 1,2012-01-02,"fb" 2,2010-07-15, diff --git a/backend/src/test/resources/tests/aggregator/QUARTER_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json b/backend/src/test/resources/tests/aggregator/QUARTER_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json index bd6aaf179b..2219901b5e 100644 --- a/backend/src/test/resources/tests/aggregator/QUARTER_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json +++ b/backend/src/test/resources/tests/aggregator/QUARTER_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json @@ -26,15 +26,15 @@ "connectors": [ { "name": "connector", - "table": "table1", + "table": "table", "validityDates": { "name": "datum", - "column": "table1.datum" + "column": "table.datum" }, "selects": { "name": "select", "type": "QUARTER", - "sample" : "LATEST" + "sample": "LATEST" } } ] @@ -44,7 +44,7 @@ "tables": [ { "csv": "tests/aggregator/QUARTER_AGGREGATOR/content.csv", - "name": "table1", + "name": "table", "primaryColumn": { "name": "pid", "type": "STRING" diff --git a/backend/src/test/resources/tests/aggregator/RANDOM_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json b/backend/src/test/resources/tests/aggregator/RANDOM_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json index bd615c6405..8d0033ea59 100644 --- a/backend/src/test/resources/tests/aggregator/RANDOM_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json +++ b/backend/src/test/resources/tests/aggregator/RANDOM_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json @@ -6,15 +6,14 @@ "type": "CONCEPT_QUERY", "root": { "ids": [ - "select" + "concept" ], "type": "CONCEPT", - "label": "select", "tables": [ { - "id": "select.connector", + "id": "concept.connector", "selects": [ - "select.connector.select" + "concept.connector.select" ] } ] @@ -22,20 +21,20 @@ }, "concepts": [ { - "label": "SELECT", + "label": "concept", "type": "TREE", "connectors": [ { "label": "connector", - "table": "table1", + "table": "table", "validityDates": { "label": "datum", - "column": "table1.datum" + "column": "table.datum" }, "selects": { "name" : "select", "type": "LAST", - "column": "table1.geschlecht" + "column": "table.value" } } ] @@ -45,7 +44,7 @@ "tables": [ { "csv": "tests/aggregator/RANDOM_AGGREGATOR/content.csv", - "name": "table1", + "name": "table", "primaryColumn": { "name": "pid", "type": "STRING" @@ -56,7 +55,7 @@ "type": "DATE" }, { - "name": "geschlecht", + "name": "value", "type": "STRING" } ] diff --git a/backend/src/test/resources/tests/aggregator/RANDOM_AGGREGATOR/content.csv b/backend/src/test/resources/tests/aggregator/RANDOM_AGGREGATOR/content.csv index 9f6ea8104b..6d99b9f9e5 100644 --- a/backend/src/test/resources/tests/aggregator/RANDOM_AGGREGATOR/content.csv +++ b/backend/src/test/resources/tests/aggregator/RANDOM_AGGREGATOR/content.csv @@ -1,4 +1,4 @@ -pid,datum,geschlecht +pid,datum,value 1,2012-01-01,"f" 2,2010-07-15, 4,,"f" diff --git a/backend/src/test/resources/tests/aggregator/SELECT_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json b/backend/src/test/resources/tests/aggregator/SELECT_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json index d313d41222..59a83c0f6e 100644 --- a/backend/src/test/resources/tests/aggregator/SELECT_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json +++ b/backend/src/test/resources/tests/aggregator/SELECT_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json @@ -6,15 +6,14 @@ "type": "CONCEPT_QUERY", "root": { "ids": [ - "select" + "concept" ], "type": "CONCEPT", - "label": "select", "tables": [ { - "id": "select.connector", + "id": "concept.connector", "selects": [ - "select.connector.select" + "concept.connector.select" ] } ] @@ -22,20 +21,20 @@ }, "concepts": [ { - "label": "select", + "name": "concept", "type": "TREE", "connectors": [ { "label": "connector", - "table": "table1", + "table": "table", "validityDates": { "label": "datum", - "column": "table1.datum" + "column": "table.datum" }, "selects": { "type": "COUNT_OCCURENCES", - "name" : "select", - "column": "table1.geschlecht", + "name": "select", + "column": "table.value", "selection": "f" } } @@ -46,7 +45,7 @@ "tables": [ { "csv": "tests/aggregator/SELECT_AGGREGATOR/content.csv", - "name": "table1", + "name": "table", "primaryColumn": { "name": "pid", "type": "STRING" @@ -57,7 +56,7 @@ "type": "DATE" }, { - "name": "geschlecht", + "name": "value", "type": "STRING" } ] diff --git a/backend/src/test/resources/tests/aggregator/SELECT_AGGREGATOR/content.csv b/backend/src/test/resources/tests/aggregator/SELECT_AGGREGATOR/content.csv index 8175833be1..cadb60f0f0 100644 --- a/backend/src/test/resources/tests/aggregator/SELECT_AGGREGATOR/content.csv +++ b/backend/src/test/resources/tests/aggregator/SELECT_AGGREGATOR/content.csv @@ -1,4 +1,4 @@ -pid,datum,geschlecht +pid,datum,value 1,2012-01-01,"f" 1,2012-01-02,"f" 2,2010-07-15, diff --git a/backend/src/test/resources/tests/aggregator/SUM_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json b/backend/src/test/resources/tests/aggregator/SUM_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json index aaf16d11a9..956b0bfd82 100644 --- a/backend/src/test/resources/tests/aggregator/SUM_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json +++ b/backend/src/test/resources/tests/aggregator/SUM_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json @@ -6,15 +6,14 @@ "type": "CONCEPT_QUERY", "root": { "ids": [ - "select" + "concept" ], "type": "CONCEPT", - "label": "select", "tables": [ { - "id": "select.connector", + "id": "concept.connector", "selects": [ - "select.connector.select" + "concept.connector.select" ] } ] @@ -22,20 +21,20 @@ }, "concepts": [ { - "label": "select", + "name": "concept", "type": "TREE", "connectors": [ { "label": "connector", - "table": "table1", + "table": "table", "validityDates": { "label": "datum", - "column": "table1.datum" + "column": "table.datum" }, "selects": { - "name" : "select", + "name": "select", "type": "SUM", - "column": "table1.geschlecht" + "column": "table.value" } } ] @@ -45,7 +44,7 @@ "tables": [ { "csv": "tests/aggregator/SUM_AGGREGATOR/content.csv", - "name": "table1", + "name": "table", "primaryColumn": { "name": "pid", "type": "STRING" @@ -56,7 +55,7 @@ "type": "DATE" }, { - "name": "geschlecht", + "name": "value", "type": "INTEGER" } ] diff --git a/backend/src/test/resources/tests/aggregator/SUM_AGGREGATOR/content.csv b/backend/src/test/resources/tests/aggregator/SUM_AGGREGATOR/content.csv index 99ca608108..f947317227 100644 --- a/backend/src/test/resources/tests/aggregator/SUM_AGGREGATOR/content.csv +++ b/backend/src/test/resources/tests/aggregator/SUM_AGGREGATOR/content.csv @@ -1,4 +1,4 @@ -pid,datum,geschlecht +pid,datum,value 1,2012-01-01,1 1,2012-01-02,1 2,2010-07-15, diff --git a/backend/src/test/resources/tests/aggregator/SUM_DIFF_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json b/backend/src/test/resources/tests/aggregator/SUM_DIFF_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json index 24313f7625..d7928fb1f6 100644 --- a/backend/src/test/resources/tests/aggregator/SUM_DIFF_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json +++ b/backend/src/test/resources/tests/aggregator/SUM_DIFF_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json @@ -6,15 +6,14 @@ "type": "CONCEPT_QUERY", "root": { "ids": [ - "select" + "concept" ], "type": "CONCEPT", - "label": "select", "tables": [ { - "id": "select.connector", + "id": "concept.connector", "selects": [ - "select.connector.select" + "concept.connector.select" ] } ] @@ -22,21 +21,21 @@ }, "concepts": [ { - "label": "select", + "name": "concept", "type": "TREE", "connectors": [ { "label": "connector", - "table": "table1", + "table": "table", "validityDates": { "label": "datum", - "column": "table1.datum" + "column": "table.datum" }, "selects": { "type": "SUM", - "name" : "select", - "column": "table1.sum", - "subtractColumn": "table1.sub" + "name": "select", + "column": "table.sum", + "subtractColumn": "table.sub" } } ] @@ -46,7 +45,7 @@ "tables": [ { "csv": "tests/aggregator/SUM_DIFF_AGGREGATOR/content.csv", - "name": "table1", + "name": "table", "primaryColumn": { "name": "pid", "type": "STRING" diff --git a/backend/src/test/resources/tests/aggregator/VALUES_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json b/backend/src/test/resources/tests/aggregator/VALUES_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json index 1316d09bfa..304bd33c69 100644 --- a/backend/src/test/resources/tests/aggregator/VALUES_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json +++ b/backend/src/test/resources/tests/aggregator/VALUES_AGGREGATOR/SIMPLE_VIRTUAL_CONCEPT_Query.test.json @@ -6,15 +6,14 @@ "type": "CONCEPT_QUERY", "root": { "ids": [ - "select" + "concept" ], "type": "CONCEPT", - "label": "select", "tables": [ { - "id": "select.connector", + "id": "concept.connector", "selects": [ - "select.connector.select" + "concept.connector.select" ] } ] @@ -22,20 +21,20 @@ }, "concepts": [ { - "label": "SELECT", + "label": "concept", "type": "TREE", "connectors": [ { "label": "connector", - "table": "table1", + "table": "table", "validityDates": { "label": "datum", - "column": "table1.datum" + "column": "table.datum" }, "selects": { "name" : "select", "type": "DISTINCT", - "column": "table1.geschlecht" + "column": "table.value" } } ] @@ -45,7 +44,7 @@ "tables": [ { "csv": "tests/aggregator/VALUES_AGGREGATOR/content.csv", - "name": "table1", + "name": "table", "primaryColumn": { "name": "pid", "type": "STRING" @@ -56,7 +55,7 @@ "type": "DATE" }, { - "name": "geschlecht", + "name": "value", "type": "STRING" } ] diff --git a/backend/src/test/resources/tests/aggregator/VALUES_AGGREGATOR/content.csv b/backend/src/test/resources/tests/aggregator/VALUES_AGGREGATOR/content.csv index bd5ab04b1b..a54008f3a2 100644 --- a/backend/src/test/resources/tests/aggregator/VALUES_AGGREGATOR/content.csv +++ b/backend/src/test/resources/tests/aggregator/VALUES_AGGREGATOR/content.csv @@ -1,4 +1,4 @@ -pid,datum,geschlecht +pid,datum,value 1,2012-01-01,"f" 1,2012-01-02,"f" 2,2010-07-15, From 92afa4f86833c378699b85b34f0bc89a00b079ad Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Wed, 28 Jul 2021 13:07:12 +0200 Subject: [PATCH 48/82] fix headers for aggregator tests --- .../aggregator/COUNT_AGGREGATOR/expected.csv | 2 +- .../COUNT_DISTINCT_AGGREGATOR/expected.csv | 2 +- .../COUNT_QUARTERS_AGGREGATOR/expected.csv | 2 +- .../DATE_DISTANCE.test.json | 60 ++++++++----------- .../DATE_DISTANCE2.test.json | 60 ++++++++----------- .../DATE_DISTANCE_AGGREGATOR/content.csv | 2 +- .../DATE_DISTANCE_AGGREGATOR/expected.csv | 2 +- .../DURATION_SUM_AGGREGATOR/expected.csv | 2 +- .../expected_no_restriction.csv | 2 +- .../expected_restriction.csv | 2 +- .../aggregator/EXISTS_AGGREGATOR/expected.csv | 2 +- .../EXISTS_AGGREGATOR_AGGREGATED/expected.csv | 2 +- .../EXISTS_AGGREGATOR_OR/expected.csv | 2 +- .../aggregator/FIRST_AGGREGATOR/expected.csv | 2 +- .../aggregator/LAST_AGGREGATOR/expected.csv | 2 +- .../MULTI_SELECT_AGGREGATOR/expected.csv | 2 +- .../aggregator/PREFIX_AGGREGATOR/expected.csv | 2 +- .../QUARTER_AGGREGATOR/expected.csv | 2 +- .../aggregator/RANDOM_AGGREGATOR/expected.csv | 2 +- .../aggregator/SELECT_AGGREGATOR/expected.csv | 2 +- .../aggregator/SUM_AGGREGATOR/expected.csv | 2 +- .../SUM_DIFF_AGGREGATOR/expected.csv | 2 +- .../aggregator/VALUES_AGGREGATOR/expected.csv | 2 +- 23 files changed, 73 insertions(+), 89 deletions(-) diff --git a/backend/src/test/resources/tests/aggregator/COUNT_AGGREGATOR/expected.csv b/backend/src/test/resources/tests/aggregator/COUNT_AGGREGATOR/expected.csv index 4028c4804d..cd113058fd 100644 --- a/backend/src/test/resources/tests/aggregator/COUNT_AGGREGATOR/expected.csv +++ b/backend/src/test/resources/tests/aggregator/COUNT_AGGREGATOR/expected.csv @@ -1,4 +1,4 @@ -result,dates,select.connector.count_genders +result,dates,concept - select 1,{2012-01-01/2012-01-02},2 2,{2010-07-15/2010-07-15}, 3,{2013-11-10/2013-11-10},1 diff --git a/backend/src/test/resources/tests/aggregator/COUNT_DISTINCT_AGGREGATOR/expected.csv b/backend/src/test/resources/tests/aggregator/COUNT_DISTINCT_AGGREGATOR/expected.csv index 91c395d9e2..f1637cd722 100644 --- a/backend/src/test/resources/tests/aggregator/COUNT_DISTINCT_AGGREGATOR/expected.csv +++ b/backend/src/test/resources/tests/aggregator/COUNT_DISTINCT_AGGREGATOR/expected.csv @@ -1,4 +1,4 @@ -result,dates,select.connector.count_gender +result,dates,concept - select 1,{2012-01-01/2012-01-02},1 2,{2010-07-15/2010-07-15}, 3,{2013-11-10/2013-11-10},1 diff --git a/backend/src/test/resources/tests/aggregator/COUNT_QUARTERS_AGGREGATOR/expected.csv b/backend/src/test/resources/tests/aggregator/COUNT_QUARTERS_AGGREGATOR/expected.csv index 982c5372df..c5b411da06 100644 --- a/backend/src/test/resources/tests/aggregator/COUNT_QUARTERS_AGGREGATOR/expected.csv +++ b/backend/src/test/resources/tests/aggregator/COUNT_QUARTERS_AGGREGATOR/expected.csv @@ -1,4 +1,4 @@ -result,dates,select.connector.count_quarters +result,dates,concept - select 1,{2015-01-01/2016-12-31},1 2,{2015-01-01/2016-12-31},2 3,{2015-01-01/2016-12-31},4 diff --git a/backend/src/test/resources/tests/aggregator/DATE_DISTANCE_AGGREGATOR/DATE_DISTANCE.test.json b/backend/src/test/resources/tests/aggregator/DATE_DISTANCE_AGGREGATOR/DATE_DISTANCE.test.json index 9afdc03bd2..2255eac608 100644 --- a/backend/src/test/resources/tests/aggregator/DATE_DISTANCE_AGGREGATOR/DATE_DISTANCE.test.json +++ b/backend/src/test/resources/tests/aggregator/DATE_DISTANCE_AGGREGATOR/DATE_DISTANCE.test.json @@ -5,50 +5,42 @@ "query": { "type": "CONCEPT_QUERY", "root": { - "type":"DATE_RESTRICTION", - "dateRange":{ - "min":"2010-01-01", - "max":"2010-03-31" - }, - "child":{ - "type":"OR", - "children":[ - { - "ids": [ - "con" - ], - "type": "CONCEPT", - "label": "Geschlecht SELECT", - "tables": [ - { - "id": "con.geschlecht_connector", - "selects": "con.geschlecht_connector.geb" - } - ] - - } - ] - } + "type": "DATE_RESTRICTION", + "dateRange": { + "min": "2010-01-01", + "max": "2010-03-31" + }, + "child": { + "ids": [ + "concept" + ], + "type": "CONCEPT", + "tables": [ + { + "id": "concept.connector", + "selects": "concept.connector.select" + } + ] + } } }, "concepts": [ { - "label": "Geschlecht SELECT", - "name": "con", + "name": "concept", "type": "TREE", "connectors": [ { - "label": "geschlecht_connector", + "label": "connector", "table": "table", "validityDates": { "label": "indexdatum", "column": "table.indexdatum" }, - "selects": { - "name": "geb", - "type": "DATE_DISTANCE", - "column": "table.geburtsdatum" - } + "selects": { + "name": "select", + "type": "DATE_DISTANCE", + "column": "table.datum" + } } ] } @@ -67,8 +59,8 @@ "name": "indexdatum", "type": "DATE_RANGE" }, - { - "name": "geburtsdatum", + { + "name": "datum", "type": "DATE" } ] diff --git a/backend/src/test/resources/tests/aggregator/DATE_DISTANCE_AGGREGATOR/DATE_DISTANCE2.test.json b/backend/src/test/resources/tests/aggregator/DATE_DISTANCE_AGGREGATOR/DATE_DISTANCE2.test.json index f92e2e31f4..6ec9fd4f80 100644 --- a/backend/src/test/resources/tests/aggregator/DATE_DISTANCE_AGGREGATOR/DATE_DISTANCE2.test.json +++ b/backend/src/test/resources/tests/aggregator/DATE_DISTANCE_AGGREGATOR/DATE_DISTANCE2.test.json @@ -5,50 +5,42 @@ "query": { "type": "CONCEPT_QUERY", "root": { - "type":"DATE_RESTRICTION", - "dateRange":{ - "min":"2005-01-01", - "max":"2010-03-31" - }, - "child":{ - "type":"OR", - "children":[ - { - "ids": [ - "con" - ], - "type": "CONCEPT", - "label": "Geschlecht SELECT", - "tables": [ - { - "id": "con.geschlecht_connector", - "selects": "con.geschlecht_connector.geb" - } - ] - - } - ] - } + "type": "DATE_RESTRICTION", + "dateRange": { + "min": "2005-01-01", + "max": "2010-03-31" + }, + "child": { + "ids": [ + "concept" + ], + "type": "CONCEPT", + "tables": [ + { + "id": "concept.connector", + "selects": "concept.connector.select" + } + ] + } } }, "concepts": [ { - "label": "Geschlecht SELECT", - "name": "con", + "name": "concept", "type": "TREE", "connectors": [ { - "label": "geschlecht_connector", + "label": "connector", "table": "table", "validityDates": { "label": "indexdatum", "column": "table.indexdatum" }, - "selects": { - "name": "geb", - "type": "DATE_DISTANCE", - "column": "table.geburtsdatum" - } + "selects": { + "name": "select", + "type": "DATE_DISTANCE", + "column": "table.datum" + } } ] } @@ -67,8 +59,8 @@ "name": "indexdatum", "type": "DATE_RANGE" }, - { - "name": "geburtsdatum", + { + "name": "datum", "type": "DATE" } ] diff --git a/backend/src/test/resources/tests/aggregator/DATE_DISTANCE_AGGREGATOR/content.csv b/backend/src/test/resources/tests/aggregator/DATE_DISTANCE_AGGREGATOR/content.csv index ddfdc58317..d0e88a7db7 100644 --- a/backend/src/test/resources/tests/aggregator/DATE_DISTANCE_AGGREGATOR/content.csv +++ b/backend/src/test/resources/tests/aggregator/DATE_DISTANCE_AGGREGATOR/content.csv @@ -1,4 +1,4 @@ -pid,indexdatum,geburtsdatum +pid,indexdatum,datum 1,"2010-01-01/2010-03-31",2010-01-31 2,"2010-01-01/2010-03-31",1998-01-01 3,"2010-01-01/2010-03-31",1997-12-31 diff --git a/backend/src/test/resources/tests/aggregator/DATE_DISTANCE_AGGREGATOR/expected.csv b/backend/src/test/resources/tests/aggregator/DATE_DISTANCE_AGGREGATOR/expected.csv index 591eadcf2a..692e212a89 100644 --- a/backend/src/test/resources/tests/aggregator/DATE_DISTANCE_AGGREGATOR/expected.csv +++ b/backend/src/test/resources/tests/aggregator/DATE_DISTANCE_AGGREGATOR/expected.csv @@ -1,4 +1,4 @@ -result,dates,con.geschlecht_connector.geb +result,dates,concept - select 1,{2010-01-01/2010-03-31},0 2,{2010-01-01/2010-03-31},12 3,{2010-01-01/2010-03-31},12 diff --git a/backend/src/test/resources/tests/aggregator/DURATION_SUM_AGGREGATOR/expected.csv b/backend/src/test/resources/tests/aggregator/DURATION_SUM_AGGREGATOR/expected.csv index c4e7966a2d..57c38bb809 100644 --- a/backend/src/test/resources/tests/aggregator/DURATION_SUM_AGGREGATOR/expected.csv +++ b/backend/src/test/resources/tests/aggregator/DURATION_SUM_AGGREGATOR/expected.csv @@ -1,4 +1,4 @@ -result,dates,select.connector.select +result,dates,concept - select 1,{2010-01-01/2010-01-31},31 3,{2013-08-10/2013-08-11},2 4,{2010-06-15/2010-06-20},6 diff --git a/backend/src/test/resources/tests/aggregator/EVENT_DATE_AGGREGATOR/expected_no_restriction.csv b/backend/src/test/resources/tests/aggregator/EVENT_DATE_AGGREGATOR/expected_no_restriction.csv index cf97cea443..e7a4b365ee 100644 --- a/backend/src/test/resources/tests/aggregator/EVENT_DATE_AGGREGATOR/expected_no_restriction.csv +++ b/backend/src/test/resources/tests/aggregator/EVENT_DATE_AGGREGATOR/expected_no_restriction.csv @@ -1,4 +1,4 @@ -result,dates,con.event-date,con.geschlecht_connector.event-date +result,dates,concept - dates,concept - dates_1 1,{2010-01-01/2010-03-31},{2010-01-01/2010-03-31},{2010-01-01/2010-03-31} 10,{2010-01-01/2010-03-31},{2010-01-01/2010-03-31},{2010-01-01/2010-03-31} 2,{2010-01-01/2010-03-31},{2010-01-01/2010-03-31},{2010-01-01/2010-03-31} diff --git a/backend/src/test/resources/tests/aggregator/EVENT_DATE_AGGREGATOR/expected_restriction.csv b/backend/src/test/resources/tests/aggregator/EVENT_DATE_AGGREGATOR/expected_restriction.csv index fe87f5b064..ca73bdee9b 100644 --- a/backend/src/test/resources/tests/aggregator/EVENT_DATE_AGGREGATOR/expected_restriction.csv +++ b/backend/src/test/resources/tests/aggregator/EVENT_DATE_AGGREGATOR/expected_restriction.csv @@ -1,4 +1,4 @@ -result,dates,con.event-date,con.geschlecht_connector.event-date +result,dates,concept - dates,concept - dates_1 1,{2010-02-01/2010-02-28},{2010-02-01/2010-02-28},{2010-02-01/2010-02-28} 10,{2010-02-01/2010-02-28},{2010-02-01/2010-02-28},{2010-02-01/2010-02-28} 2,{2010-02-01/2010-02-28},{2010-02-01/2010-02-28},{2010-02-01/2010-02-28} diff --git a/backend/src/test/resources/tests/aggregator/EXISTS_AGGREGATOR/expected.csv b/backend/src/test/resources/tests/aggregator/EXISTS_AGGREGATOR/expected.csv index 6d495e3bf1..962b85f37b 100644 --- a/backend/src/test/resources/tests/aggregator/EXISTS_AGGREGATOR/expected.csv +++ b/backend/src/test/resources/tests/aggregator/EXISTS_AGGREGATOR/expected.csv @@ -1,4 +1,4 @@ -result,dates,number.exists +result,dates,concept - exists 1,{2014-06-30/2015-06-30},1 2,{2014-06-30/2015-06-30},1 5,{2014-06-30/2015-06-30},1 \ No newline at end of file diff --git a/backend/src/test/resources/tests/aggregator/EXISTS_AGGREGATOR_AGGREGATED/expected.csv b/backend/src/test/resources/tests/aggregator/EXISTS_AGGREGATOR_AGGREGATED/expected.csv index 513b781f4e..676fbd1f39 100644 --- a/backend/src/test/resources/tests/aggregator/EXISTS_AGGREGATOR_AGGREGATED/expected.csv +++ b/backend/src/test/resources/tests/aggregator/EXISTS_AGGREGATOR_AGGREGATED/expected.csv @@ -1,4 +1,4 @@ -result,dates,number.exists,number.exists_1,AND,number.exists_2,number.exists_3,OR +result,dates,concept - 1 - exists,concept - 2 - exists,AND,concept - 1 - exists_1,concept - 2 - exists_1,OR 1,{2020-01-01/2020-01-01},1,0,0,1,0,1 2,{2020-01-01/2020-01-01},0,1,0,0,1,1 3,{2020-01-01/2020-01-01},1,1,1,1,1,1 \ No newline at end of file diff --git a/backend/src/test/resources/tests/aggregator/EXISTS_AGGREGATOR_OR/expected.csv b/backend/src/test/resources/tests/aggregator/EXISTS_AGGREGATOR_OR/expected.csv index 767f82e6bc..d60a5ac7d7 100644 --- a/backend/src/test/resources/tests/aggregator/EXISTS_AGGREGATOR_OR/expected.csv +++ b/backend/src/test/resources/tests/aggregator/EXISTS_AGGREGATOR_OR/expected.csv @@ -1,4 +1,4 @@ -result,dates,number.exists,number.exists_1 +result,dates,concept - exists,concept - exists_1 1,{2014-06-30/2015-06-30},1,0 2,{2014-06-30/2015-06-30},1,1 5,{2014-06-30/2015-06-30},1,0 \ No newline at end of file diff --git a/backend/src/test/resources/tests/aggregator/FIRST_AGGREGATOR/expected.csv b/backend/src/test/resources/tests/aggregator/FIRST_AGGREGATOR/expected.csv index 78dc313788..08e60abefd 100644 --- a/backend/src/test/resources/tests/aggregator/FIRST_AGGREGATOR/expected.csv +++ b/backend/src/test/resources/tests/aggregator/FIRST_AGGREGATOR/expected.csv @@ -1,4 +1,4 @@ -result,dates,select.connector.select +result,dates,concept - select 1,{2012-01-01/2012-01-02},f 2,{2010-07-15/2010-07-15}, 3,{2012-01-01/2012-01-02},f diff --git a/backend/src/test/resources/tests/aggregator/LAST_AGGREGATOR/expected.csv b/backend/src/test/resources/tests/aggregator/LAST_AGGREGATOR/expected.csv index e470c6a90a..5f4675601c 100644 --- a/backend/src/test/resources/tests/aggregator/LAST_AGGREGATOR/expected.csv +++ b/backend/src/test/resources/tests/aggregator/LAST_AGGREGATOR/expected.csv @@ -1,4 +1,4 @@ -result,dates,select.connector.select +result,dates,concept - select 1,{2012-01-01/2012-01-02},m 2,{2010-07-15/2010-07-15}, 3,{2012-01-01/2012-01-02},f diff --git a/backend/src/test/resources/tests/aggregator/MULTI_SELECT_AGGREGATOR/expected.csv b/backend/src/test/resources/tests/aggregator/MULTI_SELECT_AGGREGATOR/expected.csv index 67b04165cc..33a9b7b834 100644 --- a/backend/src/test/resources/tests/aggregator/MULTI_SELECT_AGGREGATOR/expected.csv +++ b/backend/src/test/resources/tests/aggregator/MULTI_SELECT_AGGREGATOR/expected.csv @@ -1,4 +1,4 @@ -result,dates,select.connector.select +result,dates,concept - select 2,{2010-07-15/2010-07-15}, 3,{2013-11-10/2013-11-10},{f=1} 6,{2012-11-11/2012-11-11},"{f=1, m=1}" diff --git a/backend/src/test/resources/tests/aggregator/PREFIX_AGGREGATOR/expected.csv b/backend/src/test/resources/tests/aggregator/PREFIX_AGGREGATOR/expected.csv index d692a149e8..0c3194be1d 100644 --- a/backend/src/test/resources/tests/aggregator/PREFIX_AGGREGATOR/expected.csv +++ b/backend/src/test/resources/tests/aggregator/PREFIX_AGGREGATOR/expected.csv @@ -1,4 +1,4 @@ -result,dates,select.connector.select +result,dates,concept - select 1,{2012-01-01/2012-01-02},"[fa, fb]" 2,{2010-07-15/2010-07-15}, 3,{2013-11-10/2013-11-10},[f] diff --git a/backend/src/test/resources/tests/aggregator/QUARTER_AGGREGATOR/expected.csv b/backend/src/test/resources/tests/aggregator/QUARTER_AGGREGATOR/expected.csv index 6e4980e6b5..cecf2b3864 100644 --- a/backend/src/test/resources/tests/aggregator/QUARTER_AGGREGATOR/expected.csv +++ b/backend/src/test/resources/tests/aggregator/QUARTER_AGGREGATOR/expected.csv @@ -1,3 +1,3 @@ -result,dates,concept.connector.select +result,dates,concept - select 1,{2015-01-17/2015-01-17},2015-Q1 2,"{2015-03-31/2015-03-31, 2015-09-01/2015-09-01}",2015-Q3 \ No newline at end of file diff --git a/backend/src/test/resources/tests/aggregator/RANDOM_AGGREGATOR/expected.csv b/backend/src/test/resources/tests/aggregator/RANDOM_AGGREGATOR/expected.csv index 8039c9daa3..8c1d4d01f1 100644 --- a/backend/src/test/resources/tests/aggregator/RANDOM_AGGREGATOR/expected.csv +++ b/backend/src/test/resources/tests/aggregator/RANDOM_AGGREGATOR/expected.csv @@ -1,4 +1,4 @@ -result,dates,select.connector.select +result,dates,concept - select 1,{2012-01-01/2012-01-01},f 2,{2010-07-15/2010-07-15}, 5,{2012-01-01/2012-01-01},m diff --git a/backend/src/test/resources/tests/aggregator/SELECT_AGGREGATOR/expected.csv b/backend/src/test/resources/tests/aggregator/SELECT_AGGREGATOR/expected.csv index 9c14d92c65..fecdbf81f1 100644 --- a/backend/src/test/resources/tests/aggregator/SELECT_AGGREGATOR/expected.csv +++ b/backend/src/test/resources/tests/aggregator/SELECT_AGGREGATOR/expected.csv @@ -1,4 +1,4 @@ -result,dates,select.connector.select +result,dates,concept - select 1,{2012-01-01/2012-01-02},2 2,{2010-07-15/2010-07-15}, 3,{2013-11-10/2013-11-10},1 diff --git a/backend/src/test/resources/tests/aggregator/SUM_AGGREGATOR/expected.csv b/backend/src/test/resources/tests/aggregator/SUM_AGGREGATOR/expected.csv index 09959c985a..7a2b15acb1 100644 --- a/backend/src/test/resources/tests/aggregator/SUM_AGGREGATOR/expected.csv +++ b/backend/src/test/resources/tests/aggregator/SUM_AGGREGATOR/expected.csv @@ -1,4 +1,4 @@ -result,dates,select.connector.select +result,dates,concept - select 1,{2012-01-01/2012-01-02},2 2,{2010-07-15/2010-07-15}, 3,{2013-11-10/2013-11-10},1 diff --git a/backend/src/test/resources/tests/aggregator/SUM_DIFF_AGGREGATOR/expected.csv b/backend/src/test/resources/tests/aggregator/SUM_DIFF_AGGREGATOR/expected.csv index fdb6815d5f..87440f8dda 100644 --- a/backend/src/test/resources/tests/aggregator/SUM_DIFF_AGGREGATOR/expected.csv +++ b/backend/src/test/resources/tests/aggregator/SUM_DIFF_AGGREGATOR/expected.csv @@ -1,4 +1,4 @@ -result,dates,select.connector.select +result,dates,concept - select 1,{2012-01-01/2012-01-02},2 2,{2010-07-15/2010-07-15}, 3,{2013-11-10/2013-11-10},0 diff --git a/backend/src/test/resources/tests/aggregator/VALUES_AGGREGATOR/expected.csv b/backend/src/test/resources/tests/aggregator/VALUES_AGGREGATOR/expected.csv index 9d9bdf7ed4..43e1c3f939 100644 --- a/backend/src/test/resources/tests/aggregator/VALUES_AGGREGATOR/expected.csv +++ b/backend/src/test/resources/tests/aggregator/VALUES_AGGREGATOR/expected.csv @@ -1,4 +1,4 @@ -result,dates,select.connector.select +result,dates,concept - select 1,{2012-01-01/2012-01-02},{f} 2,{2010-07-15/2010-07-15}, 3,{2012-01-01/2012-01-02},"{f, m}" From ae361e05decd0a02e1142ae4ebf15ea1d8bc9885 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Wed, 28 Jul 2021 14:45:43 +0200 Subject: [PATCH 49/82] Adds missing CentralRegistry for PrimaryDictionary --- .../conquery/io/storage/NamespaceStorage.java | 12 ++++++++++-- .../bakdata/conquery/models/config/StoreFactory.java | 2 +- .../conquery/models/config/XodusStoreFactory.java | 4 ++-- .../models/identifiable/mapping/EntityIdMap.java | 1 + .../integration/json/AbstractQueryEngineTest.java | 1 - .../conquery/util/NonPersistentStoreFactory.java | 2 +- 6 files changed, 15 insertions(+), 7 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java index ee5bd5b426..de44da278d 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java @@ -41,9 +41,10 @@ public NamespaceStorage(Validator validator, StoreFactory storageFactory, String super(validator, storageFactory, pathName); idMapping = storageFactory.createIdMappingStore(pathName); - structure = storageFactory.createStructureStore(pathName, new SingletonNamespaceCollection(getCentralRegistry())); + final SingletonNamespaceCollection namespaceCollection = new SingletonNamespaceCollection(getCentralRegistry()); + structure = storageFactory.createStructureStore(pathName, namespaceCollection); workerToBuckets = storageFactory.createWorkerToBucketsStore(pathName); - primaryDictionary = storageFactory.createPrimaryDictionaryStore(pathName); + primaryDictionary = storageFactory.createPrimaryDictionaryStore(pathName, namespaceCollection); decorateIdMapping(idMapping); } @@ -81,6 +82,7 @@ public void loadData() { idMapping.loadData(); structure.loadData(); workerToBuckets.loadData(); + primaryDictionary.loadData(); } @Override @@ -89,6 +91,8 @@ public void clear() { idMapping.clear(); structure.clear(); workerToBuckets.clear(); + primaryDictionary.clear(); + } @Override @@ -97,6 +101,8 @@ public void removeStorage() { idMapping.removeStore(); structure.removeStore(); workerToBuckets.removeStore(); + primaryDictionary.removeStore(); + } @Override @@ -105,6 +111,8 @@ public void close() throws IOException { idMapping.close(); structure.close(); workerToBuckets.close(); + primaryDictionary.close(); + } public EntityIdMap getIdMapping() { diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java index 0c642243ab..12544591ce 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java @@ -67,5 +67,5 @@ public interface StoreFactory { IdentifiableStore createRoleStore(CentralRegistry centralRegistry, String pathName); IdentifiableStore createGroupStore(CentralRegistry centralRegistry, String pathName); - SingletonStore createPrimaryDictionaryStore(String pathName); + SingletonStore createPrimaryDictionaryStore(String pathName, SingletonNamespaceCollection namespaceCollection); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java index 5e9a3b7afa..c3ce35368b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java @@ -347,8 +347,8 @@ public IdentifiableStore createGroupStore(CentralRegistry centralRegistry } @Override - public SingletonStore createPrimaryDictionaryStore(String pathName) { - return PRIMARY_DICTIONARY.singleton(createStore(findEnvironment(pathName), validator, PRIMARY_DICTIONARY)); + public SingletonStore createPrimaryDictionaryStore(String pathName, SingletonNamespaceCollection namespaceCollection) { + return PRIMARY_DICTIONARY.singleton(createStore(findEnvironment(pathName), validator, PRIMARY_DICTIONARY), namespaceCollection); } private File resolveSubDir(String... subdirs) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java index 6979661322..ce5bb1cea2 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java @@ -34,6 +34,7 @@ public class EntityIdMap { @Setter @JsonIgnore + @EqualsAndHashCode.Exclude private NamespaceStorage storage; /** diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java b/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java index c8c29c8fbe..3d4b9f1600 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java @@ -54,7 +54,6 @@ public void executeTest(StandaloneSupport standaloneSupport) throws IOException log.info("{} QUERY INIT", getLabel()); - final ConqueryConfig config = standaloneSupport.getConfig(); final User testUser = standaloneSupport.getTestUser(); final ManagedExecutionId executionId = IntegrationUtils.assertQueryResult(standaloneSupport, query, -1, ExecutionState.DONE, testUser, 201); diff --git a/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java b/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java index d2f06bee60..a88c4f8c30 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java +++ b/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java @@ -132,7 +132,7 @@ public IdentifiableStore createGroupStore(CentralRegistry centralRegistry } @Override - public SingletonStore createPrimaryDictionaryStore(String pathName) { + public SingletonStore createPrimaryDictionaryStore(String pathName, SingletonNamespaceCollection namespaceCollection) { return DICTIONARIES.singleton(new NonPersistentStore<>()); } From e181cd593d652ab417a369acb0eee90fc36f022f Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Wed, 28 Jul 2021 15:11:05 +0200 Subject: [PATCH 50/82] fix headers of query tests --- backend/src/test/resources/tests/query/ABS_EXPORT/expected.csv | 2 +- .../test/resources/tests/query/ARRAY_CONCEPT_QUERY/expected.csv | 2 +- .../CONCEPT_DATE_RESTRICTION_WITHOUT_VALIDITYDATE/expected.csv | 2 +- .../tests/query/CONCEPT_WITHOUT_VALIDITYDATE/expected.csv | 2 +- .../resources/tests/query/LOGICAL/AND_DURATION_SUM/expected.csv | 2 +- .../query/LOGICAL/AND_DURATION_SUM_MULTI_CONCEPT/expected.csv | 2 +- .../resources/tests/query/MERGE/AND_DURATION_SUM/expected.csv | 2 +- .../tests/query/MERGE/AND_EVENT_DATE_EXCLUDED/expected.csv | 2 +- .../tests/query/MERGE/OR_EVENT_DATE_EXCLUDED/expected.csv | 2 +- .../tests/query/MULTI_CONCEPT_QUERY_SEPARATE_DATES/expected.csv | 2 +- .../query/MULTI_CONNECTOR_QUERY_SEPARATE_DATES/expected.csv | 2 +- .../query/REL_EXPORT/PREREQUISITE_WITHOUT_DATES/expected.csv | 2 +- .../tests/query/REL_EXPORT/PREREQUISITE_WITH_DATES/expected.csv | 2 +- 13 files changed, 13 insertions(+), 13 deletions(-) diff --git a/backend/src/test/resources/tests/query/ABS_EXPORT/expected.csv b/backend/src/test/resources/tests/query/ABS_EXPORT/expected.csv index b2021c2844..db0f379c9d 100644 --- a/backend/src/test/resources/tests/query/ABS_EXPORT/expected.csv +++ b/backend/src/test/resources/tests/query/ABS_EXPORT/expected.csv @@ -1,4 +1,4 @@ -result,resolution,index,date_range,test_tree.exists +result,resolution,index,date_range,test_tree - test_child1 - exists 0,complete,,2000-01-01/2002-12-31,1 0,year,1,2000-01-01/2000-12-31,1 0,year,2,2001-01-01/2001-12-31,1 diff --git a/backend/src/test/resources/tests/query/ARRAY_CONCEPT_QUERY/expected.csv b/backend/src/test/resources/tests/query/ARRAY_CONCEPT_QUERY/expected.csv index 91288c5568..09906e1ffe 100644 --- a/backend/src/test/resources/tests/query/ARRAY_CONCEPT_QUERY/expected.csv +++ b/backend/src/test/resources/tests/query/ARRAY_CONCEPT_QUERY/expected.csv @@ -1,4 +1,4 @@ -result,dates,select.connector.count_genders,select.connector.count_genders_1 +result,dates,select - count_genders,select - count_genders_1 1,"{2012-01-01/2012-01-02, 2013-01-01/2013-01-02}",4,4 2,{2010-07-15/2010-07-15},, 3,{2013-11-10/2013-11-10},1, diff --git a/backend/src/test/resources/tests/query/CONCEPT_DATE_RESTRICTION_WITHOUT_VALIDITYDATE/expected.csv b/backend/src/test/resources/tests/query/CONCEPT_DATE_RESTRICTION_WITHOUT_VALIDITYDATE/expected.csv index d4cad06fc9..fc2d86e6d7 100644 --- a/backend/src/test/resources/tests/query/CONCEPT_DATE_RESTRICTION_WITHOUT_VALIDITYDATE/expected.csv +++ b/backend/src/test/resources/tests/query/CONCEPT_DATE_RESTRICTION_WITHOUT_VALIDITYDATE/expected.csv @@ -1,3 +1,3 @@ -result,dates,test_tree.test_column.test_select +result,dates,test_tree - test_child1 - Ausgabe test 1,{2017-01-01/2017-12-31},A1 3,{2017-01-01/2017-12-31},A1 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/CONCEPT_WITHOUT_VALIDITYDATE/expected.csv b/backend/src/test/resources/tests/query/CONCEPT_WITHOUT_VALIDITYDATE/expected.csv index bfefa4120c..879b21b0eb 100644 --- a/backend/src/test/resources/tests/query/CONCEPT_WITHOUT_VALIDITYDATE/expected.csv +++ b/backend/src/test/resources/tests/query/CONCEPT_WITHOUT_VALIDITYDATE/expected.csv @@ -1,3 +1,3 @@ -result,dates,test_tree.test_column.test_select +result,dates,test_tree - test_child1 - Ausgabe test 1,{-∞/+∞},A1 3,{-∞/+∞},A1 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/LOGICAL/AND_DURATION_SUM/expected.csv b/backend/src/test/resources/tests/query/LOGICAL/AND_DURATION_SUM/expected.csv index f36fb6e85e..b6c5f4834e 100644 --- a/backend/src/test/resources/tests/query/LOGICAL/AND_DURATION_SUM/expected.csv +++ b/backend/src/test/resources/tests/query/LOGICAL/AND_DURATION_SUM/expected.csv @@ -1,4 +1,4 @@ -result,dates,tree.connector.event_duration_sum,tree - a and tree - b +result,dates,tree - a - event_duration,tree - a and tree - b 2,{2012-01-01/2012-01-01},1,1 3,{},0,1 4,{2012-01-02/2012-01-02},1,1 diff --git a/backend/src/test/resources/tests/query/LOGICAL/AND_DURATION_SUM_MULTI_CONCEPT/expected.csv b/backend/src/test/resources/tests/query/LOGICAL/AND_DURATION_SUM_MULTI_CONCEPT/expected.csv index 46b8664c10..3f192c6956 100644 --- a/backend/src/test/resources/tests/query/LOGICAL/AND_DURATION_SUM_MULTI_CONCEPT/expected.csv +++ b/backend/src/test/resources/tests/query/LOGICAL/AND_DURATION_SUM_MULTI_CONCEPT/expected.csv @@ -1,4 +1,4 @@ -result,secondary,dates,tree1.connector.event_duration_sum,tree1 - a and tree2 - b +result,secondary,dates,tree1 - a - event_duration,tree1 - a and tree2 - b 1,1,{},0,1 1,2,{},0,1 1,3,{},0,1 diff --git a/backend/src/test/resources/tests/query/MERGE/AND_DURATION_SUM/expected.csv b/backend/src/test/resources/tests/query/MERGE/AND_DURATION_SUM/expected.csv index 32acd28e84..6c21ee6f85 100644 --- a/backend/src/test/resources/tests/query/MERGE/AND_DURATION_SUM/expected.csv +++ b/backend/src/test/resources/tests/query/MERGE/AND_DURATION_SUM/expected.csv @@ -1,4 +1,4 @@ -result,dates,tree.connector.event_duration_sum,tree - a and tree - b +result,dates,tree - a - event_duration,tree - a and tree - b 2,{2012-01-01/2012-01-01},1,1 3,{2012-01-01/2012-01-02},1,1 4,{2012-01-01/2012-01-02},2,1 diff --git a/backend/src/test/resources/tests/query/MERGE/AND_EVENT_DATE_EXCLUDED/expected.csv b/backend/src/test/resources/tests/query/MERGE/AND_EVENT_DATE_EXCLUDED/expected.csv index 17213a8f0e..802b298718 100644 --- a/backend/src/test/resources/tests/query/MERGE/AND_EVENT_DATE_EXCLUDED/expected.csv +++ b/backend/src/test/resources/tests/query/MERGE/AND_EVENT_DATE_EXCLUDED/expected.csv @@ -1,4 +1,4 @@ -result,dates,tree.connector.date_union +result,dates,tree - a - dates 2,{2012-01-01/2012-01-01},{2012-01-01/2012-01-01} 3,{2012-01-02/2012-01-02},{2012-01-01/2012-01-01} 4,{2012-01-02/2012-01-02},{2012-01-01/2012-01-02} diff --git a/backend/src/test/resources/tests/query/MERGE/OR_EVENT_DATE_EXCLUDED/expected.csv b/backend/src/test/resources/tests/query/MERGE/OR_EVENT_DATE_EXCLUDED/expected.csv index ca4cceb7c6..fb2aab296b 100644 --- a/backend/src/test/resources/tests/query/MERGE/OR_EVENT_DATE_EXCLUDED/expected.csv +++ b/backend/src/test/resources/tests/query/MERGE/OR_EVENT_DATE_EXCLUDED/expected.csv @@ -1,4 +1,4 @@ -result,dates,tree.connector.date_union +result,dates,tree - a - dates 1,{},{2012-01-01/2012-01-01} 2,{2012-01-01/2012-01-01},{2012-01-01/2012-01-01} 3,{2012-01-02/2012-01-02},{2012-01-01/2012-01-01} diff --git a/backend/src/test/resources/tests/query/MULTI_CONCEPT_QUERY_SEPARATE_DATES/expected.csv b/backend/src/test/resources/tests/query/MULTI_CONCEPT_QUERY_SEPARATE_DATES/expected.csv index c30a163fd1..28b3c9346a 100644 --- a/backend/src/test/resources/tests/query/MULTI_CONCEPT_QUERY_SEPARATE_DATES/expected.csv +++ b/backend/src/test/resources/tests/query/MULTI_CONCEPT_QUERY_SEPARATE_DATES/expected.csv @@ -1,3 +1,3 @@ -result,dates,test_tree.event-date,test_tree2.event-date +result,dates,test_tree - test_child1 - dates,test_tree2 - test_child1 - dates 1,{2012-01-01/2012-01-02},{2012-01-01/2012-01-01},{2012-01-02/2012-01-02} 3,{2013-11-10/2013-11-10},{2013-11-10/2013-11-10},{2013-11-10/2013-11-10} \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/MULTI_CONNECTOR_QUERY_SEPARATE_DATES/expected.csv b/backend/src/test/resources/tests/query/MULTI_CONNECTOR_QUERY_SEPARATE_DATES/expected.csv index 648bd8f7ee..bc799d5786 100644 --- a/backend/src/test/resources/tests/query/MULTI_CONNECTOR_QUERY_SEPARATE_DATES/expected.csv +++ b/backend/src/test/resources/tests/query/MULTI_CONNECTOR_QUERY_SEPARATE_DATES/expected.csv @@ -1,3 +1,3 @@ -result,dates,test_tree.test_column.event-dates +result,dates,test_tree - test_child1 - tree_label dates 1,{2012-01-01/2012-01-02},{2012-01-01/2012-01-01} 3,{2013-11-10/2013-11-10},{2013-11-10/2013-11-10} \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/REL_EXPORT/PREREQUISITE_WITHOUT_DATES/expected.csv b/backend/src/test/resources/tests/query/REL_EXPORT/PREREQUISITE_WITHOUT_DATES/expected.csv index 70fdd1554d..a157e6d6f2 100644 --- a/backend/src/test/resources/tests/query/REL_EXPORT/PREREQUISITE_WITHOUT_DATES/expected.csv +++ b/backend/src/test/resources/tests/query/REL_EXPORT/PREREQUISITE_WITHOUT_DATES/expected.csv @@ -1,2 +1,2 @@ -result,resolution,index,event_date,feature_date_range,outcome_date_range,concept.exists,concept.exists_1 +result,resolution,index,event_date,feature_date_range,outcome_date_range,concept - child1 - exists,concept - child1 - exists_1 0,,,,,,0,0 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/REL_EXPORT/PREREQUISITE_WITH_DATES/expected.csv b/backend/src/test/resources/tests/query/REL_EXPORT/PREREQUISITE_WITH_DATES/expected.csv index f2c4d76be9..e2c0a829a1 100644 --- a/backend/src/test/resources/tests/query/REL_EXPORT/PREREQUISITE_WITH_DATES/expected.csv +++ b/backend/src/test/resources/tests/query/REL_EXPORT/PREREQUISITE_WITH_DATES/expected.csv @@ -1,4 +1,4 @@ -result,resolution,index,event_date,feature_date_range,outcome_date_range,test_tree.exists,test_tree.exists_1 +result,resolution,index,event_date,feature_date_range,outcome_date_range,test_tree - test_child1 - exists,test_tree - test_child1 - exists_1 0,complete,,2000-06-01,1999-07-01/2000-06-30,2000-07-01/2001-06-30,1,1 0,quarter,-3,2000-06-01,1999-07-01/1999-09-30,,0, 0,quarter,-2,2000-06-01,1999-10-01/1999-12-31,,0, From d8e4a418e97b3a5b93b65f72f3d9817a13e67a89 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Tue, 3 Aug 2021 16:43:53 +0200 Subject: [PATCH 51/82] extracts api classes into api --- .../conquery/apiv1/QueryProcessor.java | 31 +-- .../conquery/apiv1/query/ExternalUpload.java | 31 +++ .../apiv1/query/ExternalUploadResult.java | 32 +++ .../conquery/resources/api/QueryResource.java | 194 ++++++++---------- 4 files changed, 164 insertions(+), 124 deletions(-) create mode 100644 backend/src/main/java/com/bakdata/conquery/apiv1/query/ExternalUpload.java create mode 100644 backend/src/main/java/com/bakdata/conquery/apiv1/query/ExternalUploadResult.java diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java index e82fd97885..1294f2afc5 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java @@ -13,14 +13,17 @@ import java.util.stream.Stream; import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.BadRequestException; import javax.ws.rs.core.Response; import javax.ws.rs.core.UriBuilder; import com.bakdata.conquery.apiv1.query.CQElement; import com.bakdata.conquery.apiv1.query.ConceptQuery; +import com.bakdata.conquery.apiv1.query.ExternalUpload; import com.bakdata.conquery.apiv1.query.Query; import com.bakdata.conquery.apiv1.query.QueryDescription; import com.bakdata.conquery.apiv1.query.SecondaryIdQuery; +import com.bakdata.conquery.apiv1.query.ExternalUploadResult; import com.bakdata.conquery.apiv1.query.concept.specific.CQAnd; import com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal; import com.bakdata.conquery.io.result.ResultRender.ResultRendererProvider; @@ -45,12 +48,10 @@ import com.bakdata.conquery.models.query.visitor.QueryVisitor; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.Namespace; -import com.bakdata.conquery.resources.api.QueryResource; import com.bakdata.conquery.util.QueryUtils; import com.bakdata.conquery.util.QueryUtils.NamespacedIdentifiableCollector; import com.google.common.collect.ClassToInstanceMap; import com.google.common.collect.MutableClassToInstanceMap; -import lombok.Data; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -362,20 +363,10 @@ public FullExecutionStatus getQueryFullStatus(ManagedExecution query, User us return status; } - @Data - public static class UploadResponse { - private final ManagedExecutionId execution; - - private final int resolved; - - private final List unresolvedId; - private final List unreadableDate; - } - /** - * Try to resolve the upload, if successful, create query for the user and return id and statistics for that. + * Try to resolve the external upload, if successful, create query for the user and return id and statistics for that. */ - public Response uploadEntities(User user, Dataset dataset, QueryResource.ExternalUpload upload) { + public ExternalUploadResult uploadEntities(User user, Dataset dataset, ExternalUpload upload) { final CQExternal.ResolveStatistic statistic = CQExternal.resolveEntities(upload.getValues(), upload.getFormat(), @@ -386,9 +377,9 @@ public Response uploadEntities(User user, Dataset dataset, QueryResource.Externa // Resolving nothing is a problem thus we fail. if (statistic.getResolved().isEmpty()) { - return Response.status(Response.Status.BAD_REQUEST) - .entity(new UploadResponse(null, 0, statistic.getUnresolvedId(), statistic.getUnreadableDate())) - .build(); + throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST) + .entity(new ExternalUploadResult(null, 0, statistic.getUnresolvedId(), statistic.getUnreadableDate())) + .build()); } final ConceptQuery query = new ConceptQuery(new CQExternal(upload.getFormat(), upload.getValues())); @@ -398,13 +389,11 @@ public Response uploadEntities(User user, Dataset dataset, QueryResource.Externa datasetRegistry.get(dataset.getId()).getExecutionManager() .createExecution(datasetRegistry, query, user, dataset); - return Response.ok() - .entity(new UploadResponse( + return new ExternalUploadResult( execution.getId(), statistic.getResolved().size(), statistic.getUnresolvedId(), statistic.getUnreadableDate() - )) - .build(); + ); } } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/ExternalUpload.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/ExternalUpload.java new file mode 100644 index 0000000000..c73f9dc494 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/ExternalUpload.java @@ -0,0 +1,31 @@ +package com.bakdata.conquery.apiv1.query; + +import java.util.Arrays; +import java.util.List; + +import com.bakdata.conquery.models.auth.entities.User; +import com.fasterxml.jackson.annotation.JsonIgnore; +import io.dropwizard.validation.ValidationMethod; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + + +/** + * Data Container for API call of {@link com.bakdata.conquery.resources.api.QueryResource#upload(User, ExternalUpload)}. + * + * This class acts as a wrapper for {@link com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal}. + */ +@RequiredArgsConstructor +@Getter +public class ExternalUpload { + + private final List format; + private final String[][] values; + + @JsonIgnore + @ValidationMethod(message = "Values and Format are not of same width.") + public boolean isAllSameLength() { + final int expected = format.size(); + return Arrays.stream(values).mapToInt(a -> a.length).allMatch(v -> expected == v); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/ExternalUploadResult.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/ExternalUploadResult.java new file mode 100644 index 0000000000..baf88f3aed --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/ExternalUploadResult.java @@ -0,0 +1,32 @@ +package com.bakdata.conquery.apiv1.query; + +import java.util.List; + +import com.bakdata.conquery.models.auth.entities.User; +import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; +import lombok.Data; + +/** + * Result Container for responses of {@link com.bakdata.conquery.resources.api.QueryResource#upload(User, ExternalUpload)}. + */ +@Data +public class ExternalUploadResult { + /** + * Id of created execution of uploaded data. + */ + private final ManagedExecutionId execution; + + /** + * Number of successfully resolved rows. + */ + private final int resolved; + + /** + * Content of rows where no Id could be resolved. + */ + private final List unresolvedId; + /** + * Content of rows where dates could not be read. + */ + private final List unreadableDate; +} diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java index a1f42e0200..4f310ad778 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java @@ -1,40 +1,47 @@ package com.bakdata.conquery.resources.api; -import com.bakdata.conquery.apiv1.*; -import com.bakdata.conquery.apiv1.query.QueryDescription; -import com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal; -import com.bakdata.conquery.models.auth.entities.User; -import com.bakdata.conquery.models.auth.permissions.Ability; -import com.bakdata.conquery.models.datasets.Dataset; -import com.bakdata.conquery.models.exceptions.JSONException; -import com.bakdata.conquery.models.execution.ManagedExecution; -import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; -import com.fasterxml.jackson.annotation.JsonIgnore; -import io.dropwizard.auth.Auth; -import io.dropwizard.jersey.PATCH; -import io.dropwizard.validation.ValidationMethod; -import lombok.Getter; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; +import static com.bakdata.conquery.resources.ResourceConstants.DATASET; +import static com.bakdata.conquery.resources.ResourceConstants.QUERY; + +import java.util.List; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import javax.inject.Inject; import javax.servlet.http.HttpServletRequest; import javax.validation.Valid; import javax.validation.constraints.NotNull; -import javax.ws.rs.*; +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; -import java.util.Arrays; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -import static com.bakdata.conquery.resources.ResourceConstants.DATASET; -import static com.bakdata.conquery.resources.ResourceConstants.QUERY; +import com.bakdata.conquery.apiv1.AdditionalMediaTypes; +import com.bakdata.conquery.apiv1.ExecutionStatus; +import com.bakdata.conquery.apiv1.FullExecutionStatus; +import com.bakdata.conquery.apiv1.MetaDataPatch; +import com.bakdata.conquery.apiv1.QueryProcessor; +import com.bakdata.conquery.apiv1.RequestAwareUriBuilder; +import com.bakdata.conquery.apiv1.query.ExternalUpload; +import com.bakdata.conquery.apiv1.query.ExternalUploadResult; +import com.bakdata.conquery.apiv1.query.QueryDescription; +import com.bakdata.conquery.models.auth.entities.User; +import com.bakdata.conquery.models.auth.permissions.Ability; +import com.bakdata.conquery.models.datasets.Dataset; +import com.bakdata.conquery.models.exceptions.JSONException; +import com.bakdata.conquery.models.execution.ManagedExecution; +import io.dropwizard.auth.Auth; +import io.dropwizard.jersey.PATCH; +import lombok.extern.slf4j.Slf4j; @Path("datasets/{" + DATASET + "}/queries") @Consumes(AdditionalMediaTypes.JSON) @@ -42,112 +49,93 @@ @Slf4j public class QueryResource { - @Inject - private QueryProcessor processor; + @Inject + private QueryProcessor processor; - @Context - protected HttpServletRequest servletRequest; + @Context + protected HttpServletRequest servletRequest; - @PathParam(DATASET) - private Dataset dataset; + @PathParam(DATASET) + private Dataset dataset; - @GET - public List getAllQueries(@Auth User user, @QueryParam("all-providers") Optional allProviders) { + @GET + public List getAllQueries(@Auth User user, @QueryParam("all-providers") Optional allProviders) { user.authorize(dataset, Ability.READ); return processor.getAllQueries(dataset, servletRequest, user, allProviders.orElse(false)) - .collect(Collectors.toList()); - } - - @POST - public Response postQuery(@Auth User user, @QueryParam("all-providers") Optional allProviders, @NotNull @Valid QueryDescription query) { - - user.authorize(dataset, Ability.READ); - - ManagedExecution execution = processor.postQuery(dataset, query, user); + .collect(Collectors.toList()); + } - return Response.ok(processor.getQueryFullStatus(execution, user, RequestAwareUriBuilder.fromRequest(servletRequest), allProviders.orElse(false))) - .status(Status.CREATED) - .build(); - } + @POST + public Response postQuery(@Auth User user, @QueryParam("all-providers") Optional allProviders, @NotNull @Valid QueryDescription query) { - @GET - @Path("{" + QUERY + "}") - public FullExecutionStatus getStatus(@Auth User user, @PathParam(QUERY) ManagedExecution query, @QueryParam("all-providers") Optional allProviders) - throws InterruptedException { + user.authorize(dataset, Ability.READ); - user.authorize(dataset, Ability.READ); - user.authorize(query, Ability.READ); + ManagedExecution execution = processor.postQuery(dataset, query, user); - query.awaitDone(1, TimeUnit.SECONDS); + return Response.ok(processor.getQueryFullStatus(execution, user, RequestAwareUriBuilder.fromRequest(servletRequest), allProviders.orElse(false))) + .status(Status.CREATED) + .build(); + } - return processor.getQueryFullStatus(query, user, RequestAwareUriBuilder.fromRequest(servletRequest), allProviders.orElse(false)); - } + @GET + @Path("{" + QUERY + "}") + public FullExecutionStatus getStatus(@Auth User user, @PathParam(QUERY) ManagedExecution query, @QueryParam("all-providers") Optional allProviders) + throws InterruptedException { - @PATCH - @Path("{" + QUERY + "}") - public FullExecutionStatus patchQuery(@Auth User user, @PathParam(QUERY) ManagedExecution query, @QueryParam("all-providers") Optional allProviders, MetaDataPatch patch) throws JSONException { - user.authorize(dataset, Ability.READ); - user.authorize(query, Ability.READ); + user.authorize(dataset, Ability.READ); + user.authorize(query, Ability.READ); - processor.patchQuery(user, query, patch); + query.awaitDone(1, TimeUnit.SECONDS); - return processor.getQueryFullStatus(query, user, RequestAwareUriBuilder.fromRequest(servletRequest), allProviders.orElse(false)); - } + return processor.getQueryFullStatus(query, user, RequestAwareUriBuilder.fromRequest(servletRequest), allProviders.orElse(false)); + } - @DELETE - @Path("{" + QUERY + "}") - public void deleteQuery(@Auth User user, @PathParam(QUERY) ManagedExecution query) { - user.authorize(dataset, Ability.READ); - user.authorize(query, Ability.DELETE); + @PATCH + @Path("{" + QUERY + "}") + public FullExecutionStatus patchQuery(@Auth User user, @PathParam(QUERY) ManagedExecution query, @QueryParam("all-providers") Optional allProviders, MetaDataPatch patch) + throws JSONException { + user.authorize(dataset, Ability.READ); + user.authorize(query, Ability.READ); - processor.deleteQuery(user, query); - } + processor.patchQuery(user, query, patch); - @POST - @Path("{" + QUERY + "}/reexecute") - public FullExecutionStatus reexecute(@Auth User user, @PathParam(QUERY) ManagedExecution query, @QueryParam("all-providers") Optional allProviders) { - user.authorize(dataset, Ability.READ); - user.authorize(query, Ability.READ); + return processor.getQueryFullStatus(query, user, RequestAwareUriBuilder.fromRequest(servletRequest), allProviders.orElse(false)); + } - processor.reexecute(user, query); - return processor.getQueryFullStatus(query, user, RequestAwareUriBuilder.fromRequest(servletRequest), allProviders.orElse(false)); - } + @DELETE + @Path("{" + QUERY + "}") + public void deleteQuery(@Auth User user, @PathParam(QUERY) ManagedExecution query) { + user.authorize(dataset, Ability.READ); + user.authorize(query, Ability.DELETE); - @POST - @Path("{" + QUERY + "}/cancel") - public void cancel(@Auth User user, @PathParam(QUERY) ManagedExecution query) { + processor.deleteQuery(user, query); + } - user.authorize(dataset, Ability.READ); - user.authorize(query, Ability.CANCEL); + @POST + @Path("{" + QUERY + "}/reexecute") + public FullExecutionStatus reexecute(@Auth User user, @PathParam(QUERY) ManagedExecution query, @QueryParam("all-providers") Optional allProviders) { + user.authorize(dataset, Ability.READ); + user.authorize(query, Ability.READ); - processor.cancel( - user, - dataset, - query - ); - } + processor.reexecute(user, query); + return processor.getQueryFullStatus(query, user, RequestAwareUriBuilder.fromRequest(servletRequest), allProviders.orElse(false)); + } - @RequiredArgsConstructor - @Getter - public static class ExternalUpload { - private final List format; - private final String[][] values; + @POST + @Path("{" + QUERY + "}/cancel") + public void cancel(@Auth User user, @PathParam(QUERY) ManagedExecution query) { - private final ManagedExecutionId executionId; + user.authorize(dataset, Ability.READ); + user.authorize(query, Ability.CANCEL); - @JsonIgnore - @ValidationMethod(message = "Values and Format are not of same width.") - public boolean isAllSameLength() { - final int expected = format.size(); - return Arrays.stream(values).mapToInt(a -> a.length).allMatch(v -> expected == v); - } + processor.cancel(user, dataset, query); } @POST @Path("/upload") - public Response upload(@Auth User user, @Valid ExternalUpload upload) { - return processor.uploadEntities(user, dataset, upload); + public ExternalUploadResult upload(@Auth User user, @Valid ExternalUpload upload) { + return processor.uploadEntities(user, dataset, upload); } } From 938cde80269b88639416f54cc6048363963745a5 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Tue, 3 Aug 2021 16:57:01 +0200 Subject: [PATCH 52/82] Lazy get position of Column --- .../conquery/models/datasets/Column.java | 7 +- .../conquery/models/datasets/Table.java | 6 - .../admin/rest/AdminDatasetProcessor.java | 2 - .../integration/common/RequiredTable.java | 2 - .../conquery/models/SerializationTests.java | 104 +++++++++--------- 5 files changed, 59 insertions(+), 62 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java index 2c16f0f26a..b994063fc2 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.models.datasets; +import java.util.Arrays; import java.util.Map; import java.util.Objects; @@ -24,6 +25,7 @@ import lombok.Getter; import lombok.Setter; import lombok.ToString; +import org.apache.commons.lang3.ArrayUtils; @Getter @Setter @@ -39,8 +41,9 @@ public class Column extends Labeled implements NamespacedIdentifiable< @NotNull private MajorTypeId type; - @InternalOnly - private int position = UNKNOWN_POSITION; + @JsonIgnore + @Getter(lazy = true) + private final int position = ArrayUtils.indexOf(getTable().getColumns(), this); /** * if set this column should use the given dictionary * if it is of type string, instead of its own dictionary diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/Table.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/Table.java index 4bb7a5c864..6511e7484a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/Table.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/Table.java @@ -34,12 +34,6 @@ public class Table extends Labeled implements NamespacedIdentifiable col.toColumn(table, centralRegistry)).toArray(Column[]::new)); - table.init(); - return table; } diff --git a/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java b/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java index dec9bdb156..5b2a1b141c 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java +++ b/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java @@ -2,7 +2,11 @@ import java.io.IOException; import java.time.LocalDate; -import java.util.*; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.UUID; import com.bakdata.conquery.apiv1.IdLabel; import com.bakdata.conquery.apiv1.MeProcessor; @@ -11,6 +15,8 @@ import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; import com.bakdata.conquery.apiv1.query.ArrayConceptQuery; import com.bakdata.conquery.apiv1.query.ConceptQuery; +import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; +import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; import com.bakdata.conquery.apiv1.query.concept.specific.CQOr; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.jackson.serializer.SerializationTestUtil; @@ -22,12 +28,12 @@ import com.bakdata.conquery.models.auth.permissions.DatasetPermission; import com.bakdata.conquery.models.auth.permissions.ExecutionPermission; import com.bakdata.conquery.models.common.daterange.CDateRange; -import com.bakdata.conquery.models.datasets.concepts.ValidityDate; -import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeConnector; -import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.datasets.Table; +import com.bakdata.conquery.models.datasets.concepts.ValidityDate; +import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeConnector; +import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; import com.bakdata.conquery.models.error.ConqueryError; import com.bakdata.conquery.models.events.MajorTypeId; import com.bakdata.conquery.models.exceptions.JSONException; @@ -43,8 +49,6 @@ import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; import com.bakdata.conquery.models.query.ManagedQuery; -import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; -import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; import com.bakdata.conquery.models.query.entity.Entity; import com.bakdata.conquery.util.NonPersistentStoreFactory; import com.fasterxml.jackson.databind.JsonNode; @@ -61,40 +65,41 @@ public void dataset() throws IOException, JSONException { dataset.setName("dataset"); SerializationTestUtil - .forType(Dataset.class) - .test(dataset); + .forType(Dataset.class) + .test(dataset); } @Test - public void passwordCredential() throws IOException, JSONException{ + public void passwordCredential() throws IOException, JSONException { PasswordCredential credential = new PasswordCredential(new String("testPassword").toCharArray()); SerializationTestUtil - .forType(PasswordCredential.class) - .test(credential); + .forType(PasswordCredential.class) + .test(credential); } @Test - public void role() throws IOException, JSONException{ + public void role() throws IOException, JSONException { Role mandator = new Role("company", "company"); SerializationTestUtil - .forType(Role.class) - .test(mandator); + .forType(Role.class) + .test(mandator); } /* * Only way to add permission without a storage. */ @Test - public void user() throws IOException, JSONException{ - MetaStorage storage = new MetaStorage(null, new NonPersistentStoreFactory(), null); + public void user() throws IOException, JSONException { + MetaStorage storage = new MetaStorage(null, new NonPersistentStoreFactory(), null); User user = new User("user", "user"); user.addPermission(storage, DatasetPermission.onInstance(Ability.READ, new DatasetId("test"))); user - .addPermission( - storage, - ExecutionPermission.onInstance(Ability.READ, new ManagedExecutionId(new DatasetId("dataset"), UUID.randomUUID()))); + .addPermission( + storage, + ExecutionPermission.onInstance(Ability.READ, new ManagedExecutionId(new DatasetId("dataset"), UUID.randomUUID())) + ); Role role = new Role("company", "company"); user.addRole(storage, role); @@ -102,9 +107,9 @@ public void user() throws IOException, JSONException{ registry.register(role); SerializationTestUtil - .forType(User.class) - .registry(registry) - .test(user); + .forType(User.class) + .registry(registry) + .test(user); } @Test @@ -113,9 +118,10 @@ public void group() throws IOException, JSONException { Group group = new Group("group", "group"); group.addPermission(storage, DatasetPermission.onInstance(Ability.READ, new DatasetId("test"))); group - .addPermission( - storage, - ExecutionPermission.onInstance(Ability.READ, new ManagedExecutionId(new DatasetId("dataset"), UUID.randomUUID()))); + .addPermission( + storage, + ExecutionPermission.onInstance(Ability.READ, new ManagedExecutionId(new DatasetId("dataset"), UUID.randomUUID())) + ); group.addRole(storage, new Role("company", "company")); Role role = new Role("company", "company"); @@ -131,7 +137,7 @@ public void group() throws IOException, JSONException { } @Test - public void treeConcept() throws IOException, JSONException{ + public void treeConcept() throws IOException, JSONException { Dataset dataset = new Dataset(); dataset.setName("datasetName"); @@ -145,19 +151,16 @@ public void treeConcept() throws IOException, JSONException{ Column column = new Column(); column.setLabel("colLabel"); column.setName("colName"); - column.setPosition(0); column.setType(MajorTypeId.STRING); column.setTable(table); Column dateColumn = new Column(); dateColumn.setLabel("colLabel2"); dateColumn.setName("colName2"); - dateColumn.setPosition(1); dateColumn.setType(MajorTypeId.DATE); dateColumn.setTable(table); - table.setColumns(new Column[]{column, dateColumn}); table.setDataset(dataset); table.setLabel("tableLabel"); @@ -191,15 +194,15 @@ public void treeConcept() throws IOException, JSONException{ registry.register(valDate); SerializationTestUtil - .forType(TreeConcept.class) - .registry(registry) - .test(concept); + .forType(TreeConcept.class) + .registry(registry) + .test(concept); } @Test public void persistentIdMap() throws JSONException, IOException { SerializationTestUtil.forType(EntityIdMap.class) - .test(IdMapSerialisationTest.createTestPersistentMap()); + .test(IdMapSerialisationTest.createTestPersistentMap()); } @@ -223,9 +226,9 @@ public void formConfig() throws JSONException, IOException { formConfig.setDataset(dataset); SerializationTestUtil - .forType(FormConfig.class) - .registry(registry) - .test(formConfig); + .forType(FormConfig.class) + .registry(registry) + .test(formConfig); } @Test @@ -235,13 +238,13 @@ public void managedQuery() throws JSONException, IOException { final Dataset dataset = new Dataset("test-dataset"); - final User user = new User("test-user","test-user"); + final User user = new User("test-user", "test-user"); registry.register(dataset); registry.register(user); ManagedQuery execution = new ManagedQuery(null, user, dataset); - execution.setTags(new String[] {"test-tag"}); + execution.setTags(new String[]{"test-tag"}); SerializationTestUtil.forType(ManagedExecution.class) .registry(registry) @@ -289,8 +292,8 @@ public void executionCreationPlanError() throws JSONException, IOException { ConqueryError error = new ConqueryError.ExecutionCreationPlanError(); SerializationTestUtil - .forType(ConqueryError.class) - .test(error); + .forType(ConqueryError.class) + .test(error); } @Test @@ -298,15 +301,15 @@ public void executionCreationResolveError() throws JSONException, IOException { ConqueryError error = new ConqueryError.ExecutionCreationResolveError(new DatasetId("test")); SerializationTestUtil - .forType(ConqueryError.class) - .test(error); + .forType(ConqueryError.class) + .test(error); } @Test public void executionQueryJobError() throws JSONException, IOException { log.info("Beware, this test will print an ERROR message."); - ConqueryError error = new ConqueryError.ExecutionJobErrorWrapper(new Entity(5),new ConqueryError.UnknownError(null)); + ConqueryError error = new ConqueryError.ExecutionJobErrorWrapper(new Entity(5), new ConqueryError.UnknownError(null)); SerializationTestUtil .forType(ConqueryError.class) @@ -318,11 +321,11 @@ public void meInformation() throws IOException, JSONException { User user = new User("name", "labe"); MeProcessor.FEMeInformation info = MeProcessor.FEMeInformation.builder() - .userName(user.getLabel()) - .hideLogoutButton(false) - .groups(List.of(new IdLabel<>(new GroupId("test_group"), "test_group_label"))) - .datasetAbilities(Map.of(new DatasetId("testdataset"), new MeProcessor.FEDatasetAbility(true))) - .build(); + .userName(user.getLabel()) + .hideLogoutButton(false) + .groups(List.of(new IdLabel<>(new GroupId("test_group"), "test_group_label"))) + .datasetAbilities(Map.of(new DatasetId("testdataset"), new MeProcessor.FEDatasetAbility(true))) + .build(); SerializationTestUtil .forType(MeProcessor.FEMeInformation.class) @@ -344,7 +347,7 @@ public void testFormQuery() throws IOException, JSONException { testConcept.setConnectors(List.of(connector)); concept.setElements(Collections.singletonList(testConcept)); - CQTable[] tables = {new CQTable() }; + CQTable[] tables = {new CQTable()}; connector.setTable(new Table()); tables[0].setConnector(connector); tables[0].setConcept(concept); @@ -362,7 +365,8 @@ public void testFormQuery() throws IOException, JSONException { ArrayConceptQuery.createFromFeatures(Collections.singletonList(features)), List.of( ExportForm.ResolutionAndAlignment.of(DateContext.Resolution.COMPLETE, DateContext.Alignment.NO_ALIGN), - ExportForm.ResolutionAndAlignment.of(DateContext.Resolution.QUARTERS, DateContext.Alignment.QUARTER)) + ExportForm.ResolutionAndAlignment.of(DateContext.Resolution.QUARTERS, DateContext.Alignment.QUARTER) + ) ); CentralRegistry centralRegistry = new CentralRegistry(); From b444e1b4dab05b5e5cf84073d359b302c49e2eea Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Tue, 3 Aug 2021 17:20:37 +0200 Subject: [PATCH 53/82] Fix START/END and provide test --- .../concept/specific/external/CQExternal.java | 2 +- .../concept/specific/external/DateFormat.java | 26 +++++++- ...IMPLE_CQEXTERNAL_QUERY_START_END.test.json | 66 +++++++++++++++++++ .../expected_start_end.csv | 3 + 4 files changed, 94 insertions(+), 3 deletions(-) create mode 100644 backend/src/test/resources/tests/query/SIMPLE_CQEXTERNAL_QUERY/SIMPLE_CQEXTERNAL_QUERY_START_END.test.json create mode 100644 backend/src/test/resources/tests/query/SIMPLE_CQEXTERNAL_QUERY/expected_start_end.csv diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java index e76f6d2390..b212de98ec 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java @@ -106,7 +106,7 @@ private static Int2ObjectMap readDates(String[][] values, List continue; } - out.put(row, dates); + out.computeIfAbsent(row, (ignored) -> CDateSet.create()).addAll(dates); } catch (Exception e) { log.warn("Failed to parse Date from {}", row, e); diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java index 1ce4206ede..75c180bae5 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java @@ -19,13 +19,35 @@ public void readDates(String value, DateReader dateReader, CDateSet out) { END_DATE { @Override public void readDates(String value, DateReader dateReader, CDateSet out) { - out.add(CDateRange.atMost(dateReader.parseToLocalDate(value))); + final CDateRange parsed = CDateRange.atMost(dateReader.parseToLocalDate(value)); + + if(out.isEmpty()){ + out.add(parsed); + return; + } + + final CDateRange span = out.span(); + + out.clear(); + + out.add(span.spanClosed(parsed)); } }, START_DATE { @Override public void readDates(String value, DateReader dateReader, CDateSet out) { - out.add(CDateRange.atLeast(dateReader.parseToLocalDate(value))); + final CDateRange parsed = CDateRange.atLeast(dateReader.parseToLocalDate(value)); + + if(out.isEmpty()){ + out.add(parsed); + return; + } + + final CDateRange span = out.span(); + + out.clear(); + + out.add(span.spanClosed(parsed)); } }, DATE_RANGE { diff --git a/backend/src/test/resources/tests/query/SIMPLE_CQEXTERNAL_QUERY/SIMPLE_CQEXTERNAL_QUERY_START_END.test.json b/backend/src/test/resources/tests/query/SIMPLE_CQEXTERNAL_QUERY/SIMPLE_CQEXTERNAL_QUERY_START_END.test.json new file mode 100644 index 0000000000..53970ad8d3 --- /dev/null +++ b/backend/src/test/resources/tests/query/SIMPLE_CQEXTERNAL_QUERY/SIMPLE_CQEXTERNAL_QUERY_START_END.test.json @@ -0,0 +1,66 @@ +{ + "type": "QUERY_TEST", + "label": "SIMPLE_CQEXTERNAL_QUERY Test", + "expectedCsv": "tests/query/SIMPLE_CQEXTERNAL_QUERY/expected_start_end.csv", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "type": "EXTERNAL", + "format": [ + "ID", + "START_DATE", + "END_DATE" + ], + "values": [ + [ + "result", + "start", + "end" + ], + [ + 1, + "2012-01-01", + "2012-01-10" + ], + [ + 3, + "2013-11-10", + "2013-11-20" + ] + ] + } + }, + "concepts": [ + { + "label": "test_tree", + "type": "TREE", + "connectors": { + "label": "tree_label", + "name": "test_concept", + "table": "test_table", + "validityDates": { + "label": "datum", + "column": "test_table.datum" + } + } + } + ], + "content": { + "tables": [ + { + "csv": "tests/query/SIMPLE_CQEXTERNAL_QUERY/content.csv", + "name": "test_table", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "datum", + "type": "DATE" + } + ] + } + ] + } +} diff --git a/backend/src/test/resources/tests/query/SIMPLE_CQEXTERNAL_QUERY/expected_start_end.csv b/backend/src/test/resources/tests/query/SIMPLE_CQEXTERNAL_QUERY/expected_start_end.csv new file mode 100644 index 0000000000..b90197cfad --- /dev/null +++ b/backend/src/test/resources/tests/query/SIMPLE_CQEXTERNAL_QUERY/expected_start_end.csv @@ -0,0 +1,3 @@ +result,dates +1,{2012-01-01/2012-01-10} +3,{2013-11-10/2013-11-20} \ No newline at end of file From 022ca068e8bf0a0f905980b8cc63eb6d1fa9ac92 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Tue, 3 Aug 2021 17:21:43 +0200 Subject: [PATCH 54/82] delete unused StoreInfo --- .../src/main/java/com/bakdata/conquery/io/storage/StoreInfo.java | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/StoreInfo.java b/backend/src/main/java/com/bakdata/conquery/io/storage/StoreInfo.java index 1a97525fe1..98a27ab30c 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/StoreInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/StoreInfo.java @@ -54,7 +54,6 @@ public enum StoreInfo implements IStoreInfo { DATASET(Dataset.class, Boolean.class), ID_MAPPING(EntityIdMap.class, Boolean.class), NAMESPACES(DatasetRegistry.class, Boolean.class), - SLAVE(ShardNodeInformation.class, Boolean.class), DICTIONARIES(Dictionary.class, DictionaryId.class), IMPORTS(Import.class, ImportId.class), SECONDARY_IDS(SecondaryIdDescription.class, SecondaryIdDescriptionId.class), From 1021ce88ea97630ee91a126ae4991486f2b9a4f5 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Tue, 3 Aug 2021 18:02:03 +0200 Subject: [PATCH 55/82] adds much needed documentation to ColumnConfig --- .../conquery/models/config/ColumnConfig.java | 30 ++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java index 18c70fe764..bf3439ab81 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java @@ -10,6 +10,7 @@ import c10n.C10N; import com.bakdata.conquery.io.jackson.InternalOnly; import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; +import com.bakdata.conquery.resources.admin.rest.AdminDatasetProcessor; import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.base.Strings; import io.dropwizard.validation.ValidationMethod; @@ -21,6 +22,11 @@ import lombok.ToString; import org.apache.commons.lang3.StringUtils; +/** + * Configuration class for QueryUpload and IdMapping. + * + * Describes how rows are mapped for {@link EntityIdMap}/{@link AdminDatasetProcessor#setIdMapping(java.io.InputStream, com.bakdata.conquery.models.worker.Namespace)}. + */ @Builder @AllArgsConstructor @ToString @@ -48,7 +54,9 @@ public EntityIdMap.ExternalId read(String value) { } - + /** + * Name of the Column-Config to be used when resolving with Upload. + */ @NotEmpty private String name; @@ -65,24 +73,44 @@ public EntityIdMap.ExternalId read(String value) { @Builder.Default private Map description = Collections.emptyMap(); + /** + * Name of column in csv for {@link AdminDatasetProcessor#setIdMapping(java.io.InputStream, com.bakdata.conquery.models.worker.Namespace)}. + * + * Also Name of output column for {@link FrontendConfig.UploadConfig#getPrintIdFields()}, ergo output csv-Columns. + */ @InternalOnly private String field; + /** + * Pad-String when uploading data, this avoids problems with some tools truncating leading zeros or similar. + */ @InternalOnly private String pad = null; + /** + * In conjunction with pad, the length of the padded string. + */ @InternalOnly @Builder.Default private int length = -1; + /** + * Set to true, if the column should be resolvable in upload. This can be used to add supplemental information to an entity, for example it's data-source, which would not be unique among entities. + */ @InternalOnly @Builder.Default private boolean resolvable = false; + /** + * Set to true, if the Column should be printed to output. This can be used to have resolvable but not printable fields in mapping. + */ @InternalOnly @Builder.Default private boolean print = true; + /** + * Used in conjunction with {@link com.bakdata.conquery.models.identifiable.mapping.AutoIncrementingPseudomizer}: One column is required to have fillAnon true, which will be filled with pseudomized data. + */ @InternalOnly @Builder.Default private boolean fillAnon = false; From d393bf3a7a74bd6f1478982a7f1d9c1da61ed05a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 3 Aug 2021 16:03:42 +0000 Subject: [PATCH 56/82] Update AutoDoc --- docs/REST API JSONs.md | 18 +++++++++--------- docs/Table JSONs.md | 9 ++++----- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/docs/REST API JSONs.md b/docs/REST API JSONs.md index f4c8e5be55..d8d152f2eb 100644 --- a/docs/REST API JSONs.md +++ b/docs/REST API JSONs.md @@ -103,7 +103,7 @@ Returns: [ResolvedConceptsResult](#Type-ResolvedConceptsResult)

-### GET datasets/{dataset}/queries [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L54) +### GET datasets/{dataset}/queries [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L61)
Details

@@ -117,7 +117,7 @@ Returns: list of [ExecutionStatus](#Type-ExecutionStatus)

-### POST datasets/{dataset}/queries [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L63) +### POST datasets/{dataset}/queries [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L70)
Details

@@ -132,7 +132,7 @@ Returns: `Response`

-### POST datasets/{dataset}/queries/upload [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L148) +### POST datasets/{dataset}/queries/upload [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L136)
Details

@@ -142,11 +142,11 @@ Java Type: `com.bakdata.conquery.resources.api.QueryResource` Method: `upload` Expects: `@Valid ExternalUpload` -Returns: `Response` +Returns: `ExternalUploadResult`

-### GET datasets/{dataset}/queries/{query} [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L75) +### GET datasets/{dataset}/queries/{query} [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L82)
Details

@@ -160,7 +160,7 @@ Returns: [FullExecutionStatus](#Type-FullExecutionStatus)

-### PATCH datasets/{dataset}/queries/{query} [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L88) +### PATCH datasets/{dataset}/queries/{query} [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L95)
Details

@@ -175,7 +175,7 @@ Returns: [FullExecutionStatus](#Type-FullExecutionStatus)

-### DELETE datasets/{dataset}/queries/{query} [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L99) +### DELETE datasets/{dataset}/queries/{query} [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L107)
Details

@@ -188,7 +188,7 @@ Returns: `void`

-### POST datasets/{dataset}/queries/{query}/cancel [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L118) +### POST datasets/{dataset}/queries/{query}/cancel [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L126)
Details

@@ -201,7 +201,7 @@ Returns: `void`

-### POST datasets/{dataset}/queries/{query}/reexecute [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L108) +### POST datasets/{dataset}/queries/{query}/reexecute [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java#L116)
Details

diff --git a/docs/Table JSONs.md b/docs/Table JSONs.md index 070b05bba8..ee944a78ce 100644 --- a/docs/Table JSONs.md +++ b/docs/Table JSONs.md @@ -11,7 +11,7 @@ Each `*.table.json` has to contain exactly one [Tabel](#Type-Tabel). ## Other Types -### Type Column [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java#L28) +### Type Column [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java#L30)

Details

@@ -22,10 +22,9 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java#L42) | position | `int` | ␀ | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java#L49-L52) | secondaryId | ID of `@NsIdRef SecondaryIdDescription` | `null` | | if this is set this column counts as the secondary id of the given name for this table | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java#L44-L47) | sharedDictionary | `String` | `null` | | if set this column should use the given dictionary if it is of type string, instead of its own dictionary | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java#L39) | type | one of STRING, INTEGER, BOOLEAN, REAL, DECIMAL, MONEY, DATE, DATE_RANGE | `null` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java#L52-L55) | secondaryId | ID of `@NsIdRef SecondaryIdDescription` | `null` | | if this is set this column counts as the secondary id of the given name for this table | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java#L47-L50) | sharedDictionary | `String` | `null` | | if set this column should use the given dictionary if it is of type string, instead of its own dictionary | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java#L41) | type | one of STRING, INTEGER, BOOLEAN, REAL, DECIMAL, MONEY, DATE, DATE_RANGE | `null` | | | | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/identifiable/Labeled.java#L25-L29) | label | `String` | `null` | "someLabel" | shown in the frontend | | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/identifiable/NamedImpl.java#L17) | name | `String` | `null` | | |

From fe8c1a504124ed1ba20a6bf36dcbbd930b776f66 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Wed, 4 Aug 2021 11:20:56 +0200 Subject: [PATCH 57/82] remove unused declarations and cleanup validation --- .../models/config/FrontendConfig.java | 59 ++++--------------- 1 file changed, 10 insertions(+), 49 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java index 0d5a0ff920..a2050fc110 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java @@ -1,10 +1,13 @@ package com.bakdata.conquery.models.config; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -113,65 +116,23 @@ public int getIdIndex(List format) { return -1; } - @NotNull - @Valid - private ColumnConfig dateStart = ColumnConfig.builder() - .name(DateFormat.START_DATE.name()) - .label(Map.of("en", "Begin")) - .description(Map.of("en", "Begin of Date-range")) - .build(); - - @NotNull - @Valid - private ColumnConfig dateEnd = ColumnConfig.builder() - .name(DateFormat.END_DATE.name()) - .label(Map.of("en", "End")) - .description(Map.of("en", "End of Date-range")) - .build(); - - - @NotNull - @Valid - private ColumnConfig dateRange = ColumnConfig.builder() - .name(DateFormat.DATE_RANGE.name()) - .label(Map.of("en", "Date Range")) - .description(Map.of("en", "Full Date Range")) - .build(); - - - @NotNull - @Valid - private ColumnConfig dateSet = ColumnConfig.builder() - .name(DateFormat.DATE_SET.name()) - .label(Map.of("en", "Dateset")) - .description(Map.of("en", "Set of Date-Ranges")) - .build(); - - - @NotNull - @Valid - private ColumnConfig eventDate = ColumnConfig.builder() - .name(DateFormat.EVENT_DATE.name()) - .label(Map.of("en", "Event Date")) - .description(Map.of("en", "Single event")) - .build(); @ValidationMethod(message = "Duplicate Claims for Mapping Columns.") @JsonIgnore public boolean isAllColsUnique() { - Map dupes = new HashMap<>(); + Set dupes = new HashSet<>(); - final ArrayList candidates = new ArrayList<>(ids); - candidates.addAll(List.of(dateStart, dateEnd, dateSet, dateRange, eventDate)); + final List candidates = new ArrayList<>(); - for (ColumnConfig config : candidates) { - final ColumnConfig prior = dupes.put(config.getName(), config); + ids.stream().map(ColumnConfig::getName).forEach(candidates::add); + Arrays.stream(DateFormat.values()).map(DateFormat::name).forEach(candidates::add); - if (prior == null) { + for (String name : candidates) { + if (dupes.add(name)) { continue; } - log.error("Duplicate claims for Name = `{}` ({} / {})", config.getName(), config, prior); + log.error("Duplicate claims for Name = `{}`", name); return false; } From 549ccc56bcc2358b1c73d833badd0c114c741e2e Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Wed, 4 Aug 2021 12:34:16 +0200 Subject: [PATCH 58/82] Allow reading of multiple ids Adds a lot of documentation --- .../concept/specific/external/CQExternal.java | 30 ++++++-- .../models/config/FrontendConfig.java | 71 ++++++++++--------- 2 files changed, 63 insertions(+), 38 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java index b212de98ec..9382564f0d 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java @@ -5,6 +5,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.Function; import java.util.stream.Collectors; import javax.validation.constraints.NotEmpty; @@ -30,6 +31,7 @@ import it.unimi.dsi.fastutil.ints.Int2ObjectAVLTreeMap; import it.unimi.dsi.fastutil.ints.Int2ObjectMap; import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; +import it.unimi.dsi.fastutil.ints.IntList; import lombok.Data; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -171,17 +173,35 @@ public static ResolveStatistic resolveEntities(@NotEmpty String[][] values, @Not // extract dates from rows final Int2ObjectMap rowDates = readDates(values, format, dateReader, queryUpload); - // TODO allow multiple ids - final int idIndex = queryUpload.getIdIndex(format); + final List> readers = queryUpload.getIdReaders(format); - final ColumnConfig reader = queryUpload.getIdMapper(format.get(idIndex)); + if(readers.isEmpty()){ + throw new IllegalArgumentException("No readers configured."); + } // ignore the first row, because this is the header for (int rowNum = 1; rowNum < values.length; rowNum++) { final String[] row = values[rowNum]; - final EntityIdMap.ExternalId externalId = reader.read(row[idIndex]); - final int resolvedId = mapping.resolve(externalId); + int resolvedId = -1; + + for (Function reader : readers) { + final EntityIdMap.ExternalId externalId = reader.apply(row); + + int innerResolved = mapping.resolve(externalId); + + if(innerResolved == -1){ + continue; + } + + if(resolvedId != -1 && innerResolved != resolvedId){ + log.error("`{}` maps to different Entities", (Object) row); + continue; + } + + resolvedId = innerResolved; + } + if (resolvedId == -1) { unresolvedId.add(row); diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java index a2050fc110..144b5725d2 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java @@ -2,12 +2,12 @@ import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -20,6 +20,7 @@ import com.bakdata.conquery.models.auth.permissions.Ability; import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.identifiable.mapping.AutoIncrementingPseudomizer; +import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; import com.bakdata.conquery.models.identifiable.mapping.FullIdPrinter; import com.bakdata.conquery.models.identifiable.mapping.IdPrinter; import com.bakdata.conquery.models.worker.Namespace; @@ -53,6 +54,10 @@ public class FrontendConfig { private UploadConfig queryUpload = new UploadConfig(); + + /** + * Class configuring {@link EntityIdMap} and {@link com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal} resolving. + */ @Getter @Setter @With @@ -60,6 +65,11 @@ public class FrontendConfig { @NoArgsConstructor public static class UploadConfig { + /** + * List of resolvable and printable ids. + * + * @apiNote Sort order determines output order of ids with where {@link ColumnConfig#isPrint()} is true in result. + */ @NotEmpty @Valid private List ids = List.of( @@ -68,52 +78,46 @@ public static class UploadConfig { .field("result") .resolvable(true) .fillAnon(true) + .print(true) .build() ); - @JsonIgnore - @Setter(AccessLevel.NONE) - @Getter(AccessLevel.NONE) - private List idFieldsCached; - /** * Headers for Output CSV. */ @JsonIgnore - public List getPrintIdFields() { - if (idFieldsCached == null) { - idFieldsCached = ids.stream() - .filter(ColumnConfig::isPrint) - .map(ColumnConfig::getField) - .collect(Collectors.toUnmodifiableList()); - } + @Getter(lazy = true) + private final List printIdFields = ids.stream() + .filter(ColumnConfig::isPrint) + .map(ColumnConfig::getField) + .collect(Collectors.toUnmodifiableList()); - return idFieldsCached; - } @JsonIgnore @Setter(AccessLevel.NONE) - private Map idMappers; + @Getter(lazy = true, value = AccessLevel.PRIVATE) + private final Map idMappers = ids.stream().filter(ColumnConfig::isResolvable) + .collect(Collectors.toMap(ColumnConfig::getName, Functions.identity())); - public ColumnConfig getIdMapper(String name) { - if (idMappers == null) { - idMappers = ids.stream().filter(ColumnConfig::isResolvable) - .collect(Collectors.toMap(ColumnConfig::getName, Functions.identity())); - } - return idMappers.get(name); - } + /** + * If a column contains an ID, create a reader for that ID. + */ + public List> getIdReaders(List format) { + List> out = new ArrayList<>(format.size()); - public int getIdIndex(List format) { for (int index = 0; index < format.size(); index++) { - final String current = format.get(index); + final ColumnConfig mapper = getIdMappers().get(format.get(index)); - if (getIdMapper(current) != null) { - return index; + if (mapper == null) { + continue; } + + final int finalIndex = index; + out.add(row -> mapper.read(row[finalIndex])); } - return -1; + return out; } @@ -154,10 +158,12 @@ public boolean isExactlyOnePseudo() { .count() == 1; } - - public DateFormat resolveDateFormat(String handle) { + /** + * Try to resolve a date format, return nothing if not possible. + */ + public DateFormat resolveDateFormat(String name) { try { - return DateFormat.valueOf(handle); + return DateFormat.valueOf(name); } catch (IllegalArgumentException e) { return null; // Does not exist @@ -166,7 +172,7 @@ public DateFormat resolveDateFormat(String handle) { /** - * Try to create a {@link FullIdPrinter} for user if they are allowed. If not allowed to read ids, they will receive ab pseudomized result instead. + * Try to create a {@link FullIdPrinter} for user if they are allowed. If not allowed to read ids, they will receive a pseudomized result instead. */ public IdPrinter getIdPrinter(User owner, ManagedExecution execution, Namespace namespace) { final int size = getPrintIdFields().size(); @@ -180,7 +186,6 @@ public IdPrinter getIdPrinter(User owner, ManagedExecution execution, Namespa } - return new AutoIncrementingPseudomizer(size, pos); } } From 667840ddad925b5f6760bb26f6398b1915fdc921 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Wed, 4 Aug 2021 12:46:41 +0200 Subject: [PATCH 59/82] more documentation --- .../models/identifiable/mapping/FullIdPrinter.java | 6 ++++-- .../conquery/models/identifiable/mapping/IdPrinter.java | 3 +++ .../bakdata/conquery/integration/IntegrationTests.java | 8 ++------ 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/FullIdPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/FullIdPrinter.java index 81711a7fbf..8fd8bf6fb3 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/FullIdPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/FullIdPrinter.java @@ -4,8 +4,9 @@ import com.bakdata.conquery.models.query.results.EntityResult; import lombok.RequiredArgsConstructor; + /** - * Maker interface for implementation specific state object during query result to csv rendering. + * IdPrinter using {@link EncodedDictionary} and {@link EntityIdMap} to generate full ids. */ @RequiredArgsConstructor public class FullIdPrinter implements IdPrinter { @@ -17,7 +18,7 @@ public class FullIdPrinter implements IdPrinter { private final int idPos; @Override - public EntityPrintId createId(EntityResult entityResult){ + public EntityPrintId createId(EntityResult entityResult) { String csvEntityId = dictionary.getElement(entityResult.getEntityId()); // The state may be uses by implementations of this class @@ -28,6 +29,7 @@ public EntityPrintId createId(EntityResult entityResult){ EntityPrintId externalEntityId = idMapping.toExternal(csvEntityId); + // Fallback, when we have no mapping. if (externalEntityId == null) { final String[] parts = new String[size]; parts[idPos] = csvEntityId; diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdPrinter.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdPrinter.java index 25a746694e..ba62bc3a22 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdPrinter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/IdPrinter.java @@ -2,6 +2,9 @@ import com.bakdata.conquery.models.query.results.EntityResult; +/** + * Implementing classes are used to create Printable Ids for results. + */ public interface IdPrinter { EntityPrintId createId(EntityResult entityResult); } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java b/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java index ab73d84f56..9037ec99c9 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java @@ -80,10 +80,7 @@ public List jsonTests() { final String testRoot = Objects.requireNonNullElse(System.getenv(TestTags.TEST_DIRECTORY_ENVIRONMENT_VARIABLE), defaultTestRoot); ResourceTree tree = new ResourceTree(null, null); - tree.addAll( - CPSTypeIdResolver.SCAN_RESULT - .getResourcesMatchingPattern(Pattern.compile("^" + testRoot + ".*\\.test\\.json$")) - ); + tree.addAll(CPSTypeIdResolver.SCAN_RESULT.getResourcesMatchingPattern(Pattern.compile("^" + testRoot + ".*\\.test\\.json$"))); // collect tests from directory if (tree.getChildren().isEmpty()) { @@ -102,8 +99,7 @@ public List jsonTests() { @SneakyThrows public Stream programmaticTests() { - List> programmatic = CPSTypeIdResolver - .SCAN_RESULT + List> programmatic = CPSTypeIdResolver.SCAN_RESULT .getClassesImplementing(ProgrammaticIntegrationTest.class.getName()) .filter(info -> info.getPackageName().startsWith(defaultTestRootPackage)) .loadClasses(); From 62b33ac271e6b2ff320c0b673e69c716f98122cb Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Wed, 4 Aug 2021 13:39:42 +0200 Subject: [PATCH 60/82] adds even mor documentation and removes a double catch --- .../models/execution/ManagedExecution.java | 10 ++----- .../identifiable/mapping/EntityIdMap.java | 15 +++++----- .../integration/IntegrationTests.java | 28 +++++++++---------- 3 files changed, 24 insertions(+), 29 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java b/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java index 38d88fa03c..5780e68751 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java +++ b/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java @@ -137,14 +137,8 @@ public void initExecutable(DatasetRegistry datasetRegistry, ConqueryConfig confi executionManager = datasetRegistry.get(getDataset().getId()).getExecutionManager(); - try { - doInitExecutable(datasetRegistry, config); - initialized = true; - } - catch (Exception e) { - log.error("Failed to initialize", e); - //TODO what do? - } + doInitExecutable(datasetRegistry, config); + initialized = true; } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java index ce5bb1cea2..5b115f885c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java @@ -23,8 +23,8 @@ import lombok.extern.slf4j.Slf4j; /** - * Mapping from Csv Entity Id to External Entity Id and back from the - * combinations of Accessor + IDs to the Entity Id. + * Mapping from uploaded {@link ExternalId} for resolving in {@link com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal}, and also for printing with {@link EntityPrintId}. + * */ @Getter @EqualsAndHashCode @@ -50,7 +50,7 @@ public class EntityIdMap { private final Map external2Internal = new HashMap<>(); /** - * Read incoming CSV-file extracting Id-Mappings for in and Output. + * Read incoming CSV-file extracting Id-Mappings for {@link ExternalId} and {@link EntityPrintId}. */ public static EntityIdMap generateIdMapping(CsvParser parser, List mappers) { @@ -68,14 +68,15 @@ public static EntityIdMap generateIdMapping(CsvParser parser, List final String otherId = record.getString(columnConfig.getField()); - if (otherId == null) { - continue; - } - + // Collect printable parts into id if(columnConfig.isPrint()) { idParts.add(otherId); } + if (otherId == null) { + continue; + } + if (!columnConfig.isResolvable()) { continue; } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java b/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java index 9037ec99c9..8bb624870a 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java @@ -99,22 +99,22 @@ public List jsonTests() { @SneakyThrows public Stream programmaticTests() { - List> programmatic = CPSTypeIdResolver.SCAN_RESULT - .getClassesImplementing(ProgrammaticIntegrationTest.class.getName()) - .filter(info -> info.getPackageName().startsWith(defaultTestRootPackage)) - .loadClasses(); + List> programmatic = + CPSTypeIdResolver.SCAN_RESULT.getClassesImplementing(ProgrammaticIntegrationTest.class.getName()) + .filter(info -> info.getPackageName().startsWith(defaultTestRootPackage)) + .loadClasses(); return programmatic - .stream() - .map(c -> { - try { - return c.asSubclass(ProgrammaticIntegrationTest.class).getDeclaredConstructor().newInstance(); - } - catch (Exception e) { - throw new RuntimeException(e); - } - }) - .map(this::createDynamicProgrammaticTestNode); + .stream() + .map(c -> { + try { + return c.asSubclass(ProgrammaticIntegrationTest.class).getDeclaredConstructor().newInstance(); + } + catch (Exception e) { + throw new RuntimeException(e); + } + }) + .map(this::createDynamicProgrammaticTestNode); } private DynamicTest createDynamicProgrammaticTestNode(ProgrammaticIntegrationTest test) { From 7e2b3732949beca6ec33d513c8f693fea1a9e28d Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Mon, 9 Aug 2021 12:00:30 +0200 Subject: [PATCH 61/82] also check if we have no id in the column --- .../concept/specific/external/CQExternal.java | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java index 9382564f0d..c989a91772 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java @@ -2,6 +2,7 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; @@ -15,7 +16,6 @@ import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.jackson.InternalOnly; import com.bakdata.conquery.models.common.CDateSet; -import com.bakdata.conquery.models.config.ColumnConfig; import com.bakdata.conquery.models.config.FrontendConfig; import com.bakdata.conquery.models.error.ConqueryError; import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; @@ -31,7 +31,6 @@ import it.unimi.dsi.fastutil.ints.Int2ObjectAVLTreeMap; import it.unimi.dsi.fastutil.ints.Int2ObjectMap; import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; -import it.unimi.dsi.fastutil.ints.IntList; import lombok.Data; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -175,8 +174,9 @@ public static ResolveStatistic resolveEntities(@NotEmpty String[][] values, @Not final List> readers = queryUpload.getIdReaders(format); - if(readers.isEmpty()){ - throw new IllegalArgumentException("No readers configured."); + // We will not be able to resolve anything... + if (readers.isEmpty()) { + return new ResolveStatistic(Collections.emptyMap(), Collections.emptyList(), List.of(values)); } // ignore the first row, because this is the header @@ -188,13 +188,17 @@ public static ResolveStatistic resolveEntities(@NotEmpty String[][] values, @Not for (Function reader : readers) { final EntityIdMap.ExternalId externalId = reader.apply(row); + if (externalId == null) { + continue; + } + int innerResolved = mapping.resolve(externalId); - if(innerResolved == -1){ + if (innerResolved == -1) { continue; } - if(resolvedId != -1 && innerResolved != resolvedId){ + if (resolvedId != -1 && innerResolved != resolvedId) { log.error("`{}` maps to different Entities", (Object) row); continue; } From 85cbd4bd7869aaeebcd30406a860407f3e03c4a1 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Tue, 10 Aug 2021 12:09:51 +0200 Subject: [PATCH 62/82] add more decoration to ExternalUpload and resulting ManagedQuery --- .../conquery/apiv1/QueryProcessor.java | 30 ++++++++++++------- .../conquery/apiv1/query/ExternalUpload.java | 1 + 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java index 1294f2afc5..eec4dc0629 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java @@ -20,10 +20,10 @@ import com.bakdata.conquery.apiv1.query.CQElement; import com.bakdata.conquery.apiv1.query.ConceptQuery; import com.bakdata.conquery.apiv1.query.ExternalUpload; +import com.bakdata.conquery.apiv1.query.ExternalUploadResult; import com.bakdata.conquery.apiv1.query.Query; import com.bakdata.conquery.apiv1.query.QueryDescription; import com.bakdata.conquery.apiv1.query.SecondaryIdQuery; -import com.bakdata.conquery.apiv1.query.ExternalUploadResult; import com.bakdata.conquery.apiv1.query.concept.specific.CQAnd; import com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal; import com.bakdata.conquery.io.result.ResultRender.ResultRendererProvider; @@ -378,22 +378,30 @@ public ExternalUploadResult uploadEntities(User user, Dataset dataset, ExternalU // Resolving nothing is a problem thus we fail. if (statistic.getResolved().isEmpty()) { throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST) - .entity(new ExternalUploadResult(null, 0, statistic.getUnresolvedId(), statistic.getUnreadableDate())) - .build()); + .entity(new ExternalUploadResult(null, 0, statistic.getUnresolvedId(), statistic.getUnreadableDate())) + .build()); } final ConceptQuery query = new ConceptQuery(new CQExternal(upload.getFormat(), upload.getValues())); // We only create the Query, really no need to execute it as it's only useful for composition. - final ManagedExecution execution = - datasetRegistry.get(dataset.getId()).getExecutionManager() - .createExecution(datasetRegistry, query, user, dataset); + final ManagedQuery execution = + ((ManagedQuery) datasetRegistry.get(dataset.getId()).getExecutionManager() + .createExecution(datasetRegistry, query, user, dataset)); + + execution.setLastResultCount((long) statistic.getResolved().size()); + + if (upload.getLabel() != null) { + execution.setLabel(upload.getLabel()); + } + + execution.initExecutable(datasetRegistry, config); return new ExternalUploadResult( - execution.getId(), - statistic.getResolved().size(), - statistic.getUnresolvedId(), - statistic.getUnreadableDate() - ); + execution.getId(), + statistic.getResolved().size(), + statistic.getUnresolvedId(), + statistic.getUnreadableDate() + ); } } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/ExternalUpload.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/ExternalUpload.java index c73f9dc494..017114b9e4 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/ExternalUpload.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/ExternalUpload.java @@ -19,6 +19,7 @@ @Getter public class ExternalUpload { + private final String label; private final List format; private final String[][] values; From 40e3a4bf02c9c3721bc8bcddd2ac01f960eb187e Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 10 Aug 2021 15:05:38 +0200 Subject: [PATCH 63/82] allow date format configuration via locale --- .../conquery/models/config/LocaleConfig.java | 25 +++++++++++++++++-- .../models/externalservice/ResultType.java | 14 ++++++++--- .../conquery/models/query/PrintSettings.java | 12 +++++++++ .../externalservice/ResultTypeTest.java | 17 ++++++++----- 4 files changed, 56 insertions(+), 12 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java index 67d9daebe2..8c87d54b2d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java @@ -1,10 +1,19 @@ package com.bakdata.conquery.models.config; -import java.util.Currency; -import java.util.Locale; +import java.io.IOException; +import java.text.SimpleDateFormat; +import java.time.format.DateTimeFormatter; +import java.util.*; +import javax.validation.constraints.NotEmpty; import javax.validation.constraints.NotNull; +import com.fasterxml.jackson.annotation.JsonValue; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import lombok.Getter; import lombok.Setter; @@ -12,4 +21,16 @@ public class LocaleConfig { @NotNull private Locale frontend = Locale.ROOT; + + @NotNull + @JsonDeserialize(contentUsing = DateTimeFormatterDeserializer.class) + private Map dateFormatMapping = Collections.emptyMap(); + + public static class DateTimeFormatterDeserializer extends JsonDeserializer { + + @Override + public DateTimeFormatter deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { + return DateTimeFormatter.ofPattern(p.getText()); + } + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/externalservice/ResultType.java b/backend/src/main/java/com/bakdata/conquery/models/externalservice/ResultType.java index 30909a0d5f..3326bfddf7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/externalservice/ResultType.java +++ b/backend/src/main/java/com/bakdata/conquery/models/externalservice/ResultType.java @@ -1,6 +1,7 @@ package com.bakdata.conquery.models.externalservice; import c10n.C10N; +import c10n.share.LocaleMapping; import com.bakdata.conquery.internationalization.Results; import com.bakdata.conquery.io.cps.CPSBase; import com.bakdata.conquery.io.cps.CPSType; @@ -23,6 +24,8 @@ import org.apache.arrow.vector.types.pojo.FieldType; import java.math.BigDecimal; +import java.text.DateFormat; +import java.text.SimpleDateFormat; import java.time.LocalDate; import java.util.List; import java.util.StringJoiner; @@ -198,9 +201,12 @@ public static class DateT extends PrimitiveResultType { @Override public String print(PrintSettings cfg, @NonNull Object f) { if(!(f instanceof Number)) { - throw new IllegalStateException("Expected an Number but got an '" + (f != null ? f.getClass().getName() : "no type") + "' with the value: " + f ); + throw new IllegalStateException("Expected an Number but got an '" + f.getClass().getName() + "' with the value: " + f); } - return CDate.toLocalDate(((Number)f).intValue()).toString(); + if (cfg.isPrettyPrint()) { + return cfg.getDateFormat().format(CDate.toLocalDate(((Number) f).intValue())); + } + return CDate.toLocalDate(((Number) f).intValue()).toString(); } @Override @@ -224,9 +230,9 @@ public static class DateRangeT extends PrimitiveResultType { @Override public String print(PrintSettings cfg, @NonNull Object f) { if(!(f instanceof List)) { - throw new IllegalStateException(String.format("Expected a List got %s (Type: %s, as string: %s)", f, f != null ? f.getClass().getName() : "no type", f)); + throw new IllegalStateException(String.format("Expected a List got %s (Type: %s, as string: %s)", f, f.getClass().getName(), f)); } - List list = (List) f; + List list = (List) f; if(list.size() != 2) { throw new IllegalStateException("Expected a list with 2 elements, one min, one max. The list was: " + list); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/PrintSettings.java b/backend/src/main/java/com/bakdata/conquery/models/query/PrintSettings.java index 6fe366cde7..ab730f7334 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/PrintSettings.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/PrintSettings.java @@ -1,8 +1,12 @@ package com.bakdata.conquery.models.query; import java.text.NumberFormat; +import java.text.SimpleDateFormat; +import java.time.format.DateTimeFormatter; import java.util.Currency; +import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.function.Function; import com.bakdata.conquery.models.config.ConqueryConfig; @@ -13,6 +17,8 @@ import lombok.Getter; import lombok.ToString; +import javax.validation.constraints.NotNull; + @Getter @ToString(onlyExplicitlyIncluded = true) public class PrintSettings { @@ -28,6 +34,7 @@ public class PrintSettings { private final boolean prettyPrint; @ToString.Include private final Locale locale; + private DateTimeFormatter dateFormat; private final NumberFormat decimalFormat; private final NumberFormat integerFormat; private final Currency currency; @@ -56,6 +63,11 @@ public PrintSettings(boolean prettyPrint, Locale locale, DatasetRegistry dataset this.integerFormat = NUMBER_FORMAT.apply(locale); this.decimalFormat = DECIMAL_FORMAT.apply(locale); + + @NotNull Map dfMapping = config.getLocale().getDateFormatMapping(); + Locale closestLocale = Locale.lookup(Locale.LanguageRange.parse(locale.toString()), dfMapping.keySet()); + // fallback to iso date if no specific formater is available + this.dateFormat = closestLocale != null ? dfMapping.get(closestLocale) : DateTimeFormatter.ISO_DATE; } public PrintSettings(boolean prettyPrint, Locale locale, DatasetRegistry datasetRegistry, ConqueryConfig config, PrintIdMapper idMapper) { diff --git a/backend/src/test/java/com/bakdata/conquery/models/externalservice/ResultTypeTest.java b/backend/src/test/java/com/bakdata/conquery/models/externalservice/ResultTypeTest.java index 2b217c4d8d..42a6aa1a45 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/externalservice/ResultTypeTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/externalservice/ResultTypeTest.java @@ -5,8 +5,11 @@ import java.io.IOException; import java.math.BigDecimal; import java.time.LocalDate; +import java.time.format.DateTimeFormatter; import java.util.Currency; +import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.stream.Stream; import com.bakdata.conquery.models.forms.util.DateContext; @@ -20,29 +23,31 @@ import org.junit.jupiter.params.provider.MethodSource; public class ResultTypeTest { + + public static final ConqueryConfig CONFIG = new ConqueryConfig(); static { // Initialization of the internationalization I18n.init(); + //init global default config + CONFIG.getPreprocessor().getParsers().setCurrency(Currency.getInstance("EUR")); + CONFIG.getLocale().setDateFormatMapping(Map.of(Locale.GERMAN, DateTimeFormatter.ofPattern("dd.MM.yyyy"))); } - public static final ConqueryConfig CONFIG = new ConqueryConfig(); private static final PrintSettings PRETTY = new PrintSettings(true, Locale.ENGLISH, null, CONFIG, null); private static final PrintSettings PRETTY_DE = new PrintSettings(true, Locale.GERMAN, null, CONFIG, null); private static final PrintSettings PLAIN = new PrintSettings(false, Locale.ENGLISH, null, CONFIG, null); @SuppressWarnings("unused") - public static Stream testData() { - //init global default config - ConqueryConfig cfg = new ConqueryConfig(); - cfg.getPreprocessor().getParsers().setCurrency(Currency.getInstance("EUR")); - return Stream.of( + public static List testData() { + return List.of( Arguments.of(PRETTY, ResultType.BooleanT.INSTANCE, true, "Yes"), Arguments.of(PRETTY, ResultType.BooleanT.INSTANCE, false, "No"), Arguments.of(PRETTY, ResultType.CategoricalT.INSTANCE, "test", "test"), Arguments.of(PRETTY, ResultType.ResolutionT.INSTANCE, DateContext.Resolution.COMPLETE.name(), "complete"), Arguments.of(PRETTY_DE, ResultType.ResolutionT.INSTANCE, DateContext.Resolution.COMPLETE.name(), "Gesamt"), Arguments.of(PRETTY, ResultType.DateT.INSTANCE, LocalDate.of(2013, 7, 12).toEpochDay(), "2013-07-12"), + Arguments.of(PRETTY_DE, ResultType.DateT.INSTANCE, LocalDate.of(2013, 7, 12).toEpochDay(), "12.07.2013"), Arguments.of(PRETTY, ResultType.IntegerT.INSTANCE, 51839274, "51,839,274"), Arguments.of(PRETTY_DE, ResultType.IntegerT.INSTANCE, 51839274, "51.839.274"), Arguments.of(PRETTY, ResultType.MoneyT.INSTANCE, 51839274L, "518,392.74"), From 4997a0fa4c3a04f0d970d432194daed06d96e61d Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 10 Aug 2021 15:25:51 +0200 Subject: [PATCH 64/82] allows to generalize locale mapping --- .../conquery/models/config/LocaleConfig.java | 38 +++++++++++++++++++ .../conquery/models/query/PrintSettings.java | 5 +-- .../externalservice/ResultTypeTest.java | 2 +- 3 files changed, 40 insertions(+), 5 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java index 8c87d54b2d..fd3c001288 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java @@ -33,4 +33,42 @@ public DateTimeFormatter deserialize(JsonParser p, DeserializationContext ctxt) return DateTimeFormatter.ofPattern(p.getText()); } } + + /** + * Finds the best formatter according to the locale and mapped date formatters. + * If there is no perfect match, the locale is abstracted, see findClosestMatch. + * @param locale + * @return + */ + public DateTimeFormatter findDateTimeFormater(Locale locale) { + final Locale closestMatch = findClosestMatch(locale); + return closestMatch != null ? dateFormatMapping.get(closestMatch) : DateTimeFormatter.ISO_DATE; + } + + /** + * Adapted from {@link c10n.share.DefaultLocaleMapping} + */ + public Locale findClosestMatch(Locale forLocale) { + Set fromSet = dateFormatMapping.keySet(); + String variant = forLocale.getDisplayVariant(); + String country = forLocale.getCountry(); + String language = forLocale.getLanguage(); + List c = new ArrayList<>(4); + if (null != variant && !variant.isEmpty()) { + c.add(forLocale); + } + if (null != country && !country.isEmpty()) { + c.add(new Locale(language, country)); + } + if (null != language && !language.isEmpty()) { + c.add(new Locale(language)); + } + c.add(Locale.ROOT); + for (Locale candidateLocale : c) { + if (fromSet.contains(candidateLocale)) { + return candidateLocale; + } + } + return null; + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/PrintSettings.java b/backend/src/main/java/com/bakdata/conquery/models/query/PrintSettings.java index ab730f7334..141d9b858d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/PrintSettings.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/PrintSettings.java @@ -64,10 +64,7 @@ public PrintSettings(boolean prettyPrint, Locale locale, DatasetRegistry dataset this.integerFormat = NUMBER_FORMAT.apply(locale); this.decimalFormat = DECIMAL_FORMAT.apply(locale); - @NotNull Map dfMapping = config.getLocale().getDateFormatMapping(); - Locale closestLocale = Locale.lookup(Locale.LanguageRange.parse(locale.toString()), dfMapping.keySet()); - // fallback to iso date if no specific formater is available - this.dateFormat = closestLocale != null ? dfMapping.get(closestLocale) : DateTimeFormatter.ISO_DATE; + this.dateFormat = config.getLocale().findDateTimeFormater(locale); } public PrintSettings(boolean prettyPrint, Locale locale, DatasetRegistry datasetRegistry, ConqueryConfig config, PrintIdMapper idMapper) { diff --git a/backend/src/test/java/com/bakdata/conquery/models/externalservice/ResultTypeTest.java b/backend/src/test/java/com/bakdata/conquery/models/externalservice/ResultTypeTest.java index 42a6aa1a45..b92765f8d4 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/externalservice/ResultTypeTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/externalservice/ResultTypeTest.java @@ -35,7 +35,7 @@ public class ResultTypeTest { } private static final PrintSettings PRETTY = new PrintSettings(true, Locale.ENGLISH, null, CONFIG, null); - private static final PrintSettings PRETTY_DE = new PrintSettings(true, Locale.GERMAN, null, CONFIG, null); + private static final PrintSettings PRETTY_DE = new PrintSettings(true, Locale.GERMANY, null, CONFIG, null); private static final PrintSettings PLAIN = new PrintSettings(false, Locale.ENGLISH, null, CONFIG, null); @SuppressWarnings("unused") From f42783f9d5e44a3f4c10b70c5dbdad2d8b4478ed Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 11 Aug 2021 10:30:38 +0200 Subject: [PATCH 65/82] adds locale to dateformat mapping --- .../conquery/apiv1/QueryProcessor.java | 2 +- .../concept/specific/external/CQExternal.java | 2 +- .../models/config/ConqueryConfig.java | 2 +- .../conquery/models/config/LocaleConfig.java | 27 +++++++++++++------ .../conquery/models/config/ParserConfig.java | 8 ------ .../conquery/models/events/MajorTypeId.java | 5 ++-- .../conquery/models/jobs/ImportJob.java | 14 +++++++--- .../conquery/models/preproc/Preprocessed.java | 7 ++--- .../conquery/models/preproc/Preprocessor.java | 6 ++--- .../models/preproc/parser/Parser.java | 4 +-- .../parser/specific/BooleanParser.java | 3 ++- .../preproc/parser/specific/DateParser.java | 5 ++-- .../parser/specific/DateRangeParser.java | 5 ++-- .../parser/specific/DecimalParser.java | 3 ++- .../parser/specific/IntegerParser.java | 3 ++- .../preproc/parser/specific/MoneyParser.java | 4 +-- .../preproc/parser/specific/RealParser.java | 5 ++-- .../preproc/parser/specific/StringParser.java | 3 ++- .../specific/string/NumberTypeGuesser.java | 3 ++- .../admin/rest/AdminDatasetProcessor.java | 2 +- .../com/bakdata/conquery/util/DateReader.java | 22 +++------------ .../conquery/io/result/ResultNameTest.java | 2 +- .../events/stores/types/MajorTypesTest.java | 3 ++- .../stores/types/StringEncodingTest.java | 3 ++- .../externalservice/ResultTypeTest.java | 2 +- .../parser/specific/DateRangeParserTest.java | 7 ++--- .../parser/specific/DecimalParserTest.java | 3 ++- .../parser/specific/IntegerParserTest.java | 3 ++- .../parser/specific/RealParserTest.java | 7 ++--- 29 files changed, 89 insertions(+), 76 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java index eec4dc0629..0ae319d90f 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java @@ -372,7 +372,7 @@ public ExternalUploadResult uploadEntities(User user, Dataset dataset, ExternalU CQExternal.resolveEntities(upload.getValues(), upload.getFormat(), datasetRegistry.get(dataset.getId()).getStorage().getIdMapping(), config.getFrontend().getQueryUpload(), - config.getPreprocessor().getParsers().getDateReader() + config.getLocale().getDateReader() ); // Resolving nothing is a problem thus we fail. diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java index c989a91772..95796821d1 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java @@ -123,7 +123,7 @@ public void resolve(QueryResolveContext context) { resolveEntities(values, format, context.getNamespace().getStorage().getIdMapping(), context.getConfig().getFrontend().getQueryUpload(), - context.getConfig().getPreprocessor().getParsers().getDateReader() + context.getConfig().getLocale().getDateReader() ); if (resolved.getResolved().isEmpty()) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java index 0d62738b94..ce86e8cc4e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java @@ -134,7 +134,7 @@ public ObjectMapper configureObjectMapper(ObjectMapper objectMapper) { public static class ConfiguredModule extends SimpleModule { public ConfiguredModule(ConqueryConfig config){ - DateReader dateReader = config.getPreprocessor().getParsers().getDateReader(); + DateReader dateReader = config.getLocale().getDateReader(); addDeserializer(LocalDate.class, new FormatedDateDeserializer(dateReader)); addDeserializer(CDateSet.class, new CDateSetDeserializer(dateReader)); diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java index fd3c001288..47a383ddc6 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java @@ -8,12 +8,15 @@ import javax.validation.constraints.NotEmpty; import javax.validation.constraints.NotNull; +import com.bakdata.conquery.util.DateReader; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonValue; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.JsonDeserializer; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.google.common.collect.Sets; import lombok.Getter; import lombok.Setter; @@ -23,15 +26,23 @@ public class LocaleConfig { private Locale frontend = Locale.ROOT; @NotNull - @JsonDeserialize(contentUsing = DateTimeFormatterDeserializer.class) - private Map dateFormatMapping = Collections.emptyMap(); + private Map dateFormatMapping = Map.of(Locale.GERMAN, "dd.MM.yyyy"); - public static class DateTimeFormatterDeserializer extends JsonDeserializer { - @Override - public DateTimeFormatter deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { - return DateTimeFormatter.ofPattern(p.getText()); - } + @NotNull + private Set dateParsingFormats = Set.of( + "yyyy-MM-dd", + "yyyyMMdd", + "dd.MM.yyyy" + ); + + + /** + * Date formats that are available for parsing. + */ + @JsonIgnore + public DateReader getDateReader() { + return new DateReader(Sets.union(dateParsingFormats, Set.copyOf(dateFormatMapping.values()))); } /** @@ -42,7 +53,7 @@ public DateTimeFormatter deserialize(JsonParser p, DeserializationContext ctxt) */ public DateTimeFormatter findDateTimeFormater(Locale locale) { final Locale closestMatch = findClosestMatch(locale); - return closestMatch != null ? dateFormatMapping.get(closestMatch) : DateTimeFormatter.ISO_DATE; + return closestMatch != null ? DateTimeFormatter.ofPattern(dateFormatMapping.get(closestMatch)) : DateTimeFormatter.ISO_DATE; } /** diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ParserConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/ParserConfig.java index 2080c75c0d..dc9e310132 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ParserConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ParserConfig.java @@ -26,12 +26,4 @@ public class ParserConfig { */ @NotNull private Currency currency = Currency.getInstance("EUR"); - - /** - * Date formats that are available for parsing. - */ - @NotNull - private DateReader dateReader = new DateReader(List.of( - "yyyy-MM-dd", "yyyyMMdd", "dd.MM.yyyy" - )); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/MajorTypeId.java b/backend/src/main/java/com/bakdata/conquery/models/events/MajorTypeId.java index 5265b8bcc5..e7f612a90b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/events/MajorTypeId.java +++ b/backend/src/main/java/com/bakdata/conquery/models/events/MajorTypeId.java @@ -2,6 +2,7 @@ import java.util.function.Function; +import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.config.ParserConfig; import com.bakdata.conquery.models.preproc.parser.Parser; import com.bakdata.conquery.models.preproc.parser.specific.BooleanParser; @@ -29,9 +30,9 @@ public enum MajorTypeId { @Getter private final boolean dateCompatible; - private final Function supplier; + private final Function supplier; - public Parser createParser(ParserConfig config) { + public Parser createParser(ConqueryConfig config) { return supplier.apply(config); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/jobs/ImportJob.java b/backend/src/main/java/com/bakdata/conquery/models/jobs/ImportJob.java index 1f5f8fea75..689bfc2746 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/jobs/ImportJob.java +++ b/backend/src/main/java/com/bakdata/conquery/models/jobs/ImportJob.java @@ -19,6 +19,7 @@ import com.bakdata.conquery.ConqueryConstants; import com.bakdata.conquery.io.storage.NamespaceStorage; +import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.config.ParserConfig; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.Dataset; @@ -78,10 +79,16 @@ public class ImportJob extends Job { private final PreprocessedHeader header; private final PreprocessedDictionaries dictionaries; private final PreprocessedData container; + private final ConqueryConfig config; private static final int NUMBER_OF_STEPS = /* directly in execute = */4; - public static ImportJob create(Namespace namespace, InputStream inputStream, int entityBucketSize, IdMutex sharedDictionaryLocks) + public static ImportJob create( + Namespace namespace, + InputStream inputStream, + int entityBucketSize, + IdMutex sharedDictionaryLocks, + ConqueryConfig config) throws IOException { try (PreprocessedReader parser = new PreprocessedReader(inputStream)) { @@ -133,7 +140,8 @@ public static ImportJob create(Namespace namespace, InputStream inputStream, int entityBucketSize, header, dictionaries, - container + container, + config ); } } @@ -478,7 +486,7 @@ private void applyDictionaryMappings(Map mappings, Ma log.debug("Remapping Column[{}] = {} with {}", columnName, stringStore, mapping); // we need to find a new Type for the index-Column as it's going to be remapped and might change in size - final IntegerParser indexParser = new IntegerParser(new ParserConfig()); + final IntegerParser indexParser = new IntegerParser(config); final IntSummaryStatistics statistics = mapping.target().intStream().summaryStatistics(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/Preprocessed.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/Preprocessed.java index d89ff21f4d..edbea5cf39 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/Preprocessed.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/Preprocessed.java @@ -13,6 +13,7 @@ import java.util.zip.GZIPOutputStream; import com.bakdata.conquery.io.jackson.Jackson; +import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.config.ParserConfig; import com.bakdata.conquery.models.dictionary.Dictionary; import com.bakdata.conquery.models.events.MajorTypeId; @@ -64,7 +65,7 @@ public class Preprocessed { private long rows = 0; - public Preprocessed(ParserConfig parserConfig, PreprocessingJob preprocessingJob) throws IOException { + public Preprocessed(ConqueryConfig config, PreprocessingJob preprocessingJob) throws IOException { this.job = preprocessingJob; this.descriptor = preprocessingJob.getDescriptor(); this.name = this.descriptor.getName(); @@ -72,14 +73,14 @@ public Preprocessed(ParserConfig parserConfig, PreprocessingJob preprocessingJob TableInputDescriptor input = this.descriptor.getInputs()[0]; columns = new PPColumn[input.getWidth()]; - primaryColumn = (StringParser) MajorTypeId.STRING.createParser(parserConfig); + primaryColumn = (StringParser) MajorTypeId.STRING.createParser(config); values = new ColumnValues[columns.length]; for (int index = 0; index < input.getWidth(); index++) { ColumnDescription columnDescription = input.getColumnDescription(index); columns[index] = new PPColumn(columnDescription.getName(), columnDescription.getType()); - columns[index].setParser(columnDescription.getType().createParser(parserConfig)); + columns[index].setParser(columnDescription.getType().createParser(config)); final Parser parser = columns[index].getParser(); values[index] = parser.createColumnValues(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/Preprocessor.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/Preprocessor.java index 900737ded3..aa6dd79e95 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/Preprocessor.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/Preprocessor.java @@ -84,7 +84,7 @@ public static void preprocess(PreprocessingJob preprocessingJob, ProgressBar tot int errors = 0; - final Preprocessed result = new Preprocessed(config.getPreprocessor().getParsers(), preprocessingJob); + final Preprocessed result = new Preprocessed(config, preprocessingJob); long lineId = 0; @@ -101,7 +101,7 @@ public static void preprocess(PreprocessingJob preprocessingJob, ProgressBar tot ConqueryMDC.setLocation(name); if (!(sourceFile.exists() && sourceFile.canRead())) { - throw new FileNotFoundException(sourceFile.getAbsolutePath().toString()); + throw new FileNotFoundException(sourceFile.getAbsolutePath()); } CsvParser parser = null; @@ -124,7 +124,7 @@ public static void preprocess(PreprocessingJob preprocessingJob, ProgressBar tot final GroovyPredicate filter = input.createFilter(headers); - DateReader dateReader = config.getPreprocessor().getParsers().getDateReader(); + DateReader dateReader = config.getLocale().getDateReader(); final OutputDescription.Output primaryOut = input.getPrimary().createForHeaders(headerMap, dateReader); final List outputs = new ArrayList<>(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/Parser.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/Parser.java index ebc06d2601..3a76af5af7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/Parser.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/Parser.java @@ -2,7 +2,7 @@ import javax.annotation.Nonnull; -import com.bakdata.conquery.models.config.ParserConfig; +import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.events.EmptyStore; import com.bakdata.conquery.models.events.stores.root.ColumnStore; import com.bakdata.conquery.models.exceptions.ParsingException; @@ -27,7 +27,7 @@ @Slf4j public abstract class Parser { - private final ParserConfig config; + private final ConqueryConfig config; private int lines = 0; private int nullLines = 0; diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/BooleanParser.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/BooleanParser.java index 2eda23b24b..0e3364ebfc 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/BooleanParser.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/BooleanParser.java @@ -2,6 +2,7 @@ import javax.annotation.Nonnull; +import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.config.ParserConfig; import com.bakdata.conquery.models.events.stores.primitive.BitSetStore; import com.bakdata.conquery.models.events.stores.root.BooleanStore; @@ -13,7 +14,7 @@ @ToString(callSuper = true) public class BooleanParser extends Parser { - public BooleanParser(ParserConfig config) { + public BooleanParser(ConqueryConfig config) { super(config); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/DateParser.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/DateParser.java index 6c9da6a11b..e3dc892c21 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/DateParser.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/DateParser.java @@ -3,6 +3,7 @@ import javax.annotation.Nonnull; import com.bakdata.conquery.models.common.CDate; +import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.config.ParserConfig; import com.bakdata.conquery.models.events.stores.primitive.IntegerDateStore; import com.bakdata.conquery.models.events.stores.root.DateStore; @@ -20,10 +21,10 @@ public class DateParser extends Parser { private IntegerParser subType; private DateReader dateReader; - public DateParser(ParserConfig config) { + public DateParser(ConqueryConfig config) { super(config); subType = new IntegerParser(config); - dateReader = config.getDateReader(); + dateReader = config.getLocale().getDateReader(); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/DateRangeParser.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/DateRangeParser.java index 9fb8b9e473..9afb72abf4 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/DateRangeParser.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/DateRangeParser.java @@ -3,6 +3,7 @@ import javax.annotation.Nonnull; import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.config.ParserConfig; import com.bakdata.conquery.models.events.stores.root.DateRangeStore; import com.bakdata.conquery.models.events.stores.specific.DateRangeTypeDateRange; @@ -28,11 +29,11 @@ public class DateRangeParser extends Parser { private int minValue = Integer.MAX_VALUE; private boolean anyOpen; - public DateRangeParser(ParserConfig config) { + public DateRangeParser(ConqueryConfig config) { super(config); minParser = new DateParser(config); maxParser = new DateParser(config); - dateReader = config.getDateReader(); + dateReader = config.getLocale().getDateReader(); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/DecimalParser.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/DecimalParser.java index bdf768bf6e..a2ce37129e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/DecimalParser.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/DecimalParser.java @@ -3,6 +3,7 @@ import java.math.BigDecimal; import java.math.BigInteger; +import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.config.ParserConfig; import com.bakdata.conquery.models.events.stores.primitive.DecimalArrayStore; import com.bakdata.conquery.models.events.stores.root.DecimalStore; @@ -22,7 +23,7 @@ public class DecimalParser extends Parser { private transient int maxScale = Integer.MIN_VALUE; private transient BigDecimal maxAbs; - public DecimalParser(ParserConfig config) { + public DecimalParser(ConqueryConfig config) { super(config); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/IntegerParser.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/IntegerParser.java index e91e1b9888..91daba6665 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/IntegerParser.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/IntegerParser.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.models.preproc.parser.specific; +import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.config.ParserConfig; import com.bakdata.conquery.models.events.stores.primitive.ByteArrayStore; import com.bakdata.conquery.models.events.stores.primitive.IntArrayStore; @@ -23,7 +24,7 @@ public class IntegerParser extends Parser { private long minValue = Long.MAX_VALUE; private long maxValue = Long.MIN_VALUE; - public IntegerParser(ParserConfig config) { + public IntegerParser(ConqueryConfig config) { super(config); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/MoneyParser.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/MoneyParser.java index 71cf90c661..3d8049b4d0 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/MoneyParser.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/MoneyParser.java @@ -24,9 +24,9 @@ public class MoneyParser extends Parser { @JsonIgnore private final BigDecimal moneyFactor; - public MoneyParser(ParserConfig config) { + public MoneyParser(ConqueryConfig config) { super(config); - moneyFactor = BigDecimal.valueOf(10).pow(config.getCurrency().getDefaultFractionDigits()); + moneyFactor = BigDecimal.valueOf(10).pow(config.getPreprocessor().getParsers().getCurrency().getDefaultFractionDigits()); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/RealParser.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/RealParser.java index 2a053002eb..3c53b8a111 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/RealParser.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/RealParser.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.models.preproc.parser.specific; +import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.config.ParserConfig; import com.bakdata.conquery.models.events.stores.primitive.DoubleArrayStore; import com.bakdata.conquery.models.events.stores.primitive.FloatArrayStore; @@ -20,9 +21,9 @@ public class RealParser extends Parser { private double floatULP = Float.NEGATIVE_INFINITY; - public RealParser(ParserConfig config) { + public RealParser(ConqueryConfig config) { super(config); - requiredPrecision = config.getMinPrecision(); + requiredPrecision = config.getPreprocessor().getParsers().getMinPrecision(); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/StringParser.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/StringParser.java index aaa42e51cc..995b1303d5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/StringParser.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/StringParser.java @@ -8,6 +8,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.config.ParserConfig; import com.bakdata.conquery.models.events.EmptyStore; import com.bakdata.conquery.models.events.stores.primitive.BitSetStore; @@ -56,7 +57,7 @@ public class StringParser extends Parser { private String prefix; private String suffix; - public StringParser(ParserConfig config) { + public StringParser(ConqueryConfig config) { super(config); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/string/NumberTypeGuesser.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/string/NumberTypeGuesser.java index 741f3b8b9c..d7cf236a18 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/string/NumberTypeGuesser.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/parser/specific/string/NumberTypeGuesser.java @@ -5,6 +5,7 @@ import com.bakdata.conquery.models.common.Range; import com.bakdata.conquery.models.common.Range.IntegerRange; +import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.config.ParserConfig; import com.bakdata.conquery.models.events.stores.root.IntegerStore; import com.bakdata.conquery.models.events.stores.root.StringStore; @@ -22,7 +23,7 @@ public class NumberTypeGuesser extends StringTypeGuesser { private final StringParser p; - private final ParserConfig config; + private final ConqueryConfig config; @Override public Guess createGuess() { diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java index 4d605335ad..c0034d536c 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java @@ -301,7 +301,7 @@ public void setStructure(Namespace namespace, StructureNode[] structure) { @SneakyThrows public void addImport(Namespace namespace, InputStream inputStream) throws IOException { - ImportJob job = ImportJob.create(namespace, inputStream, config.getCluster().getEntityBucketSize(), sharedDictionaryLocks); + ImportJob job = ImportJob.create(namespace, inputStream, config.getCluster().getEntityBucketSize(), sharedDictionaryLocks, config); namespace.getJobManager().addSlowJob(job); } diff --git a/backend/src/main/java/com/bakdata/conquery/util/DateReader.java b/backend/src/main/java/com/bakdata/conquery/util/DateReader.java index bd1ec3be4b..70ff6d92ed 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/DateReader.java +++ b/backend/src/main/java/com/bakdata/conquery/util/DateReader.java @@ -9,6 +9,7 @@ import java.util.List; import java.util.Locale; import java.util.Set; +import java.util.stream.Collectors; import javax.validation.constraints.NotEmpty; import javax.validation.constraints.NotNull; @@ -31,14 +32,6 @@ @NoArgsConstructor public class DateReader { - @NotNull - @NotEmpty - @Getter - private List formats = List.of( - "yyyy-MM-dd", "yyyyMMdd", "dd.MM.yyyy" - ); - - /** * All available formats for parsing. */ @@ -49,7 +42,7 @@ public class DateReader { * Last successfully parsed dateformat. */ @JsonIgnore - private ThreadLocal lastFormat = new ThreadLocal<>(); + private final ThreadLocal lastFormat = new ThreadLocal<>(); @JsonIgnore private final LocalDate ERROR_DATE = LocalDate.MIN; @@ -64,15 +57,8 @@ public class DateReader { .build(CacheLoader.from(this::tryParse)); @JsonCreator - public DateReader(@NotEmpty List formats) { - final Set formatters = new HashSet<>(); - - - for (String p : formats) { - formatters.add(createFormatter(p)); - } - - this.dateFormats = Collections.unmodifiableSet(formatters); + public DateReader(@NotEmpty Set dateParsingFormats) { + this.dateFormats = dateParsingFormats.stream().map(DateTimeFormatter::ofPattern).collect(Collectors.toSet()); } /** diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/ResultNameTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/ResultNameTest.java index 617d07ae9d..abc2aed9cc 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/ResultNameTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/ResultNameTest.java @@ -1,7 +1,7 @@ package com.bakdata.conquery.io.result; import com.bakdata.conquery.util.io.FileUtil; -import org.junit.Test; +import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; diff --git a/backend/src/test/java/com/bakdata/conquery/models/events/stores/types/MajorTypesTest.java b/backend/src/test/java/com/bakdata/conquery/models/events/stores/types/MajorTypesTest.java index 5dd6624b98..e62584181b 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/events/stores/types/MajorTypesTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/events/stores/types/MajorTypesTest.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.models.events.stores.types; +import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.config.ParserConfig; import com.bakdata.conquery.models.events.MajorTypeId; import org.junit.jupiter.params.ParameterizedTest; @@ -18,6 +19,6 @@ public static MajorTypeId[] reflection() { @ParameterizedTest @MethodSource public void reflection(MajorTypeId typeId) { - typeId.createParser(new ParserConfig()).findBestType(); + typeId.createParser(new ConqueryConfig()).findBestType(); } } diff --git a/backend/src/test/java/com/bakdata/conquery/models/events/stores/types/StringEncodingTest.java b/backend/src/test/java/com/bakdata/conquery/models/events/stores/types/StringEncodingTest.java index 78b265c1c3..f8cbe214cd 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/events/stores/types/StringEncodingTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/events/stores/types/StringEncodingTest.java @@ -6,6 +6,7 @@ import java.util.UUID; import java.util.stream.Stream; +import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.config.ParserConfig; import com.bakdata.conquery.models.events.stores.specific.string.StringTypeEncoded; import com.bakdata.conquery.models.exceptions.ParsingException; @@ -37,7 +38,7 @@ public Stream testEncodings() { @Test public void testHexStreamStringType() { - StringParser parser = new StringParser(new ParserConfig()); + StringParser parser = new StringParser(new ConqueryConfig()); Stream.generate(() -> UUID.randomUUID().toString().replace("-", "")) .map(String::toUpperCase) diff --git a/backend/src/test/java/com/bakdata/conquery/models/externalservice/ResultTypeTest.java b/backend/src/test/java/com/bakdata/conquery/models/externalservice/ResultTypeTest.java index b92765f8d4..40afdee927 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/externalservice/ResultTypeTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/externalservice/ResultTypeTest.java @@ -31,7 +31,7 @@ public class ResultTypeTest { I18n.init(); //init global default config CONFIG.getPreprocessor().getParsers().setCurrency(Currency.getInstance("EUR")); - CONFIG.getLocale().setDateFormatMapping(Map.of(Locale.GERMAN, DateTimeFormatter.ofPattern("dd.MM.yyyy"))); + CONFIG.getLocale().setDateFormatMapping(Map.of(Locale.GERMAN, "dd.MM.yyyy")); } private static final PrintSettings PRETTY = new PrintSettings(true, Locale.ENGLISH, null, CONFIG, null); diff --git a/backend/src/test/java/com/bakdata/conquery/models/preproc/parser/specific/DateRangeParserTest.java b/backend/src/test/java/com/bakdata/conquery/models/preproc/parser/specific/DateRangeParserTest.java index 9c110dd54f..5d9dd4d74f 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/preproc/parser/specific/DateRangeParserTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/preproc/parser/specific/DateRangeParserTest.java @@ -6,6 +6,7 @@ import com.bakdata.conquery.models.common.QuarterUtils; import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.config.ParserConfig; import com.bakdata.conquery.models.events.stores.primitive.ByteArrayStore; import com.bakdata.conquery.models.events.stores.primitive.IntegerDateStore; @@ -18,7 +19,7 @@ class DateRangeParserTest { @Test public void onlyClosed() { - final DateRangeParser parser = new DateRangeParser(new ParserConfig()); + final DateRangeParser parser = new DateRangeParser(new ConqueryConfig()); List.of(CDateRange.of(10,11), CDateRange.exactly(10)) .forEach(parser::addLine); @@ -34,7 +35,7 @@ public void onlyClosed() { @Test public void notOnlyClosed() { - final DateRangeParser parser = new DateRangeParser(new ParserConfig()); + final DateRangeParser parser = new DateRangeParser(new ConqueryConfig()); List.of(CDateRange.of(10,11), CDateRange.exactly(10), CDateRange.atMost(10)) .forEach(parser::addLine); @@ -44,7 +45,7 @@ public void notOnlyClosed() { @Test public void onlyQuarters() { - final DateRangeParser parser = new DateRangeParser(new ParserConfig()); + final DateRangeParser parser = new DateRangeParser(new ConqueryConfig()); List.of(CDateRange.of(QuarterUtils.getFirstDayOfQuarter(2011,1), QuarterUtils.getLastDayOfQuarter(2011,1))) .forEach(parser::addLine); diff --git a/backend/src/test/java/com/bakdata/conquery/models/preproc/parser/specific/DecimalParserTest.java b/backend/src/test/java/com/bakdata/conquery/models/preproc/parser/specific/DecimalParserTest.java index ea606b700b..cb9db1bfab 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/preproc/parser/specific/DecimalParserTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/preproc/parser/specific/DecimalParserTest.java @@ -4,6 +4,7 @@ import java.math.BigDecimal; +import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.config.ParserConfig; import com.bakdata.conquery.models.events.stores.primitive.DecimalArrayStore; import com.bakdata.conquery.models.events.stores.specific.DecimalTypeScaled; @@ -15,7 +16,7 @@ class DecimalParserTest { @Test public void test(){ - final DecimalParser parser = new DecimalParser(new ParserConfig()); + final DecimalParser parser = new DecimalParser(new ConqueryConfig()); parser.addLine(BigDecimal.valueOf(10,1000)); parser.addLine(BigDecimal.valueOf(30,1000)); diff --git a/backend/src/test/java/com/bakdata/conquery/models/preproc/parser/specific/IntegerParserTest.java b/backend/src/test/java/com/bakdata/conquery/models/preproc/parser/specific/IntegerParserTest.java index 0943e9db78..65404a6642 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/preproc/parser/specific/IntegerParserTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/preproc/parser/specific/IntegerParserTest.java @@ -5,6 +5,7 @@ import java.util.function.Consumer; import java.util.stream.Stream; +import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.config.ParserConfig; import com.bakdata.conquery.models.events.stores.primitive.ByteArrayStore; import com.bakdata.conquery.models.events.stores.primitive.IntArrayStore; @@ -61,7 +62,7 @@ public static Consumer rebased(Class clazz) { @ParameterizedTest @MethodSource("arguments") public void test(long min, long max, Consumer test) { - final IntegerParser parser = new IntegerParser(new ParserConfig()); + final IntegerParser parser = new IntegerParser(new ConqueryConfig()); parser.setMinValue(min); parser.setMaxValue(max); diff --git a/backend/src/test/java/com/bakdata/conquery/models/preproc/parser/specific/RealParserTest.java b/backend/src/test/java/com/bakdata/conquery/models/preproc/parser/specific/RealParserTest.java index 17d680b03a..962e312ea0 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/preproc/parser/specific/RealParserTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/preproc/parser/specific/RealParserTest.java @@ -2,6 +2,7 @@ import static org.assertj.core.api.Assertions.assertThat; +import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.config.ParserConfig; import com.bakdata.conquery.models.events.stores.primitive.DoubleArrayStore; import com.bakdata.conquery.models.events.stores.primitive.FloatArrayStore; @@ -11,10 +12,10 @@ class RealParserTest { @Test public void ulpInRange() { - final ParserConfig parserConfig = new ParserConfig(); - parserConfig.setMinPrecision(Math.ulp(10)); + final ConqueryConfig config = new ConqueryConfig(); + config.getPreprocessor().getParsers().setMinPrecision(Math.ulp(10)); - final RealParser realParser = new RealParser(parserConfig); + final RealParser realParser = new RealParser(config); realParser.registerValue(1d); realParser.registerValue(2d); From 37e9ddfc9ca4625d645b0106ee574bb68a704040 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 11 Aug 2021 15:59:00 +0200 Subject: [PATCH 66/82] generalize parsing format of CDateSet and CDateRange --- .../concept/specific/external/DateFormat.java | 2 +- .../serializer/CDateSetDeserializer.java | 2 +- .../conquery/models/common/CDateSet.java | 22 +-- .../conquery/models/config/LocaleConfig.java | 33 ++++- .../com/bakdata/conquery/util/DateReader.java | 136 ++++++++++++++++-- .../integration/common/CDateSetTest.java | 10 ++ 6 files changed, 172 insertions(+), 33 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java index 75c180bae5..f14b283e7d 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java @@ -59,7 +59,7 @@ public void readDates(String value, DateReader dateReader, CDateSet out) { DATE_SET { @Override public void readDates(String value, DateReader dateReader, CDateSet out) { - out.addAll(CDateSet.parse(value, dateReader)); + out.addAll(dateReader.parseToCDateSet(value)); } }, ALL { diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/CDateSetDeserializer.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/CDateSetDeserializer.java index 356cd167e7..f3d0e35318 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/CDateSetDeserializer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/CDateSetDeserializer.java @@ -36,7 +36,7 @@ public CDateSet deserialize(JsonParser p, DeserializationContext ctxt) throws IO } if (p.currentToken() == JsonToken.VALUE_STRING) { - return CDateSet.parse(p.readValueAs(String.class), formats); + return formats.parseToCDateSet(p.readValueAs(String.class)); } return (CDateSet) ctxt.handleUnexpectedToken(CDateSet.class, p.currentToken(), p, "can't deserialize CDateSet"); diff --git a/backend/src/main/java/com/bakdata/conquery/models/common/CDateSet.java b/backend/src/main/java/com/bakdata/conquery/models/common/CDateSet.java index d54efeba74..257b3725be 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/common/CDateSet.java +++ b/backend/src/main/java/com/bakdata/conquery/models/common/CDateSet.java @@ -22,6 +22,7 @@ import com.google.common.collect.ForwardingCollection; import com.google.common.math.IntMath; import lombok.EqualsAndHashCode; +import lombok.NonNull; /** * (De-)Serializers are are registered programmatically because they depend on {@link DateReader} @@ -29,7 +30,8 @@ @EqualsAndHashCode public class CDateSet { - private static final Pattern PARSE_PATTERN = Pattern.compile("(\\{|,\\s*)((\\d{4}-\\d{2}-\\d{2})?/(\\d{4}-\\d{2}-\\d{2})?)"); + private static final String PARSE_PATTERN_TEMPLATE = "(\\%1$c|%2$c\\s*)(([^%1$c%2$c%3$c%4$c]*+)/([^^%1$c%2$c%3$c%4$c]*+))"; // set_begin, range_sep, range_start_end_sep, set_end + private static final Pattern PARSE_PATTERN = Pattern.compile("(\\{|,\\s*)(([^/{}]*+)/([^/{}]*+))"); private final NavigableMap rangesByLowerBound; private transient Set asRanges; private transient Set asDescendingSetOfRanges; @@ -37,6 +39,7 @@ public class CDateSet { public static CDateSet create() { return new CDateSet(new TreeMap<>()); } + public static CDateSet createFull() { CDateSet set = new CDateSet(new TreeMap<>()); @@ -444,20 +447,5 @@ public int getMinValue() { public int getMaxValue() { return rangesByLowerBound.lastEntry().getValue().getMaxValue(); } - - public static CDateSet parse(String value, DateReader dateReader) { - List ranges = PARSE_PATTERN - .matcher(value) - .results() - .map(mr -> { - try { - return DateRangeParser.parseISORange(mr.group(2), dateReader); - } - catch(Exception e) { - throw new RuntimeException(e); - } - }) - .collect(Collectors.toList()); - return CDateSet.create(ranges); - } + } \ No newline at end of file diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java index 47a383ddc6..89e419c9d6 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java @@ -5,6 +5,8 @@ import java.time.format.DateTimeFormatter; import java.util.*; +import javax.validation.constraints.Max; +import javax.validation.constraints.Min; import javax.validation.constraints.NotEmpty; import javax.validation.constraints.NotNull; @@ -17,8 +19,7 @@ import com.fasterxml.jackson.databind.JsonDeserializer; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.google.common.collect.Sets; -import lombok.Getter; -import lombok.Setter; +import lombok.*; @Getter @Setter public class LocaleConfig { @@ -36,13 +37,37 @@ public class LocaleConfig { "dd.MM.yyyy" ); + @NotEmpty + private List rangeStartEndSeperators = List.of("-","/"); + + @NotNull + @NotEmpty + private List dateSetLayouts = List.of( + new DateSetLayout("", ",", ""), + new DateSetLayout("{", ",", "}")); + + @Data + @AllArgsConstructor + public static class DateSetLayout { + @NonNull @Max(1) + String setBegin; + @NonNull @Min(1) @Max(1) + String rangeSep; + @NonNull @Max(1) + String setEnd; + } + /** * Date formats that are available for parsing. */ @JsonIgnore public DateReader getDateReader() { - return new DateReader(Sets.union(dateParsingFormats, Set.copyOf(dateFormatMapping.values()))); + return new DateReader( + Sets.union(dateParsingFormats, Set.copyOf(dateFormatMapping.values())), + rangeStartEndSeperators, + dateSetLayouts + ); } /** @@ -82,4 +107,6 @@ public Locale findClosestMatch(Locale forLocale) { } return null; } + + } diff --git a/backend/src/main/java/com/bakdata/conquery/util/DateReader.java b/backend/src/main/java/com/bakdata/conquery/util/DateReader.java index 70ff6d92ed..6068120642 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/DateReader.java +++ b/backend/src/main/java/com/bakdata/conquery/util/DateReader.java @@ -4,32 +4,34 @@ import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatterBuilder; import java.time.format.DateTimeParseException; -import java.util.Collections; -import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Set; +import java.util.regex.Pattern; import java.util.stream.Collectors; import javax.validation.constraints.NotEmpty; -import javax.validation.constraints.NotNull; +import com.bakdata.conquery.models.common.CDateSet; +import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.bakdata.conquery.models.config.LocaleConfig; import com.bakdata.conquery.models.exceptions.ParsingException; +import com.bakdata.conquery.models.preproc.parser.specific.DateRangeParser; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.base.Strings; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; -import lombok.Getter; import lombok.NoArgsConstructor; +import lombok.NonNull; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; /** * Utility class for parsing multiple dateformats. Parsing is cached in two ways: First parsed values are cached. Second, the last used parser is cached since it's likely that it will be used again, we therefore try to use it first, then try all others. */ @Slf4j -@NoArgsConstructor public class DateReader { /** @@ -38,11 +40,28 @@ public class DateReader { @JsonIgnore private Set dateFormats; + @JsonIgnore + private List rangeStartEndSeperators; + + @JsonIgnore + private List dateSetLayouts; + + /** + * Last successfully parsed date format. + */ + @JsonIgnore + private final ThreadLocal lastDateFormat = new ThreadLocal<>(); /** - * Last successfully parsed dateformat. + * Last successfully parsed range format. */ @JsonIgnore - private final ThreadLocal lastFormat = new ThreadLocal<>(); + private final ThreadLocal lastRangeFormat = new ThreadLocal<>(); + + /** + * Last successfully parsed dateset format. + */ + @JsonIgnore + private final ThreadLocal lastDateSetLayout = new ThreadLocal<>(); @JsonIgnore private final LocalDate ERROR_DATE = LocalDate.MIN; @@ -57,8 +76,10 @@ public class DateReader { .build(CacheLoader.from(this::tryParse)); @JsonCreator - public DateReader(@NotEmpty Set dateParsingFormats) { + public DateReader(Set dateParsingFormats, List rangeStartEndSeperators, List dateSetLayouts) { this.dateFormats = dateParsingFormats.stream().map(DateTimeFormatter::ofPattern).collect(Collectors.toSet()); + this.rangeStartEndSeperators = rangeStartEndSeperators; + this.dateSetLayouts = dateSetLayouts; } /** @@ -72,12 +93,105 @@ public LocalDate parseToLocalDate(String value) throws ParsingException { final LocalDate out = DATE_CACHE.getUnchecked(value); if (out.equals(ERROR_DATE)) { - throw new IllegalArgumentException(String.format("Failed to parse `%s` as LocalDate.", value)); + throw new ParsingException(String.format("Failed to parse `%s` as LocalDate.", value)); } return out; } + public CDateRange parseToCDateRange(String value) { + if (Strings.isNullOrEmpty(value)) { + return null; + } + + CDateRange result = null; + + final String lastSep = lastRangeFormat.get(); + if (lastSep != null) { + try{ + return parseToCDateRange(value,lastSep); + } catch (ParsingException e) { + log.trace("Parsing with last used config failed for date range: " + value, e); + } + } + + for(String sep : rangeStartEndSeperators) { + try { + result = parseToCDateRange(value,sep); + } catch (ParsingException e) { + log.trace("Parsing failed for date range: " + value, e); + continue; + } + lastRangeFormat.set(sep); + } + if (result != null) { + return result; + } + throw new ParsingException("Non of the configured formats allowed to parse the date range: " + value); + } + + private CDateRange parseToCDateRange(String value, String sep) { + String[] parts = StringUtils.split(value, sep); + if (parts.length != 2) { + throw ParsingException.of(value, "daterange"); + } + + return CDateRange.of( + parseToLocalDate(parts[0]), + parseToLocalDate(parts[1]) + ); + } + + public CDateSet parseToCDateSet(String value) { + if (Strings.isNullOrEmpty(value)) { + return null; + } + + CDateSet result = null; + + final Pattern lastDateSet = lastDateSetLayout.get(); + if (lastDateSet != null) { + try{ + return parseToCDateSet(value,lastDateSet); + } catch (ParsingException e) { + log.trace("Parsing with last used config failed for date set: " + value, e); + } + } + + for(LocaleConfig.DateSetLayout sep : dateSetLayouts) { + Pattern regexPattern = generateDateSetPattern(sep.getSetBegin(), sep.getRangeSep(), sep.getSetEnd()); + try { + result = parseToCDateSet(value,regexPattern); + } catch (ParsingException e) { + log.trace("Parsing failed for date set: " + value, e); + continue; + } + lastDateSetLayout.set(regexPattern); + } + if (result != null) { + return result; + } + throw new ParsingException("Non of the configured formats allowed to parse the date set: " + value); + } + + + public CDateSet parseToCDateSet(String value, Pattern pattern) { + List ranges = pattern.matcher(value) + .results() + .map(mr -> parseToCDateRange(mr.group(2))) + .collect(Collectors.toList()); + return CDateSet.create(ranges); + + } + + private static Pattern generateDateSetPattern(@NonNull String setBegin, @NonNull String rangeSep, @NonNull String setEnd) { + assert(setBegin.length() < 2); + assert(rangeSep.length() == 1); + assert(setEnd.length() < 2); + + return Pattern.compile(String.format("(%1$s|%2$s\\s*)([^%1$s%2$s%3$s]*)", Pattern.quote(setBegin), Pattern.quote(rangeSep), Pattern.quote(setEnd))); + } + /** * Try and parse with the last successful parser. If not successful try and parse with other parsers and update the last successful parser. *

@@ -85,7 +199,7 @@ public LocalDate parseToLocalDate(String value) throws ParsingException { */ private LocalDate tryParse(String value) { - final DateTimeFormatter formatter = lastFormat.get(); + final DateTimeFormatter formatter = lastDateFormat.get(); if (formatter != null) { try { @@ -100,7 +214,7 @@ private LocalDate tryParse(String value) { if (formatter != format) { try { LocalDate res = LocalDate.parse(value, format); - lastFormat.set(format); + lastDateFormat.set(format); return res; } catch (DateTimeParseException e) { diff --git a/backend/src/test/java/com/bakdata/conquery/integration/common/CDateSetTest.java b/backend/src/test/java/com/bakdata/conquery/integration/common/CDateSetTest.java index 2eeb259d0c..93877de1dd 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/common/CDateSetTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/common/CDateSetTest.java @@ -3,11 +3,13 @@ import static org.assertj.core.api.Assertions.assertThat; import java.time.LocalDate; +import java.util.Arrays; import java.util.List; import java.util.stream.Stream; import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.bakdata.conquery.models.config.ConqueryConfig; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -195,4 +197,12 @@ public void testMaskedAddAtLeastMaskMultiple() { assertThat(set.asRanges()).containsExactly(CDateRange.of(5, 10), CDateRange.atLeast(30)); } + @ParameterizedTest(name="{0}") + @MethodSource("arguments") + public void parse(String input, CDateRange[] expected) { + ConqueryConfig config = new ConqueryConfig(); + CDateSet set = config.getLocale().getDateReader().parseToCDateSet(input); + assertThat(set).isEqualTo(CDateSet.create(Arrays.asList(expected))); + } + } From 31a78c9d369db9fef78138b57178f468cd205b97 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 11 Aug 2021 17:15:49 +0200 Subject: [PATCH 67/82] correct regex to work correctly with empty values --- .../com/bakdata/conquery/util/DateReader.java | 9 ++++--- .../integration/common/CDateSetTest.java | 24 +++++++++++++++---- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/util/DateReader.java b/backend/src/main/java/com/bakdata/conquery/util/DateReader.java index 6068120642..7159b4832d 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/DateReader.java +++ b/backend/src/main/java/com/bakdata/conquery/util/DateReader.java @@ -163,7 +163,7 @@ public CDateSet parseToCDateSet(String value) { try { result = parseToCDateSet(value,regexPattern); } catch (ParsingException e) { - log.trace("Parsing failed for date set: " + value, e); + log.trace("Parsing failed for date set '" + value + "' with pattern '" + regexPattern + "'", e); continue; } lastDateSetLayout.set(regexPattern); @@ -178,7 +178,7 @@ public CDateSet parseToCDateSet(String value) { public CDateSet parseToCDateSet(String value, Pattern pattern) { List ranges = pattern.matcher(value) .results() - .map(mr -> parseToCDateRange(mr.group(2))) + .map(mr -> parseToCDateRange(mr.group(1))) .collect(Collectors.toList()); return CDateSet.create(ranges); @@ -189,7 +189,10 @@ private static Pattern generateDateSetPattern(@NonNull String setBegin, @NonNull assert(rangeSep.length() == 1); assert(setEnd.length() < 2); - return Pattern.compile(String.format("(%1$s|%2$s\\s*)([^%1$s%2$s%3$s]*)", Pattern.quote(setBegin), Pattern.quote(rangeSep), Pattern.quote(setEnd))); + return Pattern.compile(String.format("(?:(?:%1$s)|(?:%2$s\\s*))([^%1$s%2$s%3$s]+)(?:%3$s)?", + setBegin.isEmpty() ? "" : Pattern.quote(setBegin), + Pattern.quote(rangeSep), + setEnd.isEmpty() ? "" : Pattern.quote(setEnd))); } /** diff --git a/backend/src/test/java/com/bakdata/conquery/integration/common/CDateSetTest.java b/backend/src/test/java/com/bakdata/conquery/integration/common/CDateSetTest.java index 93877de1dd..2b2522fd31 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/common/CDateSetTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/common/CDateSetTest.java @@ -10,6 +10,7 @@ import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.config.ConqueryConfig; +import net.bytebuddy.asm.Advice; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -17,6 +18,8 @@ public class CDateSetTest { + private static ConqueryConfig config = new ConqueryConfig(); + public static Stream arguments() { return Stream .of( @@ -197,12 +200,25 @@ public void testMaskedAddAtLeastMaskMultiple() { assertThat(set.asRanges()).containsExactly(CDateRange.of(5, 10), CDateRange.atLeast(30)); } + public static Stream argumentsParsing() { + return Stream + .of( + Arguments.of( + "{2000-01-01/2000-01-01}", + CDateSet.create(CDateRange.of(LocalDate.of(2000,01,01), LocalDate.of(2000,01,01))) + ), + Arguments.of( + "01.01.2000-01.01.2000", + CDateSet.create(CDateRange.of(LocalDate.of(2000,01,01), LocalDate.of(2000,01,01))) + ) + ); + } + @ParameterizedTest(name="{0}") - @MethodSource("arguments") - public void parse(String input, CDateRange[] expected) { - ConqueryConfig config = new ConqueryConfig(); + @MethodSource("argumentsParsing") + public void parse(String input, CDateSet expected) { CDateSet set = config.getLocale().getDateReader().parseToCDateSet(input); - assertThat(set).isEqualTo(CDateSet.create(Arrays.asList(expected))); + assertThat(set).isEqualTo(expected); } } From 4e0bbb077752515501cbfb379f7b419802ba5468 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 11 Aug 2021 15:18:29 +0000 Subject: [PATCH 68/82] Update AutoDoc --- docs/Config JSON.md | 26 +++++++++++++++----------- docs/REST API JSONs.md | 20 ++++++++++---------- 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/docs/Config JSON.md b/docs/Config JSON.md index 9bc446139e..5e50de37cf 100644 --- a/docs/Config JSON.md +++ b/docs/Config JSON.md @@ -223,7 +223,7 @@ Supported Fields: | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L68) | storage | `@Valid @NotNull StoreFactory` | | | |

-### Type CurrencyConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L227) +### Type CurrencyConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L193)
Details

@@ -234,13 +234,13 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L232) | decimalScale | `int` | `2` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L231) | decimalSeparator | `String` | `","` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L229) | prefix | `String` | `"€"` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L230) | thousandSeparator | `String` | `"."` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L198) | decimalScale | `int` | `2` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L197) | decimalSeparator | `String` | `","` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L195) | prefix | `String` | `"€"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L196) | thousandSeparator | `String` | `"."` | | |

-### Type FrontendConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L37) +### Type FrontendConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L41)
Details

@@ -251,12 +251,12 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L47) | currency | [CurrencyConfig](#Type-CurrencyConfig) | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L51) | queryUpload | `UploadConfig` | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L46) | version | `String` | `"0.0.0-SNAPSHOT"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L51) | currency | [CurrencyConfig](#Type-CurrencyConfig) | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L55) | queryUpload | `UploadConfig` | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L50) | version | `String` | `"0.0.0-SNAPSHOT"` | | |

-### Type LocaleConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L11) +### Type LocaleConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L24)
Details

@@ -267,7 +267,11 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L13) | frontend | `@NotNull Locale` | `""` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L29) | dateFormatMapping | map from `Locale` to `String` | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L33) | dateParsingFormats | `@NotNull Set` | `["yyyy-MM-dd","yyyyMMdd","dd.MM.yyyy"]` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L43) | dateSetLayouts | list of `DateSetLayout` | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L26) | frontend | `@NotNull Locale` | `""` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L40) | rangeStartEndSeperators | list of `String` | `["-","/"]` | | |

### Type MinaConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/MinaConfig.java#L13) diff --git a/docs/REST API JSONs.md b/docs/REST API JSONs.md index d8d152f2eb..573fc5ba22 100644 --- a/docs/REST API JSONs.md +++ b/docs/REST API JSONs.md @@ -509,7 +509,7 @@ Supported Fields: | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQElement.java#L31-L33) | label | `String` | ? | | Allows the user to define labels. |

-### EXTERNAL [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java#L37-L39) +### EXTERNAL [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java#L38-L40) Allows uploading lists of entities.
Details

@@ -755,7 +755,7 @@ Supported Fields: | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptResource.java#L68) | concepts | list of `String` | `null` | | |

-### Type CurrencyConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L227) +### Type CurrencyConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L193)
Details

@@ -766,10 +766,10 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L232) | decimalScale | `int` | `2` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L231) | decimalSeparator | `String` | `","` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L229) | prefix | `String` | `"€"` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L230) | thousandSeparator | `String` | `"."` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L198) | decimalScale | `int` | `2` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L197) | decimalSeparator | `String` | `","` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L195) | prefix | `String` | `"€"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L196) | thousandSeparator | `String` | `"."` | | |

### Type ExecutionStatus [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/ExecutionStatus.java#L18) @@ -885,7 +885,7 @@ No fields can be set for this type.

-### Type FrontendConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L37) +### Type FrontendConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L41)
Details

@@ -896,9 +896,9 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L47) | currency | [CurrencyConfig](#Type-CurrencyConfig) | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L51) | queryUpload | `UploadConfig` | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L46) | version | `String` | `"0.0.0-SNAPSHOT"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L51) | currency | [CurrencyConfig](#Type-CurrencyConfig) | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L55) | queryUpload | `UploadConfig` | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L50) | version | `String` | `"0.0.0-SNAPSHOT"` | | |

### Type FullExecutionStatus [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/FullExecutionStatus.java#L20-L24) From 24b2b04c9db79fa4403d3139aca54d34385ebe47 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 11 Aug 2021 17:49:35 +0200 Subject: [PATCH 69/82] adds missing break statements --- backend/src/main/java/com/bakdata/conquery/util/DateReader.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/src/main/java/com/bakdata/conquery/util/DateReader.java b/backend/src/main/java/com/bakdata/conquery/util/DateReader.java index 7159b4832d..77cd49a025 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/DateReader.java +++ b/backend/src/main/java/com/bakdata/conquery/util/DateReader.java @@ -123,6 +123,7 @@ public CDateRange parseToCDateRange(String value) { continue; } lastRangeFormat.set(sep); + break; } if (result != null) { return result; @@ -167,6 +168,7 @@ public CDateSet parseToCDateSet(String value) { continue; } lastDateSetLayout.set(regexPattern); + break; } if (result != null) { return result; From dfa62b006043ff212c84ce87ac82afadb0312190 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 11 Aug 2021 17:52:52 +0200 Subject: [PATCH 70/82] moa tests --- .../integration/common/CDateSetTest.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/backend/src/test/java/com/bakdata/conquery/integration/common/CDateSetTest.java b/backend/src/test/java/com/bakdata/conquery/integration/common/CDateSetTest.java index 2b2522fd31..c74f720ed6 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/common/CDateSetTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/common/CDateSetTest.java @@ -210,6 +210,22 @@ public static Stream argumentsParsing() { Arguments.of( "01.01.2000-01.01.2000", CDateSet.create(CDateRange.of(LocalDate.of(2000,01,01), LocalDate.of(2000,01,01))) + ), + Arguments.of( + "{2000-01-01/2000-01-01, 2001-01-01/2001-01-01}", + CDateSet.create( + List.of( + CDateRange.of(LocalDate.of(2000,01,01), LocalDate.of(2000,01,01)), + CDateRange.of(LocalDate.of(2001,01,01), LocalDate.of(2001,01,01))) + ) + ), + Arguments.of( + "01.01.2000-01.01.2000, 01.01.2001-01.01.2001", + CDateSet.create( + List.of( + CDateRange.of(LocalDate.of(2000,01,01), LocalDate.of(2000,01,01)), + CDateRange.of(LocalDate.of(2001,01,01), LocalDate.of(2001,01,01))) + ) ) ); } From a528f3eeb9cfa88fc2c0362928e3e446743e3ccb Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 11 Aug 2021 18:35:35 +0200 Subject: [PATCH 71/82] WIP fix tests --- .../concept/specific/external/DateFormat.java | 2 +- .../conquery/models/config/LocaleConfig.java | 52 +++++++++---------- .../models/externalservice/ResultType.java | 20 +++++-- .../conquery/models/query/PrintSettings.java | 16 +++--- .../com/bakdata/conquery/util/DateReader.java | 14 ++--- .../aggregator/COUNT_AGGREGATOR/expected.csv | 8 +-- .../COUNT_DISTINCT_AGGREGATOR/expected.csv | 8 +-- .../COUNT_QUARTERS_AGGREGATOR/expected.csv | 10 ++-- .../DATE_DISTANCE_AGGREGATOR/expected.csv | 20 +++---- .../DURATION_SUM_AGGREGATOR/expected.csv | 8 +-- .../expected_no_restriction.csv | 20 +++---- .../expected_restriction.csv | 20 +++---- .../aggregator/EXISTS_AGGREGATOR/expected.csv | 6 +-- .../EXISTS_AGGREGATOR_AGGREGATED/expected.csv | 6 +-- .../EXISTS_AGGREGATOR_OR/expected.csv | 6 +-- .../aggregator/FIRST_AGGREGATOR/expected.csv | 10 ++-- .../aggregator/LAST_AGGREGATOR/expected.csv | 10 ++-- .../MULTI_SELECT_AGGREGATOR/expected.csv | 12 ++--- .../aggregator/PREFIX_AGGREGATOR/expected.csv | 8 +-- .../QUARTER_AGGREGATOR/expected.csv | 4 +- .../aggregator/RANDOM_AGGREGATOR/expected.csv | 8 +-- .../aggregator/SELECT_AGGREGATOR/expected.csv | 8 +-- .../aggregator/SUM_AGGREGATOR/expected.csv | 8 +-- .../SUM_DIFF_AGGREGATOR/expected.csv | 8 +-- .../aggregator/VALUES_AGGREGATOR/expected.csv | 10 ++-- .../expected_BIG_MULTI_SELECT_Filter.csv | 4 +- .../expected_BIG_MULTI_SELECT_Filter.csv | 6 +-- .../expected_BIG_MULTI_SELECT_Filter.csv | 4 +- .../expected_BIG_MULTI_SELECT_Filter.csv | 2 +- .../expected_BIG_MULTI_SELECT_Filter.csv | 2 +- .../filter/COUNT/expected_COUNT_Filter.csv | 4 +- .../COUNT_DISTINCT/expected_COUNT_Filter.csv | 4 +- .../expected_COUNT_Filter.csv | 4 +- .../expected_COUNT_QUARTERS_Filter.csv | 12 ++--- .../expected_COUNT_QUARTERS_Filter.csv | 4 +- .../COUNTfalse/expected_COUNT_Filter.csv | 4 +- .../DATE_DISTANCE/expected_NUMBER_Filter.csv | 4 +- .../DIFFSUM_INTEGER/expected_SUM_Filter.csv | 14 ++--- .../DIFFSUM_REAL/expected_SUM_Filter.csv | 14 ++--- .../expected_DURATION_SUM_Filter.csv | 6 +-- .../expected_DURATION_SUM_2_Filter.csv | 8 +-- .../MULTI_SELECT/expected_SELECT_Filter.csv | 6 +-- .../NUMBER_DECIMAL/expected_NUMBER_Filter.csv | 10 ++-- .../NUMBER_INTEGER/expected_NUMBER_Filter.csv | 8 +-- .../expected_NUMBER_Filter.csv | 2 +- .../NUMBER_MONEY/expected_NUMBER_Filter.csv | 10 ++-- .../NUMBER_REAL/expected_NUMBER_Filter.csv | 10 ++-- .../expected_NUMBER_Filter.csv | 2 +- .../filter/PREFIX/expected_SELECT_Filter.csv | 6 +-- .../expected_QUARTERS_IN_YEAR_Filter.csv | 14 ++--- .../filter/SELECT/expected_SELECT_Filter.csv | 4 +- .../expected_SINGLE_SELECT_Filter.csv | 4 +- .../expected_SINGLE_SELECT_Filter.csv | 2 +- .../expected_SINGLE_SELECT_Filter.csv | 2 +- .../expected_SINGLE_SELECT_Filter2.csv | 2 +- .../SUM_DECIMAL/expected_SUM_Filter.csv | 8 +-- .../SUM_INTEGER/expected_SUM_Filter.csv | 8 +-- .../ABSOLUT/SIMPLE/query_results_1.csv | 2 +- .../ENTITY_DATE/SIMPLE/query_results_1.csv | 2 +- .../RELATIVE_ONLY_FEATURE/query_results_1.csv | 4 +- .../query_results_1.csv | 4 +- .../RELATIVE_ONLY_OUTCOME/query_results_1.csv | 4 +- .../query_results_1.csv | 4 +- .../RELATIVE/SIMPLE/query_results_1.csv | 4 +- .../tests/form/FULL_EXPORT_FORM/expected.csv | 14 ++--- .../query/ARRAY_CONCEPT_QUERY/expected.csv | 8 +-- .../expected.csv | 2 +- .../expected.csv | 14 ++--- .../COMMON_CONCEPT_ICD_QUERY/expected-kh.csv | 4 +- .../expected.csv | 4 +- .../CONCEPT_RESTRICTION_QUERY/expected-kh.csv | 2 +- .../CONCEPT_WITHOUT_VALIDITYDATE/expected.csv | 4 +- .../query/CONNECTOR_CONDITION/expected.csv | 4 +- .../query/DATE_DISTANCE/expected-erster.csv | 4 +- .../query/DATE_DISTANCE/expected-letzter.csv | 4 +- .../expected-versichertenzeit.csv | 8 +-- .../expected-erster.csv | 8 +-- .../expected-letzter.csv | 8 +-- .../expected-versichertenzeit.csv | 16 +++--- .../expected-erster.csv | 40 +++++++------- .../expected-letzter.csv | 40 +++++++------- .../expected-versichertenzeit.csv | 36 ++++++------- .../query/DELETE_IMPORT_TESTS/expected.csv | 4 +- .../expected-zeitspanne.csv | 4 +- .../tests/query/LOGICAL/AND/expected.csv | 4 +- .../LOGICAL/AND_DATE_LOGICAL/expected.csv | 8 +-- .../LOGICAL/AND_DURATION_SUM/expected.csv | 8 +-- .../expected.csv | 12 ++--- .../query/LOGICAL/AND_NEGATION/expected.csv | 4 +- .../AND_NEGATION_DATE_LOGICAL/expected.csv | 4 +- .../expected.csv | 4 +- .../tests/query/LOGICAL/OR/expected.csv | 8 +-- .../tests/query/LOGICAL/OR_AND/expected.csv | 8 +-- .../LOGICAL/OR_DATE_LOGICAL/expected.csv | 10 ++-- .../query/MERGE/AND_DURATION_SUM/expected.csv | 8 +-- .../AND_EVENT_DATE_EXCLUDED/expected.csv | 8 +-- .../MERGE/OR_EVENT_DATE_EXCLUDED/expected.csv | 10 ++-- .../MULTIPLE_CONNECTORS_QUERY/expected.csv | 2 +- .../MULTIPLE_TABLES_ICD_QUERY2/expected.csv | 14 ++--- .../expected.csv | 4 +- .../expected.csv | 4 +- .../expected.csv | 2 +- .../expected.csv | 12 ++--- .../expected.csv | 14 ++--- .../resources/tests/query/NUMBER/expected.csv | 8 +-- .../tests/query/NUMBER_MISSING/expected.csv | 2 +- .../expected.csv | 4 +- .../expected2.csv | 8 +-- .../tests/query/NUMBER_NEGATION/expected.csv | 8 +-- .../expected.csv | 4 +- .../query/RESTART_TEST_DATA/expected.csv | 4 +- .../tests/query/REUSED_QUERY/expected.csv | 22 ++++---- .../query/REUSED_QUERY/query_results_1.csv | 22 ++++---- .../query/REUSED_QUERY/query_results_2.csv | 20 +++---- .../tests/query/SECONDARY_ID/expected.csv | 10 ++-- .../expected.csv | 4 +- .../query/SECONDARY_ID_EXCLUDED/expected.csv | 10 ++-- .../query/SECONDARY_ID_MIXED/expected.csv | 8 +-- .../SIMPLE_CQEXTERNAL_QUERY/expected.csv | 4 +- .../expected_start_end.csv | 4 +- .../expected.csv | 4 +- .../expected.csv | 2 +- .../expected.csv | 4 +- .../SIMPLE_TREECONCEPT_QUERY/expected.csv | 4 +- .../expected.csv | 4 +- .../SIMPLE_VIRTUAL_CONCEPT_QUERY/expected.csv | 4 +- .../expected.csv | 2 +- .../expected.csv | 14 ++--- .../SUM_EMPTY_DATE_CONCEPT_QUERY/expected.csv | 4 +- .../tests/query/TABLE_EXPORT/expected.csv | 14 ++--- .../expected.csv | 2 +- .../expected.csv | 6 +-- .../expected.csv | 2 +- .../expected.csv | 4 +- .../TEMPORAL_SAME_CONCEPT_QUERY/expected.csv | 4 +- .../query/UPDATE_CONCEPT_TESTS/expected.csv | 6 +-- .../query/VALIDITY_DATE_QUERY/expected.csv | 4 +- .../VIRTUAL_CONCEPT_REUSED_QUERY/expected.csv | 8 +-- .../query_results.csv | 8 +-- 139 files changed, 571 insertions(+), 567 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java index f14b283e7d..ae5f9f640a 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java @@ -53,7 +53,7 @@ public void readDates(String value, DateReader dateReader, CDateSet out) { DATE_RANGE { @Override public void readDates(String value, DateReader dateReader, CDateSet out) { - out.add(DateRangeParser.parseISORange(value, dateReader)); + out.add(dateReader.parseToCDateRange(value)); } }, DATE_SET { diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java index 89e419c9d6..26f3a25ab7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java @@ -1,7 +1,5 @@ package com.bakdata.conquery.models.config; -import java.io.IOException; -import java.text.SimpleDateFormat; import java.time.format.DateTimeFormatter; import java.util.*; @@ -12,12 +10,6 @@ import com.bakdata.conquery.util.DateReader; import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonValue; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonDeserializer; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.google.common.collect.Sets; import lombok.*; @@ -38,23 +30,23 @@ public class LocaleConfig { ); @NotEmpty - private List rangeStartEndSeperators = List.of("-","/"); + private Map rangeStartEndSeperators = Map.of(Locale.GERMAN, "-"); @NotNull @NotEmpty - private List dateSetLayouts = List.of( - new DateSetLayout("", ",", ""), - new DateSetLayout("{", ",", "}")); + private List listFormats = List.of( + new ListFormat("", ", ", ""), + new ListFormat("{", ", ", "}")); @Data @AllArgsConstructor - public static class DateSetLayout { + public static class ListFormat { @NonNull @Max(1) - String setBegin; - @NonNull @Min(1) @Max(1) - String rangeSep; + String start; + @NonNull @Min(1) + String separator; @NonNull @Max(1) - String setEnd; + String end; } @@ -65,27 +57,34 @@ public static class DateSetLayout { public DateReader getDateReader() { return new DateReader( Sets.union(dateParsingFormats, Set.copyOf(dateFormatMapping.values())), - rangeStartEndSeperators, - dateSetLayouts + new ArrayList<>(rangeStartEndSeperators.values()), + listFormats ); } /** * Finds the best formatter according to the locale and mapped date formatters. * If there is no perfect match, the locale is abstracted, see findClosestMatch. - * @param locale - * @return + */ + public String findDateRangeSeparator(Locale locale) { + final String closestMatch = findClosestMatch(locale, rangeStartEndSeperators); + return closestMatch != null ? closestMatch : "/"; + } + + /** + * Finds the best formatter according to the locale and mapped date formatters. + * If there is no perfect match, the locale is abstracted, see findClosestMatch. */ public DateTimeFormatter findDateTimeFormater(Locale locale) { - final Locale closestMatch = findClosestMatch(locale); - return closestMatch != null ? DateTimeFormatter.ofPattern(dateFormatMapping.get(closestMatch)) : DateTimeFormatter.ISO_DATE; + final String closestMatch = findClosestMatch(locale, dateFormatMapping); + return closestMatch != null ? DateTimeFormatter.ofPattern(closestMatch) : DateTimeFormatter.ISO_DATE; } /** * Adapted from {@link c10n.share.DefaultLocaleMapping} */ - public Locale findClosestMatch(Locale forLocale) { - Set fromSet = dateFormatMapping.keySet(); + private static T findClosestMatch(Locale forLocale, Map options) { + Set fromSet = options.keySet(); String variant = forLocale.getDisplayVariant(); String country = forLocale.getCountry(); String language = forLocale.getLanguage(); @@ -99,10 +98,9 @@ public Locale findClosestMatch(Locale forLocale) { if (null != language && !language.isEmpty()) { c.add(new Locale(language)); } - c.add(Locale.ROOT); for (Locale candidateLocale : c) { if (fromSet.contains(candidateLocale)) { - return candidateLocale; + return options.get(candidateLocale); } } return null; diff --git a/backend/src/main/java/com/bakdata/conquery/models/externalservice/ResultType.java b/backend/src/main/java/com/bakdata/conquery/models/externalservice/ResultType.java index 3326bfddf7..43c49f379d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/externalservice/ResultType.java +++ b/backend/src/main/java/com/bakdata/conquery/models/externalservice/ResultType.java @@ -27,6 +27,7 @@ import java.text.DateFormat; import java.text.SimpleDateFormat; import java.time.LocalDate; +import java.time.format.DateTimeFormatter; import java.util.List; import java.util.StringJoiner; @@ -206,7 +207,7 @@ public String print(PrintSettings cfg, @NonNull Object f) { if (cfg.isPrettyPrint()) { return cfg.getDateFormat().format(CDate.toLocalDate(((Number) f).intValue())); } - return CDate.toLocalDate(((Number) f).intValue()).toString(); + return print((Number) f, cfg.getDateFormat()); } @Override @@ -214,6 +215,10 @@ public Field getArrowFieldType(ResultInfo info, PrintSettings settings) { return NAMED_FIELD_DATE_DAY.apply(info.getUniqueName(settings)); } + public static String print(Number num, DateTimeFormatter formatter) { + return formatter.format(LocalDate.ofEpochDay(num.intValue())); + } + } @@ -236,7 +241,12 @@ public String print(PrintSettings cfg, @NonNull Object f) { if(list.size() != 2) { throw new IllegalStateException("Expected a list with 2 elements, one min, one max. The list was: " + list); } - return CDateRange.of((Integer) list.get(0), (Integer) list.get(1)).toString(); + final DateTimeFormatter dateFormat = cfg.getDateFormat(); + final Integer min = (Integer) list.get(0); + final Integer max = (Integer) list.get(1); + String minString = min == Integer.MIN_VALUE ? "-∞" : ResultType.DateT.print(min, dateFormat); + String maxString = max == Integer.MAX_VALUE ? "+∞" : ResultType.DateT.print(max, dateFormat); + return minString + cfg.getDateRangeSeparator() + maxString; } @Override @@ -316,10 +326,10 @@ public String print(PrintSettings cfg, @NonNull Object f) { throw new IllegalStateException(String.format("Expected a List got %s (Type: %s, as string: %s)", f, f.getClass().getName(), f)); } // Not sure if this escaping is enough - String listDelimEscape = cfg.getListElementEscaper() + cfg.getListElementDelimiter(); - StringJoiner joiner = new StringJoiner(cfg.getListElementDelimiter(), cfg.getListPrefix(), cfg.getListPostfix()); + String listDelimEscape = cfg.getListElementEscaper() + cfg.getListFormat().getSeparator(); + StringJoiner joiner = new StringJoiner(cfg.getListFormat().getSeparator(), cfg.getListFormat().getStart(),cfg.getListFormat().getEnd()); for(Object obj : (List) f) { - joiner.add(elementType.print(cfg,obj).replace(cfg.getListElementDelimiter(), listDelimEscape)); + joiner.add(elementType.print(cfg,obj).replace(cfg.getListFormat().getSeparator(), listDelimEscape)); } return joiner.toString(); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/PrintSettings.java b/backend/src/main/java/com/bakdata/conquery/models/query/PrintSettings.java index 141d9b858d..492b45c6b6 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/PrintSettings.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/PrintSettings.java @@ -1,15 +1,13 @@ package com.bakdata.conquery.models.query; import java.text.NumberFormat; -import java.text.SimpleDateFormat; import java.time.format.DateTimeFormatter; import java.util.Currency; -import java.util.List; import java.util.Locale; -import java.util.Map; import java.util.function.Function; import com.bakdata.conquery.models.config.ConqueryConfig; +import com.bakdata.conquery.models.config.LocaleConfig; import com.bakdata.conquery.models.identifiable.mapping.PrintIdMapper; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; import com.bakdata.conquery.models.query.resultinfo.SelectResultInfo; @@ -17,8 +15,6 @@ import lombok.Getter; import lombok.ToString; -import javax.validation.constraints.NotNull; - @Getter @ToString(onlyExplicitlyIncluded = true) public class PrintSettings { @@ -46,10 +42,11 @@ public class PrintSettings { private final Function columnNamer; - private final String listElementDelimiter = ", "; + private final String dateRangeSeparator; + + private final LocaleConfig.ListFormat listFormat; + private final String listElementEscaper = "\\"; - private final String listPrefix = "{"; - private final String listPostfix = "}"; private final PrintIdMapper idMapper; @@ -64,6 +61,9 @@ public PrintSettings(boolean prettyPrint, Locale locale, DatasetRegistry dataset this.integerFormat = NUMBER_FORMAT.apply(locale); this.decimalFormat = DECIMAL_FORMAT.apply(locale); + this.listFormat = config.getLocale().getListFormats().get(0); + this.dateRangeSeparator = config.getLocale().findDateRangeSeparator(locale); + this.dateFormat = config.getLocale().findDateTimeFormater(locale); } diff --git a/backend/src/main/java/com/bakdata/conquery/util/DateReader.java b/backend/src/main/java/com/bakdata/conquery/util/DateReader.java index 77cd49a025..1dd2cf4474 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/DateReader.java +++ b/backend/src/main/java/com/bakdata/conquery/util/DateReader.java @@ -10,20 +10,16 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; -import javax.validation.constraints.NotEmpty; - import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.config.LocaleConfig; import com.bakdata.conquery.models.exceptions.ParsingException; -import com.bakdata.conquery.models.preproc.parser.specific.DateRangeParser; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.base.Strings; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; -import lombok.NoArgsConstructor; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -44,7 +40,7 @@ public class DateReader { private List rangeStartEndSeperators; @JsonIgnore - private List dateSetLayouts; + private List dateSetLayouts; /** * Last successfully parsed date format. @@ -76,7 +72,7 @@ public class DateReader { .build(CacheLoader.from(this::tryParse)); @JsonCreator - public DateReader(Set dateParsingFormats, List rangeStartEndSeperators, List dateSetLayouts) { + public DateReader(Set dateParsingFormats, List rangeStartEndSeperators, List dateSetLayouts) { this.dateFormats = dateParsingFormats.stream().map(DateTimeFormatter::ofPattern).collect(Collectors.toSet()); this.rangeStartEndSeperators = rangeStartEndSeperators; this.dateSetLayouts = dateSetLayouts; @@ -159,8 +155,8 @@ public CDateSet parseToCDateSet(String value) { } } - for(LocaleConfig.DateSetLayout sep : dateSetLayouts) { - Pattern regexPattern = generateDateSetPattern(sep.getSetBegin(), sep.getRangeSep(), sep.getSetEnd()); + for(LocaleConfig.ListFormat sep : dateSetLayouts) { + Pattern regexPattern = generateDateSetPattern(sep.getStart(), sep.getSeparator(), sep.getEnd()); try { result = parseToCDateSet(value,regexPattern); } catch (ParsingException e) { @@ -188,7 +184,7 @@ public CDateSet parseToCDateSet(String value, Pattern pattern) { private static Pattern generateDateSetPattern(@NonNull String setBegin, @NonNull String rangeSep, @NonNull String setEnd) { assert(setBegin.length() < 2); - assert(rangeSep.length() == 1); + assert(rangeSep.length() >= 1); assert(setEnd.length() < 2); return Pattern.compile(String.format("(?:(?:%1$s)|(?:%2$s\\s*))([^%1$s%2$s%3$s]+)(?:%3$s)?", diff --git a/backend/src/test/resources/tests/aggregator/COUNT_AGGREGATOR/expected.csv b/backend/src/test/resources/tests/aggregator/COUNT_AGGREGATOR/expected.csv index cd113058fd..2980ada756 100644 --- a/backend/src/test/resources/tests/aggregator/COUNT_AGGREGATOR/expected.csv +++ b/backend/src/test/resources/tests/aggregator/COUNT_AGGREGATOR/expected.csv @@ -1,5 +1,5 @@ result,dates,concept - select -1,{2012-01-01/2012-01-02},2 -2,{2010-07-15/2010-07-15}, -3,{2013-11-10/2013-11-10},1 -4,{2012-11-11/2012-11-11},1 \ No newline at end of file +1,2012-01-01/2012-01-02,2 +2,2010-07-15/2010-07-15, +3,2013-11-10/2013-11-10,1 +4,2012-11-11/2012-11-11,1 \ No newline at end of file diff --git a/backend/src/test/resources/tests/aggregator/COUNT_DISTINCT_AGGREGATOR/expected.csv b/backend/src/test/resources/tests/aggregator/COUNT_DISTINCT_AGGREGATOR/expected.csv index f1637cd722..66588601bb 100644 --- a/backend/src/test/resources/tests/aggregator/COUNT_DISTINCT_AGGREGATOR/expected.csv +++ b/backend/src/test/resources/tests/aggregator/COUNT_DISTINCT_AGGREGATOR/expected.csv @@ -1,5 +1,5 @@ result,dates,concept - select -1,{2012-01-01/2012-01-02},1 -2,{2010-07-15/2010-07-15}, -3,{2013-11-10/2013-11-10},1 -4,{2012-11-11/2012-11-12},2 \ No newline at end of file +1,2012-01-01/2012-01-02,1 +2,2010-07-15/2010-07-15, +3,2013-11-10/2013-11-10,1 +4,2012-11-11/2012-11-12,2 \ No newline at end of file diff --git a/backend/src/test/resources/tests/aggregator/COUNT_QUARTERS_AGGREGATOR/expected.csv b/backend/src/test/resources/tests/aggregator/COUNT_QUARTERS_AGGREGATOR/expected.csv index c5b411da06..ff708dd179 100644 --- a/backend/src/test/resources/tests/aggregator/COUNT_QUARTERS_AGGREGATOR/expected.csv +++ b/backend/src/test/resources/tests/aggregator/COUNT_QUARTERS_AGGREGATOR/expected.csv @@ -1,6 +1,6 @@ result,dates,concept - select -1,{2015-01-01/2016-12-31},1 -2,{2015-01-01/2016-12-31},2 -3,{2015-01-01/2016-12-31},4 -4,{2015-01-01/2016-12-31},3 -5,{2015-01-01/2016-12-31}, \ No newline at end of file +1,2015-01-01/2016-12-31,1 +2,2015-01-01/2016-12-31,2 +3,2015-01-01/2016-12-31,4 +4,2015-01-01/2016-12-31,3 +5,2015-01-01/2016-12-31, \ No newline at end of file diff --git a/backend/src/test/resources/tests/aggregator/DATE_DISTANCE_AGGREGATOR/expected.csv b/backend/src/test/resources/tests/aggregator/DATE_DISTANCE_AGGREGATOR/expected.csv index 692e212a89..1476498fed 100644 --- a/backend/src/test/resources/tests/aggregator/DATE_DISTANCE_AGGREGATOR/expected.csv +++ b/backend/src/test/resources/tests/aggregator/DATE_DISTANCE_AGGREGATOR/expected.csv @@ -1,11 +1,11 @@ result,dates,concept - select -1,{2010-01-01/2010-03-31},0 -2,{2010-01-01/2010-03-31},12 -3,{2010-01-01/2010-03-31},12 -4,{2010-01-01/2010-03-31}, -5,{2010-01-01/2010-03-31},0 -6,{2010-01-01/2010-03-31},12 -7,{2010-01-01/2010-03-31},0 -8,{2010-01-01/2010-03-31},12 -9,{2010-01-01/2010-03-31},11 -10,{2010-01-01/2010-03-31},-1 \ No newline at end of file +1,2010-01-01/2010-03-31,0 +2,2010-01-01/2010-03-31,12 +3,2010-01-01/2010-03-31,12 +4,2010-01-01/2010-03-31, +5,2010-01-01/2010-03-31,0 +6,2010-01-01/2010-03-31,12 +7,2010-01-01/2010-03-31,0 +8,2010-01-01/2010-03-31,12 +9,2010-01-01/2010-03-31,11 +10,2010-01-01/2010-03-31,-1 \ No newline at end of file diff --git a/backend/src/test/resources/tests/aggregator/DURATION_SUM_AGGREGATOR/expected.csv b/backend/src/test/resources/tests/aggregator/DURATION_SUM_AGGREGATOR/expected.csv index 57c38bb809..503a0ed6f6 100644 --- a/backend/src/test/resources/tests/aggregator/DURATION_SUM_AGGREGATOR/expected.csv +++ b/backend/src/test/resources/tests/aggregator/DURATION_SUM_AGGREGATOR/expected.csv @@ -1,5 +1,5 @@ result,dates,concept - select -1,{2010-01-01/2010-01-31},31 -3,{2013-08-10/2013-08-11},2 -4,{2010-06-15/2010-06-20},6 -5,{2011-03-02/2011-03-13},12 \ No newline at end of file +1,2010-01-01/2010-01-31,31 +3,2013-08-10/2013-08-11,2 +4,2010-06-15/2010-06-20,6 +5,2011-03-02/2011-03-13,12 \ No newline at end of file diff --git a/backend/src/test/resources/tests/aggregator/EVENT_DATE_AGGREGATOR/expected_no_restriction.csv b/backend/src/test/resources/tests/aggregator/EVENT_DATE_AGGREGATOR/expected_no_restriction.csv index e7a4b365ee..e2285bd334 100644 --- a/backend/src/test/resources/tests/aggregator/EVENT_DATE_AGGREGATOR/expected_no_restriction.csv +++ b/backend/src/test/resources/tests/aggregator/EVENT_DATE_AGGREGATOR/expected_no_restriction.csv @@ -1,11 +1,11 @@ result,dates,concept - dates,concept - dates_1 -1,{2010-01-01/2010-03-31},{2010-01-01/2010-03-31},{2010-01-01/2010-03-31} -10,{2010-01-01/2010-03-31},{2010-01-01/2010-03-31},{2010-01-01/2010-03-31} -2,{2010-01-01/2010-03-31},{2010-01-01/2010-03-31},{2010-01-01/2010-03-31} -3,{2010-01-01/2010-03-31},{2010-01-01/2010-03-31},{2010-01-01/2010-03-31} -4,{2010-01-01/2010-03-31},{2010-01-01/2010-03-31},{2010-01-01/2010-03-31} -5,{2010-01-01/2010-03-31},{2010-01-01/2010-03-31},{2010-01-01/2010-03-31} -6,{2010-01-01/2010-03-31},{2010-01-01/2010-03-31},{2010-01-01/2010-03-31} -7,{2010-01-01/2010-03-31},{2010-01-01/2010-03-31},{2010-01-01/2010-03-31} -8,{2010-01-01/2010-03-31},{2010-01-01/2010-03-31},{2010-01-01/2010-03-31} -9,{2010-01-01/2010-03-31},{2010-01-01/2010-03-31},{2010-01-01/2010-03-31} \ No newline at end of file +1,2010-01-01/2010-03-31,2010-01-01/2010-03-31,2010-01-01/2010-03-31 +10,2010-01-01/2010-03-31,2010-01-01/2010-03-31,2010-01-01/2010-03-31 +2,2010-01-01/2010-03-31,2010-01-01/2010-03-31,2010-01-01/2010-03-31 +3,2010-01-01/2010-03-31,2010-01-01/2010-03-31,2010-01-01/2010-03-31 +4,2010-01-01/2010-03-31,2010-01-01/2010-03-31,2010-01-01/2010-03-31 +5,2010-01-01/2010-03-31,2010-01-01/2010-03-31,2010-01-01/2010-03-31 +6,2010-01-01/2010-03-31,2010-01-01/2010-03-31,2010-01-01/2010-03-31 +7,2010-01-01/2010-03-31,2010-01-01/2010-03-31,2010-01-01/2010-03-31 +8,2010-01-01/2010-03-31,2010-01-01/2010-03-31,2010-01-01/2010-03-31 +9,2010-01-01/2010-03-31,2010-01-01/2010-03-31,2010-01-01/2010-03-31 \ No newline at end of file diff --git a/backend/src/test/resources/tests/aggregator/EVENT_DATE_AGGREGATOR/expected_restriction.csv b/backend/src/test/resources/tests/aggregator/EVENT_DATE_AGGREGATOR/expected_restriction.csv index ca73bdee9b..e0c06c5bd6 100644 --- a/backend/src/test/resources/tests/aggregator/EVENT_DATE_AGGREGATOR/expected_restriction.csv +++ b/backend/src/test/resources/tests/aggregator/EVENT_DATE_AGGREGATOR/expected_restriction.csv @@ -1,11 +1,11 @@ result,dates,concept - dates,concept - dates_1 -1,{2010-02-01/2010-02-28},{2010-02-01/2010-02-28},{2010-02-01/2010-02-28} -10,{2010-02-01/2010-02-28},{2010-02-01/2010-02-28},{2010-02-01/2010-02-28} -2,{2010-02-01/2010-02-28},{2010-02-01/2010-02-28},{2010-02-01/2010-02-28} -3,{2010-02-01/2010-02-28},{2010-02-01/2010-02-28},{2010-02-01/2010-02-28} -4,{2010-02-01/2010-02-28},{2010-02-01/2010-02-28},{2010-02-01/2010-02-28} -5,{2010-02-01/2010-02-28},{2010-02-01/2010-02-28},{2010-02-01/2010-02-28} -6,{2010-02-01/2010-02-28},{2010-02-01/2010-02-28},{2010-02-01/2010-02-28} -7,{2010-02-01/2010-02-28},{2010-02-01/2010-02-28},{2010-02-01/2010-02-28} -8,{2010-02-01/2010-02-28},{2010-02-01/2010-02-28},{2010-02-01/2010-02-28} -9,{2010-02-01/2010-02-28},{2010-02-01/2010-02-28},{2010-02-01/2010-02-28} \ No newline at end of file +1,2010-02-01/2010-02-28,2010-02-01/2010-02-28,2010-02-01/2010-02-28 +10,2010-02-01/2010-02-28,2010-02-01/2010-02-28,2010-02-01/2010-02-28 +2,2010-02-01/2010-02-28,2010-02-01/2010-02-28,2010-02-01/2010-02-28 +3,2010-02-01/2010-02-28,2010-02-01/2010-02-28,2010-02-01/2010-02-28 +4,2010-02-01/2010-02-28,2010-02-01/2010-02-28,2010-02-01/2010-02-28 +5,2010-02-01/2010-02-28,2010-02-01/2010-02-28,2010-02-01/2010-02-28 +6,2010-02-01/2010-02-28,2010-02-01/2010-02-28,2010-02-01/2010-02-28 +7,2010-02-01/2010-02-28,2010-02-01/2010-02-28,2010-02-01/2010-02-28 +8,2010-02-01/2010-02-28,2010-02-01/2010-02-28,2010-02-01/2010-02-28 +9,2010-02-01/2010-02-28,2010-02-01/2010-02-28,2010-02-01/2010-02-28 \ No newline at end of file diff --git a/backend/src/test/resources/tests/aggregator/EXISTS_AGGREGATOR/expected.csv b/backend/src/test/resources/tests/aggregator/EXISTS_AGGREGATOR/expected.csv index 962b85f37b..1f78e521d6 100644 --- a/backend/src/test/resources/tests/aggregator/EXISTS_AGGREGATOR/expected.csv +++ b/backend/src/test/resources/tests/aggregator/EXISTS_AGGREGATOR/expected.csv @@ -1,4 +1,4 @@ result,dates,concept - exists -1,{2014-06-30/2015-06-30},1 -2,{2014-06-30/2015-06-30},1 -5,{2014-06-30/2015-06-30},1 \ No newline at end of file +1,2014-06-30/2015-06-30,1 +2,2014-06-30/2015-06-30,1 +5,2014-06-30/2015-06-30,1 \ No newline at end of file diff --git a/backend/src/test/resources/tests/aggregator/EXISTS_AGGREGATOR_AGGREGATED/expected.csv b/backend/src/test/resources/tests/aggregator/EXISTS_AGGREGATOR_AGGREGATED/expected.csv index 676fbd1f39..1afcf20d5f 100644 --- a/backend/src/test/resources/tests/aggregator/EXISTS_AGGREGATOR_AGGREGATED/expected.csv +++ b/backend/src/test/resources/tests/aggregator/EXISTS_AGGREGATOR_AGGREGATED/expected.csv @@ -1,4 +1,4 @@ result,dates,concept - 1 - exists,concept - 2 - exists,AND,concept - 1 - exists_1,concept - 2 - exists_1,OR -1,{2020-01-01/2020-01-01},1,0,0,1,0,1 -2,{2020-01-01/2020-01-01},0,1,0,0,1,1 -3,{2020-01-01/2020-01-01},1,1,1,1,1,1 \ No newline at end of file +1,2020-01-01/2020-01-01,1,0,0,1,0,1 +2,2020-01-01/2020-01-01,0,1,0,0,1,1 +3,2020-01-01/2020-01-01,1,1,1,1,1,1 \ No newline at end of file diff --git a/backend/src/test/resources/tests/aggregator/EXISTS_AGGREGATOR_OR/expected.csv b/backend/src/test/resources/tests/aggregator/EXISTS_AGGREGATOR_OR/expected.csv index d60a5ac7d7..2da3ef7171 100644 --- a/backend/src/test/resources/tests/aggregator/EXISTS_AGGREGATOR_OR/expected.csv +++ b/backend/src/test/resources/tests/aggregator/EXISTS_AGGREGATOR_OR/expected.csv @@ -1,4 +1,4 @@ result,dates,concept - exists,concept - exists_1 -1,{2014-06-30/2015-06-30},1,0 -2,{2014-06-30/2015-06-30},1,1 -5,{2014-06-30/2015-06-30},1,0 \ No newline at end of file +1,2014-06-30/2015-06-30,1,0 +2,2014-06-30/2015-06-30,1,1 +5,2014-06-30/2015-06-30,1,0 \ No newline at end of file diff --git a/backend/src/test/resources/tests/aggregator/FIRST_AGGREGATOR/expected.csv b/backend/src/test/resources/tests/aggregator/FIRST_AGGREGATOR/expected.csv index 08e60abefd..c19c558617 100644 --- a/backend/src/test/resources/tests/aggregator/FIRST_AGGREGATOR/expected.csv +++ b/backend/src/test/resources/tests/aggregator/FIRST_AGGREGATOR/expected.csv @@ -1,6 +1,6 @@ result,dates,concept - select -1,{2012-01-01/2012-01-02},f -2,{2010-07-15/2010-07-15}, -3,{2012-01-01/2012-01-02},f -5,{2012-01-01/2012-01-01},m -6,{2012-01-01/2012-01-01},m \ No newline at end of file +1,2012-01-01/2012-01-02,f +2,2010-07-15/2010-07-15, +3,2012-01-01/2012-01-02,f +5,2012-01-01/2012-01-01,m +6,2012-01-01/2012-01-01,m \ No newline at end of file diff --git a/backend/src/test/resources/tests/aggregator/LAST_AGGREGATOR/expected.csv b/backend/src/test/resources/tests/aggregator/LAST_AGGREGATOR/expected.csv index 5f4675601c..65debd8e64 100644 --- a/backend/src/test/resources/tests/aggregator/LAST_AGGREGATOR/expected.csv +++ b/backend/src/test/resources/tests/aggregator/LAST_AGGREGATOR/expected.csv @@ -1,6 +1,6 @@ result,dates,concept - select -1,{2012-01-01/2012-01-02},m -2,{2010-07-15/2010-07-15}, -3,{2012-01-01/2012-01-02},f -5,{2012-01-01/2012-01-01},m -6,{2012-01-01/2012-01-01},m \ No newline at end of file +1,2012-01-01/2012-01-02,m +2,2010-07-15/2010-07-15, +3,2012-01-01/2012-01-02,f +5,2012-01-01/2012-01-01,m +6,2012-01-01/2012-01-01,m \ No newline at end of file diff --git a/backend/src/test/resources/tests/aggregator/MULTI_SELECT_AGGREGATOR/expected.csv b/backend/src/test/resources/tests/aggregator/MULTI_SELECT_AGGREGATOR/expected.csv index 33a9b7b834..c08081f3a7 100644 --- a/backend/src/test/resources/tests/aggregator/MULTI_SELECT_AGGREGATOR/expected.csv +++ b/backend/src/test/resources/tests/aggregator/MULTI_SELECT_AGGREGATOR/expected.csv @@ -1,7 +1,7 @@ result,dates,concept - select -2,{2010-07-15/2010-07-15}, -3,{2013-11-10/2013-11-10},{f=1} -6,{2012-11-11/2012-11-11},"{f=1, m=1}" -1,{2012-01-01/2012-01-02},{f=2} -4,{2012-11-11/2012-11-11},{m=1} -5,{2012-11-11/2012-11-11},{f=1} \ No newline at end of file +2,2010-07-15/2010-07-15, +3,2013-11-10/2013-11-10,f=1 +6,2012-11-11/2012-11-11,"f=1, m=1" +1,2012-01-01/2012-01-02,f=2 +4,2012-11-11/2012-11-11,m=1 +5,2012-11-11/2012-11-11,f=1 \ No newline at end of file diff --git a/backend/src/test/resources/tests/aggregator/PREFIX_AGGREGATOR/expected.csv b/backend/src/test/resources/tests/aggregator/PREFIX_AGGREGATOR/expected.csv index 0c3194be1d..d7a66069eb 100644 --- a/backend/src/test/resources/tests/aggregator/PREFIX_AGGREGATOR/expected.csv +++ b/backend/src/test/resources/tests/aggregator/PREFIX_AGGREGATOR/expected.csv @@ -1,5 +1,5 @@ result,dates,concept - select -1,{2012-01-01/2012-01-02},"[fa, fb]" -2,{2010-07-15/2010-07-15}, -3,{2013-11-10/2013-11-10},[f] -4,{2012-11-11/2012-11-11}, \ No newline at end of file +1,2012-01-01/2012-01-02,"[fa, fb]" +2,2010-07-15/2010-07-15, +3,2013-11-10/2013-11-10,[f] +4,2012-11-11/2012-11-11, \ No newline at end of file diff --git a/backend/src/test/resources/tests/aggregator/QUARTER_AGGREGATOR/expected.csv b/backend/src/test/resources/tests/aggregator/QUARTER_AGGREGATOR/expected.csv index cecf2b3864..47b2f2a65a 100644 --- a/backend/src/test/resources/tests/aggregator/QUARTER_AGGREGATOR/expected.csv +++ b/backend/src/test/resources/tests/aggregator/QUARTER_AGGREGATOR/expected.csv @@ -1,3 +1,3 @@ result,dates,concept - select -1,{2015-01-17/2015-01-17},2015-Q1 -2,"{2015-03-31/2015-03-31, 2015-09-01/2015-09-01}",2015-Q3 \ No newline at end of file +1,2015-01-17/2015-01-17,2015-Q1 +2,"2015-03-31/2015-03-31, 2015-09-01/2015-09-01",2015-Q3 \ No newline at end of file diff --git a/backend/src/test/resources/tests/aggregator/RANDOM_AGGREGATOR/expected.csv b/backend/src/test/resources/tests/aggregator/RANDOM_AGGREGATOR/expected.csv index 8c1d4d01f1..45b4350abc 100644 --- a/backend/src/test/resources/tests/aggregator/RANDOM_AGGREGATOR/expected.csv +++ b/backend/src/test/resources/tests/aggregator/RANDOM_AGGREGATOR/expected.csv @@ -1,5 +1,5 @@ result,dates,concept - select -1,{2012-01-01/2012-01-01},f -2,{2010-07-15/2010-07-15}, -5,{2012-01-01/2012-01-01},m -6,{2012-01-01/2012-01-01},m \ No newline at end of file +1,2012-01-01/2012-01-01,f +2,2010-07-15/2010-07-15, +5,2012-01-01/2012-01-01,m +6,2012-01-01/2012-01-01,m \ No newline at end of file diff --git a/backend/src/test/resources/tests/aggregator/SELECT_AGGREGATOR/expected.csv b/backend/src/test/resources/tests/aggregator/SELECT_AGGREGATOR/expected.csv index fecdbf81f1..2631695c79 100644 --- a/backend/src/test/resources/tests/aggregator/SELECT_AGGREGATOR/expected.csv +++ b/backend/src/test/resources/tests/aggregator/SELECT_AGGREGATOR/expected.csv @@ -1,5 +1,5 @@ result,dates,concept - select -1,{2012-01-01/2012-01-02},2 -2,{2010-07-15/2010-07-15}, -3,{2013-11-10/2013-11-10},1 -4,{2012-11-11/2012-11-11}, \ No newline at end of file +1,2012-01-01/2012-01-02,2 +2,2010-07-15/2010-07-15, +3,2013-11-10/2013-11-10,1 +4,2012-11-11/2012-11-11, \ No newline at end of file diff --git a/backend/src/test/resources/tests/aggregator/SUM_AGGREGATOR/expected.csv b/backend/src/test/resources/tests/aggregator/SUM_AGGREGATOR/expected.csv index 7a2b15acb1..531c3fb0a1 100644 --- a/backend/src/test/resources/tests/aggregator/SUM_AGGREGATOR/expected.csv +++ b/backend/src/test/resources/tests/aggregator/SUM_AGGREGATOR/expected.csv @@ -1,5 +1,5 @@ result,dates,concept - select -1,{2012-01-01/2012-01-02},2 -2,{2010-07-15/2010-07-15}, -3,{2013-11-10/2013-11-10},1 -4,{2012-11-11/2012-11-11},-1 \ No newline at end of file +1,2012-01-01/2012-01-02,2 +2,2010-07-15/2010-07-15, +3,2013-11-10/2013-11-10,1 +4,2012-11-11/2012-11-11,-1 \ No newline at end of file diff --git a/backend/src/test/resources/tests/aggregator/SUM_DIFF_AGGREGATOR/expected.csv b/backend/src/test/resources/tests/aggregator/SUM_DIFF_AGGREGATOR/expected.csv index 87440f8dda..ce044e4c7b 100644 --- a/backend/src/test/resources/tests/aggregator/SUM_DIFF_AGGREGATOR/expected.csv +++ b/backend/src/test/resources/tests/aggregator/SUM_DIFF_AGGREGATOR/expected.csv @@ -1,5 +1,5 @@ result,dates,concept - select -1,{2012-01-01/2012-01-02},2 -2,{2010-07-15/2010-07-15}, -3,{2013-11-10/2013-11-10},0 -4,{2012-11-11/2012-11-11},-2 \ No newline at end of file +1,2012-01-01/2012-01-02,2 +2,2010-07-15/2010-07-15, +3,2013-11-10/2013-11-10,0 +4,2012-11-11/2012-11-11,-2 \ No newline at end of file diff --git a/backend/src/test/resources/tests/aggregator/VALUES_AGGREGATOR/expected.csv b/backend/src/test/resources/tests/aggregator/VALUES_AGGREGATOR/expected.csv index 43e1c3f939..ebcca298cf 100644 --- a/backend/src/test/resources/tests/aggregator/VALUES_AGGREGATOR/expected.csv +++ b/backend/src/test/resources/tests/aggregator/VALUES_AGGREGATOR/expected.csv @@ -1,6 +1,6 @@ result,dates,concept - select -1,{2012-01-01/2012-01-02},{f} -2,{2010-07-15/2010-07-15}, -3,{2012-01-01/2012-01-02},"{f, m}" -4,{2012-01-01/2012-01-04},"{f, m}" -5,{2012-01-01/2012-01-04},"{m , f, f , m}" \ No newline at end of file +1,2012-01-01/2012-01-02,f +2,2010-07-15/2010-07-15, +3,2012-01-01/2012-01-02,"f, m" +4,2012-01-01/2012-01-04,"f, m" +5,2012-01-01/2012-01-04,"m , f, f , m" \ No newline at end of file diff --git a/backend/src/test/resources/tests/filter/BIG_MULTI_SELECT/expected_BIG_MULTI_SELECT_Filter.csv b/backend/src/test/resources/tests/filter/BIG_MULTI_SELECT/expected_BIG_MULTI_SELECT_Filter.csv index 528a9786b2..6ae7d52095 100644 --- a/backend/src/test/resources/tests/filter/BIG_MULTI_SELECT/expected_BIG_MULTI_SELECT_Filter.csv +++ b/backend/src/test/resources/tests/filter/BIG_MULTI_SELECT/expected_BIG_MULTI_SELECT_Filter.csv @@ -1,3 +1,3 @@ result,dates -1,{2012-01-01/2012-01-01} -2,{2010-07-15/2010-07-15} \ No newline at end of file +1,2012-01-01/2012-01-01 +2,2010-07-15/2010-07-15 \ No newline at end of file diff --git a/backend/src/test/resources/tests/filter/BIG_MULTI_SELECT_2VALUES/expected_BIG_MULTI_SELECT_Filter.csv b/backend/src/test/resources/tests/filter/BIG_MULTI_SELECT_2VALUES/expected_BIG_MULTI_SELECT_Filter.csv index dad81fdbf2..88a34c57b1 100644 --- a/backend/src/test/resources/tests/filter/BIG_MULTI_SELECT_2VALUES/expected_BIG_MULTI_SELECT_Filter.csv +++ b/backend/src/test/resources/tests/filter/BIG_MULTI_SELECT_2VALUES/expected_BIG_MULTI_SELECT_Filter.csv @@ -1,4 +1,4 @@ result,dates -1,"{2012-01-01/2012-01-01, 2012-02-01/2012-02-01}" -2,{2010-07-15/2010-07-15} -3,{2013-11-10/2013-11-10} \ No newline at end of file +1,"2012-01-01/2012-01-01, 2012-02-01/2012-02-01" +2,2010-07-15/2010-07-15 +3,2013-11-10/2013-11-10 \ No newline at end of file diff --git a/backend/src/test/resources/tests/filter/BIG_MULTI_SELECT_EMPTY_VALUES/expected_BIG_MULTI_SELECT_Filter.csv b/backend/src/test/resources/tests/filter/BIG_MULTI_SELECT_EMPTY_VALUES/expected_BIG_MULTI_SELECT_Filter.csv index 528a9786b2..6ae7d52095 100644 --- a/backend/src/test/resources/tests/filter/BIG_MULTI_SELECT_EMPTY_VALUES/expected_BIG_MULTI_SELECT_Filter.csv +++ b/backend/src/test/resources/tests/filter/BIG_MULTI_SELECT_EMPTY_VALUES/expected_BIG_MULTI_SELECT_Filter.csv @@ -1,3 +1,3 @@ result,dates -1,{2012-01-01/2012-01-01} -2,{2010-07-15/2010-07-15} \ No newline at end of file +1,2012-01-01/2012-01-01 +2,2010-07-15/2010-07-15 \ No newline at end of file diff --git a/backend/src/test/resources/tests/filter/BIG_MULTI_SELECT_SPEZIAL_CHARACTER/expected_BIG_MULTI_SELECT_Filter.csv b/backend/src/test/resources/tests/filter/BIG_MULTI_SELECT_SPEZIAL_CHARACTER/expected_BIG_MULTI_SELECT_Filter.csv index 3265d716b0..ec2f5318ca 100644 --- a/backend/src/test/resources/tests/filter/BIG_MULTI_SELECT_SPEZIAL_CHARACTER/expected_BIG_MULTI_SELECT_Filter.csv +++ b/backend/src/test/resources/tests/filter/BIG_MULTI_SELECT_SPEZIAL_CHARACTER/expected_BIG_MULTI_SELECT_Filter.csv @@ -1,2 +1,2 @@ result,dates -2,{2010-07-15/2010-07-15} \ No newline at end of file +2,2010-07-15/2010-07-15 \ No newline at end of file diff --git a/backend/src/test/resources/tests/filter/BIG_MULTI_SELECT_SPEZIAL_CHARACTER2/expected_BIG_MULTI_SELECT_Filter.csv b/backend/src/test/resources/tests/filter/BIG_MULTI_SELECT_SPEZIAL_CHARACTER2/expected_BIG_MULTI_SELECT_Filter.csv index 3265d716b0..ec2f5318ca 100644 --- a/backend/src/test/resources/tests/filter/BIG_MULTI_SELECT_SPEZIAL_CHARACTER2/expected_BIG_MULTI_SELECT_Filter.csv +++ b/backend/src/test/resources/tests/filter/BIG_MULTI_SELECT_SPEZIAL_CHARACTER2/expected_BIG_MULTI_SELECT_Filter.csv @@ -1,2 +1,2 @@ result,dates -2,{2010-07-15/2010-07-15} \ No newline at end of file +2,2010-07-15/2010-07-15 \ No newline at end of file diff --git a/backend/src/test/resources/tests/filter/COUNT/expected_COUNT_Filter.csv b/backend/src/test/resources/tests/filter/COUNT/expected_COUNT_Filter.csv index 944bad7c3b..ccbef54e0e 100644 --- a/backend/src/test/resources/tests/filter/COUNT/expected_COUNT_Filter.csv +++ b/backend/src/test/resources/tests/filter/COUNT/expected_COUNT_Filter.csv @@ -1,3 +1,3 @@ result,dates -2,{2010-01-01/2010-01-31} -4,{2010-01-01/2010-01-31} \ No newline at end of file +2,2010-01-01/2010-01-31 +4,2010-01-01/2010-01-31 \ No newline at end of file diff --git a/backend/src/test/resources/tests/filter/COUNT_DISTINCT/expected_COUNT_Filter.csv b/backend/src/test/resources/tests/filter/COUNT_DISTINCT/expected_COUNT_Filter.csv index fc0d69adc1..ce235b7c15 100644 --- a/backend/src/test/resources/tests/filter/COUNT_DISTINCT/expected_COUNT_Filter.csv +++ b/backend/src/test/resources/tests/filter/COUNT_DISTINCT/expected_COUNT_Filter.csv @@ -1,3 +1,3 @@ result,dates -2,{2010-01-01/2010-01-31} -3,{2010-01-01/2010-01-31} \ No newline at end of file +2,2010-01-01/2010-01-31 +3,2010-01-01/2010-01-31 \ No newline at end of file diff --git a/backend/src/test/resources/tests/filter/COUNT_DISTINCT_MULTI/expected_COUNT_Filter.csv b/backend/src/test/resources/tests/filter/COUNT_DISTINCT_MULTI/expected_COUNT_Filter.csv index fc0d69adc1..ce235b7c15 100644 --- a/backend/src/test/resources/tests/filter/COUNT_DISTINCT_MULTI/expected_COUNT_Filter.csv +++ b/backend/src/test/resources/tests/filter/COUNT_DISTINCT_MULTI/expected_COUNT_Filter.csv @@ -1,3 +1,3 @@ result,dates -2,{2010-01-01/2010-01-31} -3,{2010-01-01/2010-01-31} \ No newline at end of file +2,2010-01-01/2010-01-31 +3,2010-01-01/2010-01-31 \ No newline at end of file diff --git a/backend/src/test/resources/tests/filter/COUNT_QUARTERS/expected_COUNT_QUARTERS_Filter.csv b/backend/src/test/resources/tests/filter/COUNT_QUARTERS/expected_COUNT_QUARTERS_Filter.csv index e681a27fe8..748d03f2be 100644 --- a/backend/src/test/resources/tests/filter/COUNT_QUARTERS/expected_COUNT_QUARTERS_Filter.csv +++ b/backend/src/test/resources/tests/filter/COUNT_QUARTERS/expected_COUNT_QUARTERS_Filter.csv @@ -1,7 +1,7 @@ result,dates -2,{2015-01-01/2016-12-31} -4,{2015-01-01/2016-12-31} -6,{2015-01-01/2016-12-31} -7,{2015-01-01/2016-12-31} -10,{2015-01-01/2016-12-31} -11,{2015-01-01/2016-12-31} \ No newline at end of file +2,2015-01-01/2016-12-31 +4,2015-01-01/2016-12-31 +6,2015-01-01/2016-12-31 +7,2015-01-01/2016-12-31 +10,2015-01-01/2016-12-31 +11,2015-01-01/2016-12-31 \ No newline at end of file diff --git a/backend/src/test/resources/tests/filter/COUNT_QUARTERS_RANGE/expected_COUNT_QUARTERS_Filter.csv b/backend/src/test/resources/tests/filter/COUNT_QUARTERS_RANGE/expected_COUNT_QUARTERS_Filter.csv index aad244b5d3..d3bfa2943c 100644 --- a/backend/src/test/resources/tests/filter/COUNT_QUARTERS_RANGE/expected_COUNT_QUARTERS_Filter.csv +++ b/backend/src/test/resources/tests/filter/COUNT_QUARTERS_RANGE/expected_COUNT_QUARTERS_Filter.csv @@ -1,3 +1,3 @@ result,dates -3,{2015-01-01/2015-01-01} -4,{2015-01-01/2015-01-01} \ No newline at end of file +3,2015-01-01/2015-01-01 +4,2015-01-01/2015-01-01 \ No newline at end of file diff --git a/backend/src/test/resources/tests/filter/COUNTfalse/expected_COUNT_Filter.csv b/backend/src/test/resources/tests/filter/COUNTfalse/expected_COUNT_Filter.csv index 944bad7c3b..ccbef54e0e 100644 --- a/backend/src/test/resources/tests/filter/COUNTfalse/expected_COUNT_Filter.csv +++ b/backend/src/test/resources/tests/filter/COUNTfalse/expected_COUNT_Filter.csv @@ -1,3 +1,3 @@ result,dates -2,{2010-01-01/2010-01-31} -4,{2010-01-01/2010-01-31} \ No newline at end of file +2,2010-01-01/2010-01-31 +4,2010-01-01/2010-01-31 \ No newline at end of file diff --git a/backend/src/test/resources/tests/filter/DATE_DISTANCE/expected_NUMBER_Filter.csv b/backend/src/test/resources/tests/filter/DATE_DISTANCE/expected_NUMBER_Filter.csv index 38b789e735..1790401bf1 100644 --- a/backend/src/test/resources/tests/filter/DATE_DISTANCE/expected_NUMBER_Filter.csv +++ b/backend/src/test/resources/tests/filter/DATE_DISTANCE/expected_NUMBER_Filter.csv @@ -1,3 +1,3 @@ result,dates -1,{2000-01-31/2000-02-01} -2,{2000-01-31/2000-01-31} \ No newline at end of file +1,2000-01-31/2000-02-01 +2,2000-01-31/2000-01-31 \ No newline at end of file diff --git a/backend/src/test/resources/tests/filter/DIFFSUM_INTEGER/expected_SUM_Filter.csv b/backend/src/test/resources/tests/filter/DIFFSUM_INTEGER/expected_SUM_Filter.csv index 26c5af160f..92e95e61ff 100644 --- a/backend/src/test/resources/tests/filter/DIFFSUM_INTEGER/expected_SUM_Filter.csv +++ b/backend/src/test/resources/tests/filter/DIFFSUM_INTEGER/expected_SUM_Filter.csv @@ -1,8 +1,8 @@ result,dates -2,{2017-05-06/2017-05-06} -3,{2012-05-29/2012-05-29} -4,{2010-08-14/2010-08-14} -7,{2016-06-23/2016-06-23} -9,{2018-03-22/2018-03-24} -10,{2011-02-07/2011-02-08} -11,{2013-03-09/2013-03-11} \ No newline at end of file +2,2017-05-06/2017-05-06 +3,2012-05-29/2012-05-29 +4,2010-08-14/2010-08-14 +7,2016-06-23/2016-06-23 +9,2018-03-22/2018-03-24 +10,2011-02-07/2011-02-08 +11,2013-03-09/2013-03-11 \ No newline at end of file diff --git a/backend/src/test/resources/tests/filter/DIFFSUM_REAL/expected_SUM_Filter.csv b/backend/src/test/resources/tests/filter/DIFFSUM_REAL/expected_SUM_Filter.csv index 26c5af160f..92e95e61ff 100644 --- a/backend/src/test/resources/tests/filter/DIFFSUM_REAL/expected_SUM_Filter.csv +++ b/backend/src/test/resources/tests/filter/DIFFSUM_REAL/expected_SUM_Filter.csv @@ -1,8 +1,8 @@ result,dates -2,{2017-05-06/2017-05-06} -3,{2012-05-29/2012-05-29} -4,{2010-08-14/2010-08-14} -7,{2016-06-23/2016-06-23} -9,{2018-03-22/2018-03-24} -10,{2011-02-07/2011-02-08} -11,{2013-03-09/2013-03-11} \ No newline at end of file +2,2017-05-06/2017-05-06 +3,2012-05-29/2012-05-29 +4,2010-08-14/2010-08-14 +7,2016-06-23/2016-06-23 +9,2018-03-22/2018-03-24 +10,2011-02-07/2011-02-08 +11,2013-03-09/2013-03-11 \ No newline at end of file diff --git a/backend/src/test/resources/tests/filter/DURATION_SUM/expected_DURATION_SUM_Filter.csv b/backend/src/test/resources/tests/filter/DURATION_SUM/expected_DURATION_SUM_Filter.csv index 2a827e2b50..b445a21e43 100644 --- a/backend/src/test/resources/tests/filter/DURATION_SUM/expected_DURATION_SUM_Filter.csv +++ b/backend/src/test/resources/tests/filter/DURATION_SUM/expected_DURATION_SUM_Filter.csv @@ -1,4 +1,4 @@ result,dates -1,{2010-01-01/2010-01-31} -4,{2010-06-15/2010-06-20} -5,{2011-03-02/2011-03-13} \ No newline at end of file +1,2010-01-01/2010-01-31 +4,2010-06-15/2010-06-20 +5,2011-03-02/2011-03-13 \ No newline at end of file diff --git a/backend/src/test/resources/tests/filter/DURATION_SUM_2/expected_DURATION_SUM_2_Filter.csv b/backend/src/test/resources/tests/filter/DURATION_SUM_2/expected_DURATION_SUM_2_Filter.csv index 90f7386239..14916517d3 100644 --- a/backend/src/test/resources/tests/filter/DURATION_SUM_2/expected_DURATION_SUM_2_Filter.csv +++ b/backend/src/test/resources/tests/filter/DURATION_SUM_2/expected_DURATION_SUM_2_Filter.csv @@ -1,5 +1,5 @@ result,dates -1,{2010-01-01/2010-01-31} -3,{2013-08-10/2013-08-10} -4,{2010-06-15/2010-06-20} -5,{2011-03-02/2011-03-13} \ No newline at end of file +1,2010-01-01/2010-01-31 +3,2013-08-10/2013-08-10 +4,2010-06-15/2010-06-20 +5,2011-03-02/2011-03-13 \ No newline at end of file diff --git a/backend/src/test/resources/tests/filter/MULTI_SELECT/expected_SELECT_Filter.csv b/backend/src/test/resources/tests/filter/MULTI_SELECT/expected_SELECT_Filter.csv index 7439330f69..24e12dd9d7 100644 --- a/backend/src/test/resources/tests/filter/MULTI_SELECT/expected_SELECT_Filter.csv +++ b/backend/src/test/resources/tests/filter/MULTI_SELECT/expected_SELECT_Filter.csv @@ -1,4 +1,4 @@ result,dates -1,{2012-01-01/2012-01-01} -3,{2013-11-10/2013-11-10} -5,{2013-11-10/2013-11-10} \ No newline at end of file +1,2012-01-01/2012-01-01 +3,2013-11-10/2013-11-10 +5,2013-11-10/2013-11-10 \ No newline at end of file diff --git a/backend/src/test/resources/tests/filter/NUMBER_DECIMAL/expected_NUMBER_Filter.csv b/backend/src/test/resources/tests/filter/NUMBER_DECIMAL/expected_NUMBER_Filter.csv index 0c5c5ac598..96ceb912fc 100644 --- a/backend/src/test/resources/tests/filter/NUMBER_DECIMAL/expected_NUMBER_Filter.csv +++ b/backend/src/test/resources/tests/filter/NUMBER_DECIMAL/expected_NUMBER_Filter.csv @@ -1,6 +1,6 @@ result,dates -3,{2012-05-29/2012-05-30} -4,{2010-08-15/2010-08-15} -5,{2017-03-14/2017-03-14} -7,{2016-06-23/2016-06-23} -12,{2012-05-29/2012-05-29} \ No newline at end of file +3,2012-05-29/2012-05-30 +4,2010-08-15/2010-08-15 +5,2017-03-14/2017-03-14 +7,2016-06-23/2016-06-23 +12,2012-05-29/2012-05-29 \ No newline at end of file diff --git a/backend/src/test/resources/tests/filter/NUMBER_INTEGER/expected_NUMBER_Filter.csv b/backend/src/test/resources/tests/filter/NUMBER_INTEGER/expected_NUMBER_Filter.csv index f279fa27f1..8dab95b397 100644 --- a/backend/src/test/resources/tests/filter/NUMBER_INTEGER/expected_NUMBER_Filter.csv +++ b/backend/src/test/resources/tests/filter/NUMBER_INTEGER/expected_NUMBER_Filter.csv @@ -1,5 +1,5 @@ result,dates -3,{2012-05-29/2012-05-30} -4,{2010-08-15/2010-08-15} -5,{2017-03-14/2017-03-14} -8,{2012-05-29/2012-05-29} \ No newline at end of file +3,2012-05-29/2012-05-30 +4,2010-08-15/2010-08-15 +5,2017-03-14/2017-03-14 +8,2012-05-29/2012-05-29 \ No newline at end of file diff --git a/backend/src/test/resources/tests/filter/NUMBER_INTEGER2/expected_NUMBER_Filter.csv b/backend/src/test/resources/tests/filter/NUMBER_INTEGER2/expected_NUMBER_Filter.csv index 5305f53f05..1d1e4c3824 100644 --- a/backend/src/test/resources/tests/filter/NUMBER_INTEGER2/expected_NUMBER_Filter.csv +++ b/backend/src/test/resources/tests/filter/NUMBER_INTEGER2/expected_NUMBER_Filter.csv @@ -1,2 +1,2 @@ result,dates -1,{2015-03-17/2015-12-31} \ No newline at end of file +1,2015-03-17/2015-12-31 \ No newline at end of file diff --git a/backend/src/test/resources/tests/filter/NUMBER_MONEY/expected_NUMBER_Filter.csv b/backend/src/test/resources/tests/filter/NUMBER_MONEY/expected_NUMBER_Filter.csv index 0c5c5ac598..96ceb912fc 100644 --- a/backend/src/test/resources/tests/filter/NUMBER_MONEY/expected_NUMBER_Filter.csv +++ b/backend/src/test/resources/tests/filter/NUMBER_MONEY/expected_NUMBER_Filter.csv @@ -1,6 +1,6 @@ result,dates -3,{2012-05-29/2012-05-30} -4,{2010-08-15/2010-08-15} -5,{2017-03-14/2017-03-14} -7,{2016-06-23/2016-06-23} -12,{2012-05-29/2012-05-29} \ No newline at end of file +3,2012-05-29/2012-05-30 +4,2010-08-15/2010-08-15 +5,2017-03-14/2017-03-14 +7,2016-06-23/2016-06-23 +12,2012-05-29/2012-05-29 \ No newline at end of file diff --git a/backend/src/test/resources/tests/filter/NUMBER_REAL/expected_NUMBER_Filter.csv b/backend/src/test/resources/tests/filter/NUMBER_REAL/expected_NUMBER_Filter.csv index 0c5c5ac598..96ceb912fc 100644 --- a/backend/src/test/resources/tests/filter/NUMBER_REAL/expected_NUMBER_Filter.csv +++ b/backend/src/test/resources/tests/filter/NUMBER_REAL/expected_NUMBER_Filter.csv @@ -1,6 +1,6 @@ result,dates -3,{2012-05-29/2012-05-30} -4,{2010-08-15/2010-08-15} -5,{2017-03-14/2017-03-14} -7,{2016-06-23/2016-06-23} -12,{2012-05-29/2012-05-29} \ No newline at end of file +3,2012-05-29/2012-05-30 +4,2010-08-15/2010-08-15 +5,2017-03-14/2017-03-14 +7,2016-06-23/2016-06-23 +12,2012-05-29/2012-05-29 \ No newline at end of file diff --git a/backend/src/test/resources/tests/filter/NUMBER_REAL_MISSING/expected_NUMBER_Filter.csv b/backend/src/test/resources/tests/filter/NUMBER_REAL_MISSING/expected_NUMBER_Filter.csv index 3799e57909..e61be43859 100644 --- a/backend/src/test/resources/tests/filter/NUMBER_REAL_MISSING/expected_NUMBER_Filter.csv +++ b/backend/src/test/resources/tests/filter/NUMBER_REAL_MISSING/expected_NUMBER_Filter.csv @@ -1,2 +1,2 @@ result,dates -1,{2015-03-17/2015-03-31} \ No newline at end of file +1,2015-03-17/2015-03-31 \ No newline at end of file diff --git a/backend/src/test/resources/tests/filter/PREFIX/expected_SELECT_Filter.csv b/backend/src/test/resources/tests/filter/PREFIX/expected_SELECT_Filter.csv index cf687cf2b1..0e1f6aa20f 100644 --- a/backend/src/test/resources/tests/filter/PREFIX/expected_SELECT_Filter.csv +++ b/backend/src/test/resources/tests/filter/PREFIX/expected_SELECT_Filter.csv @@ -1,4 +1,4 @@ result,dates -1,{2012-01-01/2012-01-01} -2,{2012-01-01/2012-01-01} -5,{2012-01-01/2012-01-01} \ No newline at end of file +1,2012-01-01/2012-01-01 +2,2012-01-01/2012-01-01 +5,2012-01-01/2012-01-01 \ No newline at end of file diff --git a/backend/src/test/resources/tests/filter/QUARTERS_IN_YEAR/expected_QUARTERS_IN_YEAR_Filter.csv b/backend/src/test/resources/tests/filter/QUARTERS_IN_YEAR/expected_QUARTERS_IN_YEAR_Filter.csv index 8845f4e8f7..a6fe638862 100644 --- a/backend/src/test/resources/tests/filter/QUARTERS_IN_YEAR/expected_QUARTERS_IN_YEAR_Filter.csv +++ b/backend/src/test/resources/tests/filter/QUARTERS_IN_YEAR/expected_QUARTERS_IN_YEAR_Filter.csv @@ -1,8 +1,8 @@ result,dates -2,{2015-01-01/2015-12-31} -4,{2015-01-01/2015-12-31} -6,{2015-01-01/2015-12-31} -7,{2015-01-01/2015-12-31} -9,{2015-01-01/2016-12-31} -11,{2015-01-01/2016-12-31} -12,{2015-01-01/2016-12-31} \ No newline at end of file +2,2015-01-01/2015-12-31 +4,2015-01-01/2015-12-31 +6,2015-01-01/2015-12-31 +7,2015-01-01/2015-12-31 +9,2015-01-01/2016-12-31 +11,2015-01-01/2016-12-31 +12,2015-01-01/2016-12-31 \ No newline at end of file diff --git a/backend/src/test/resources/tests/filter/SELECT/expected_SELECT_Filter.csv b/backend/src/test/resources/tests/filter/SELECT/expected_SELECT_Filter.csv index 528a9786b2..6ae7d52095 100644 --- a/backend/src/test/resources/tests/filter/SELECT/expected_SELECT_Filter.csv +++ b/backend/src/test/resources/tests/filter/SELECT/expected_SELECT_Filter.csv @@ -1,3 +1,3 @@ result,dates -1,{2012-01-01/2012-01-01} -2,{2010-07-15/2010-07-15} \ No newline at end of file +1,2012-01-01/2012-01-01 +2,2010-07-15/2010-07-15 \ No newline at end of file diff --git a/backend/src/test/resources/tests/filter/SINGLE_SELECT_EMPTY_VALUES/expected_SINGLE_SELECT_Filter.csv b/backend/src/test/resources/tests/filter/SINGLE_SELECT_EMPTY_VALUES/expected_SINGLE_SELECT_Filter.csv index 528a9786b2..6ae7d52095 100644 --- a/backend/src/test/resources/tests/filter/SINGLE_SELECT_EMPTY_VALUES/expected_SINGLE_SELECT_Filter.csv +++ b/backend/src/test/resources/tests/filter/SINGLE_SELECT_EMPTY_VALUES/expected_SINGLE_SELECT_Filter.csv @@ -1,3 +1,3 @@ result,dates -1,{2012-01-01/2012-01-01} -2,{2010-07-15/2010-07-15} \ No newline at end of file +1,2012-01-01/2012-01-01 +2,2010-07-15/2010-07-15 \ No newline at end of file diff --git a/backend/src/test/resources/tests/filter/SINGLE_SELECT_SPEZIAL_CHARACTER/expected_SINGLE_SELECT_Filter.csv b/backend/src/test/resources/tests/filter/SINGLE_SELECT_SPEZIAL_CHARACTER/expected_SINGLE_SELECT_Filter.csv index 3265d716b0..ec2f5318ca 100644 --- a/backend/src/test/resources/tests/filter/SINGLE_SELECT_SPEZIAL_CHARACTER/expected_SINGLE_SELECT_Filter.csv +++ b/backend/src/test/resources/tests/filter/SINGLE_SELECT_SPEZIAL_CHARACTER/expected_SINGLE_SELECT_Filter.csv @@ -1,2 +1,2 @@ result,dates -2,{2010-07-15/2010-07-15} \ No newline at end of file +2,2010-07-15/2010-07-15 \ No newline at end of file diff --git a/backend/src/test/resources/tests/filter/SINGLE_SELECT_SPEZIAL_CHARACTER2/expected_SINGLE_SELECT_Filter.csv b/backend/src/test/resources/tests/filter/SINGLE_SELECT_SPEZIAL_CHARACTER2/expected_SINGLE_SELECT_Filter.csv index 3265d716b0..ec2f5318ca 100644 --- a/backend/src/test/resources/tests/filter/SINGLE_SELECT_SPEZIAL_CHARACTER2/expected_SINGLE_SELECT_Filter.csv +++ b/backend/src/test/resources/tests/filter/SINGLE_SELECT_SPEZIAL_CHARACTER2/expected_SINGLE_SELECT_Filter.csv @@ -1,2 +1,2 @@ result,dates -2,{2010-07-15/2010-07-15} \ No newline at end of file +2,2010-07-15/2010-07-15 \ No newline at end of file diff --git a/backend/src/test/resources/tests/filter/SINGLE_SELECT_SPEZIAL_CHARACTER2/expected_SINGLE_SELECT_Filter2.csv b/backend/src/test/resources/tests/filter/SINGLE_SELECT_SPEZIAL_CHARACTER2/expected_SINGLE_SELECT_Filter2.csv index 55d21ff39d..5b1d62939d 100644 --- a/backend/src/test/resources/tests/filter/SINGLE_SELECT_SPEZIAL_CHARACTER2/expected_SINGLE_SELECT_Filter2.csv +++ b/backend/src/test/resources/tests/filter/SINGLE_SELECT_SPEZIAL_CHARACTER2/expected_SINGLE_SELECT_Filter2.csv @@ -1,2 +1,2 @@ result,dates -5,{2010-07-15/2010-07-15} \ No newline at end of file +5,2010-07-15/2010-07-15 \ No newline at end of file diff --git a/backend/src/test/resources/tests/filter/SUM_DECIMAL/expected_SUM_Filter.csv b/backend/src/test/resources/tests/filter/SUM_DECIMAL/expected_SUM_Filter.csv index 0fc41b4b5a..bcf50be069 100644 --- a/backend/src/test/resources/tests/filter/SUM_DECIMAL/expected_SUM_Filter.csv +++ b/backend/src/test/resources/tests/filter/SUM_DECIMAL/expected_SUM_Filter.csv @@ -1,5 +1,5 @@ result,dates -1,{2015-03-17/2015-03-17} -2,{2015-03-17/2015-03-17} -3,{2015-03-17/2015-03-20} -4,{2015-03-17/2015-03-18} +1,2015-03-17/2015-03-17 +2,2015-03-17/2015-03-17 +3,2015-03-17/2015-03-20 +4,2015-03-17/2015-03-18 diff --git a/backend/src/test/resources/tests/filter/SUM_INTEGER/expected_SUM_Filter.csv b/backend/src/test/resources/tests/filter/SUM_INTEGER/expected_SUM_Filter.csv index 0fc41b4b5a..bcf50be069 100644 --- a/backend/src/test/resources/tests/filter/SUM_INTEGER/expected_SUM_Filter.csv +++ b/backend/src/test/resources/tests/filter/SUM_INTEGER/expected_SUM_Filter.csv @@ -1,5 +1,5 @@ result,dates -1,{2015-03-17/2015-03-17} -2,{2015-03-17/2015-03-17} -3,{2015-03-17/2015-03-20} -4,{2015-03-17/2015-03-18} +1,2015-03-17/2015-03-17 +2,2015-03-17/2015-03-17 +3,2015-03-17/2015-03-20 +4,2015-03-17/2015-03-18 diff --git a/backend/src/test/resources/tests/form/EXPORT_FORM/ABSOLUT/SIMPLE/query_results_1.csv b/backend/src/test/resources/tests/form/EXPORT_FORM/ABSOLUT/SIMPLE/query_results_1.csv index a7d37593fe..057d13ee1e 100644 --- a/backend/src/test/resources/tests/form/EXPORT_FORM/ABSOLUT/SIMPLE/query_results_1.csv +++ b/backend/src/test/resources/tests/form/EXPORT_FORM/ABSOLUT/SIMPLE/query_results_1.csv @@ -1,2 +1,2 @@ "result","dates" -1,{2001-12-01/2016-12-01} +1,2001-12-01/2016-12-01 diff --git a/backend/src/test/resources/tests/form/EXPORT_FORM/ENTITY_DATE/SIMPLE/query_results_1.csv b/backend/src/test/resources/tests/form/EXPORT_FORM/ENTITY_DATE/SIMPLE/query_results_1.csv index ed7328dce9..6065f6b982 100644 --- a/backend/src/test/resources/tests/form/EXPORT_FORM/ENTITY_DATE/SIMPLE/query_results_1.csv +++ b/backend/src/test/resources/tests/form/EXPORT_FORM/ENTITY_DATE/SIMPLE/query_results_1.csv @@ -1,2 +1,2 @@ "result","dates" -1,"{2012-01-01/2012-03-31,2012-06-01/2013-07-31}" +1,"2012-01-01/2012-03-31,2012-06-01/2013-07-31" diff --git a/backend/src/test/resources/tests/form/EXPORT_FORM/RELATIVE/RELATIVE_ONLY_FEATURE/query_results_1.csv b/backend/src/test/resources/tests/form/EXPORT_FORM/RELATIVE/RELATIVE_ONLY_FEATURE/query_results_1.csv index 608cc5f236..2da2b88e0e 100644 --- a/backend/src/test/resources/tests/form/EXPORT_FORM/RELATIVE/RELATIVE_ONLY_FEATURE/query_results_1.csv +++ b/backend/src/test/resources/tests/form/EXPORT_FORM/RELATIVE/RELATIVE_ONLY_FEATURE/query_results_1.csv @@ -1,3 +1,3 @@ "result","dates" -1,{2012-01-01/2016-12-01} -23,{2001-12-01/2016-12-01} +1,2012-01-01/2016-12-01 +23,2001-12-01/2016-12-01 diff --git a/backend/src/test/resources/tests/form/EXPORT_FORM/RELATIVE/RELATIVE_ONLY_FEATURE_COMPLETE/query_results_1.csv b/backend/src/test/resources/tests/form/EXPORT_FORM/RELATIVE/RELATIVE_ONLY_FEATURE_COMPLETE/query_results_1.csv index 608cc5f236..2da2b88e0e 100644 --- a/backend/src/test/resources/tests/form/EXPORT_FORM/RELATIVE/RELATIVE_ONLY_FEATURE_COMPLETE/query_results_1.csv +++ b/backend/src/test/resources/tests/form/EXPORT_FORM/RELATIVE/RELATIVE_ONLY_FEATURE_COMPLETE/query_results_1.csv @@ -1,3 +1,3 @@ "result","dates" -1,{2012-01-01/2016-12-01} -23,{2001-12-01/2016-12-01} +1,2012-01-01/2016-12-01 +23,2001-12-01/2016-12-01 diff --git a/backend/src/test/resources/tests/form/EXPORT_FORM/RELATIVE/RELATIVE_ONLY_OUTCOME/query_results_1.csv b/backend/src/test/resources/tests/form/EXPORT_FORM/RELATIVE/RELATIVE_ONLY_OUTCOME/query_results_1.csv index 608cc5f236..2da2b88e0e 100644 --- a/backend/src/test/resources/tests/form/EXPORT_FORM/RELATIVE/RELATIVE_ONLY_OUTCOME/query_results_1.csv +++ b/backend/src/test/resources/tests/form/EXPORT_FORM/RELATIVE/RELATIVE_ONLY_OUTCOME/query_results_1.csv @@ -1,3 +1,3 @@ "result","dates" -1,{2012-01-01/2016-12-01} -23,{2001-12-01/2016-12-01} +1,2012-01-01/2016-12-01 +23,2001-12-01/2016-12-01 diff --git a/backend/src/test/resources/tests/form/EXPORT_FORM/RELATIVE/RELATIVE_ONLY_OUTCOME_COMPLETE/query_results_1.csv b/backend/src/test/resources/tests/form/EXPORT_FORM/RELATIVE/RELATIVE_ONLY_OUTCOME_COMPLETE/query_results_1.csv index 608cc5f236..2da2b88e0e 100644 --- a/backend/src/test/resources/tests/form/EXPORT_FORM/RELATIVE/RELATIVE_ONLY_OUTCOME_COMPLETE/query_results_1.csv +++ b/backend/src/test/resources/tests/form/EXPORT_FORM/RELATIVE/RELATIVE_ONLY_OUTCOME_COMPLETE/query_results_1.csv @@ -1,3 +1,3 @@ "result","dates" -1,{2012-01-01/2016-12-01} -23,{2001-12-01/2016-12-01} +1,2012-01-01/2016-12-01 +23,2001-12-01/2016-12-01 diff --git a/backend/src/test/resources/tests/form/EXPORT_FORM/RELATIVE/SIMPLE/query_results_1.csv b/backend/src/test/resources/tests/form/EXPORT_FORM/RELATIVE/SIMPLE/query_results_1.csv index a6adc1a26c..3ff60a6b13 100644 --- a/backend/src/test/resources/tests/form/EXPORT_FORM/RELATIVE/SIMPLE/query_results_1.csv +++ b/backend/src/test/resources/tests/form/EXPORT_FORM/RELATIVE/SIMPLE/query_results_1.csv @@ -1,3 +1,3 @@ "result","dates" -1,{2001-12-01/2016-12-01} -23,{2001-12-01/2016-12-01} +1,2001-12-01/2016-12-01 +23,2001-12-01/2016-12-01 diff --git a/backend/src/test/resources/tests/form/FULL_EXPORT_FORM/expected.csv b/backend/src/test/resources/tests/form/FULL_EXPORT_FORM/expected.csv index 3ecd749da8..2bb9da91c3 100644 --- a/backend/src/test/resources/tests/form/FULL_EXPORT_FORM/expected.csv +++ b/backend/src/test/resources/tests/form/FULL_EXPORT_FORM/expected.csv @@ -1,8 +1,8 @@ result,dates,secondary,table - column,vers_stamm - date,vers_stamm - date_end,vers_stamm - geburtsdatum,vers_stamm - geschlecht -1,{2012-03-01/2012-03-01},1,A,,,, -1,{2012-03-02/2012-03-02},1,A,,,, -1,{2012-03-04/2012-03-04},2,A,,,, -1,{2012-03-02/2012-03-02},1,B,,,, -1,{2012-03-04/2012-03-04},2,B,,,, -1,{2012-03-06/2012-03-06},3,B,,,, -1,{2012-01-01/2012-01-01},,,2012-01-01/2012-12-31,2012-12-31,1957-01-01,1 \ No newline at end of file +1,2012-03-01/2012-03-01,1,A,,,, +1,2012-03-02/2012-03-02,1,A,,,, +1,2012-03-04/2012-03-04,2,A,,,, +1,2012-03-02/2012-03-02,1,B,,,, +1,2012-03-04/2012-03-04,2,B,,,, +1,2012-03-06/2012-03-06,3,B,,,, +1,2012-01-01/2012-01-01,,,2012-01-01/2012-12-31,2012-12-31,1957-01-01,1 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/ARRAY_CONCEPT_QUERY/expected.csv b/backend/src/test/resources/tests/query/ARRAY_CONCEPT_QUERY/expected.csv index 09906e1ffe..9021e96edc 100644 --- a/backend/src/test/resources/tests/query/ARRAY_CONCEPT_QUERY/expected.csv +++ b/backend/src/test/resources/tests/query/ARRAY_CONCEPT_QUERY/expected.csv @@ -1,5 +1,5 @@ result,dates,select - count_genders,select - count_genders_1 -1,"{2012-01-01/2012-01-02, 2013-01-01/2013-01-02}",4,4 -2,{2010-07-15/2010-07-15},, -3,{2013-11-10/2013-11-10},1, -4,{2012-11-11/2012-11-11},1, \ No newline at end of file +1,"2012-01-01/2012-01-02, 2013-01-01/2013-01-02",4,4 +2,2010-07-15/2010-07-15,, +3,2013-11-10/2013-11-10,1, +4,2012-11-11/2012-11-11,1, \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/BIG_MULTI_SELECT_DATE_RESTRICTION_OR_CONCEPT_QUERY/expected.csv b/backend/src/test/resources/tests/query/BIG_MULTI_SELECT_DATE_RESTRICTION_OR_CONCEPT_QUERY/expected.csv index 587b3269ee..225e384fcd 100644 --- a/backend/src/test/resources/tests/query/BIG_MULTI_SELECT_DATE_RESTRICTION_OR_CONCEPT_QUERY/expected.csv +++ b/backend/src/test/resources/tests/query/BIG_MULTI_SELECT_DATE_RESTRICTION_OR_CONCEPT_QUERY/expected.csv @@ -1,2 +1,2 @@ result,dates -1,{2012-01-01/2012-01-01} \ No newline at end of file +1,2012-01-01/2012-01-01 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/BIG_MULTI_SELECT_NEGATION_DATE_RESTRICTION_OR_CONCEPT_QUERY/expected.csv b/backend/src/test/resources/tests/query/BIG_MULTI_SELECT_NEGATION_DATE_RESTRICTION_OR_CONCEPT_QUERY/expected.csv index fed99294c1..8ae78128e7 100644 --- a/backend/src/test/resources/tests/query/BIG_MULTI_SELECT_NEGATION_DATE_RESTRICTION_OR_CONCEPT_QUERY/expected.csv +++ b/backend/src/test/resources/tests/query/BIG_MULTI_SELECT_NEGATION_DATE_RESTRICTION_OR_CONCEPT_QUERY/expected.csv @@ -1,8 +1,8 @@ result,dates -2,{} -3,{} -4,{} -5,{} -6,{} -7,{} -8,{} \ No newline at end of file +2, +3, +4, +5, +6, +7, +8, \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/COMMON_CONCEPT_ICD_QUERY/expected-kh.csv b/backend/src/test/resources/tests/query/COMMON_CONCEPT_ICD_QUERY/expected-kh.csv index 98b10d020f..0c535e30da 100644 --- a/backend/src/test/resources/tests/query/COMMON_CONCEPT_ICD_QUERY/expected-kh.csv +++ b/backend/src/test/resources/tests/query/COMMON_CONCEPT_ICD_QUERY/expected-kh.csv @@ -1,3 +1,3 @@ result,dates -3,{2017-03-31/2017-03-31} -15,{2017-06-30/2017-06-30} \ No newline at end of file +3,2017-03-31/2017-03-31 +15,2017-06-30/2017-06-30 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/CONCEPT_DATE_RESTRICTION_WITHOUT_VALIDITYDATE/expected.csv b/backend/src/test/resources/tests/query/CONCEPT_DATE_RESTRICTION_WITHOUT_VALIDITYDATE/expected.csv index fc2d86e6d7..64daac199f 100644 --- a/backend/src/test/resources/tests/query/CONCEPT_DATE_RESTRICTION_WITHOUT_VALIDITYDATE/expected.csv +++ b/backend/src/test/resources/tests/query/CONCEPT_DATE_RESTRICTION_WITHOUT_VALIDITYDATE/expected.csv @@ -1,3 +1,3 @@ result,dates,test_tree - test_child1 - Ausgabe test -1,{2017-01-01/2017-12-31},A1 -3,{2017-01-01/2017-12-31},A1 \ No newline at end of file +1,2017-01-01/2017-12-31,A1 +3,2017-01-01/2017-12-31,A1 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/CONCEPT_RESTRICTION_QUERY/expected-kh.csv b/backend/src/test/resources/tests/query/CONCEPT_RESTRICTION_QUERY/expected-kh.csv index 77a92d88fa..9320e946da 100644 --- a/backend/src/test/resources/tests/query/CONCEPT_RESTRICTION_QUERY/expected-kh.csv +++ b/backend/src/test/resources/tests/query/CONCEPT_RESTRICTION_QUERY/expected-kh.csv @@ -1,2 +1,2 @@ result,dates -3,{2017-03-31/2017-03-31} \ No newline at end of file +3,2017-03-31/2017-03-31 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/CONCEPT_WITHOUT_VALIDITYDATE/expected.csv b/backend/src/test/resources/tests/query/CONCEPT_WITHOUT_VALIDITYDATE/expected.csv index 879b21b0eb..bb57795aad 100644 --- a/backend/src/test/resources/tests/query/CONCEPT_WITHOUT_VALIDITYDATE/expected.csv +++ b/backend/src/test/resources/tests/query/CONCEPT_WITHOUT_VALIDITYDATE/expected.csv @@ -1,3 +1,3 @@ result,dates,test_tree - test_child1 - Ausgabe test -1,{-∞/+∞},A1 -3,{-∞/+∞},A1 \ No newline at end of file +1,-∞/+∞,A1 +3,-∞/+∞,A1 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/CONNECTOR_CONDITION/expected.csv b/backend/src/test/resources/tests/query/CONNECTOR_CONDITION/expected.csv index 8114ea686d..ce93de1599 100644 --- a/backend/src/test/resources/tests/query/CONNECTOR_CONDITION/expected.csv +++ b/backend/src/test/resources/tests/query/CONNECTOR_CONDITION/expected.csv @@ -1,3 +1,3 @@ result,dates -1,{2012-01-01/2012-01-01} -3,{2013-11-10/2013-11-10} +1,2012-01-01/2012-01-01 +3,2013-11-10/2013-11-10 diff --git a/backend/src/test/resources/tests/query/DATE_DISTANCE/expected-erster.csv b/backend/src/test/resources/tests/query/DATE_DISTANCE/expected-erster.csv index d0f7975e39..9cb121265a 100644 --- a/backend/src/test/resources/tests/query/DATE_DISTANCE/expected-erster.csv +++ b/backend/src/test/resources/tests/query/DATE_DISTANCE/expected-erster.csv @@ -1,3 +1,3 @@ result,dates -2,{2011-01-01/2011-01-01} -5,{2011-01-01/2011-01-01} \ No newline at end of file +2,2011-01-01/2011-01-01 +5,2011-01-01/2011-01-01 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/DATE_DISTANCE/expected-letzter.csv b/backend/src/test/resources/tests/query/DATE_DISTANCE/expected-letzter.csv index fe8c86fc3f..19fadaa3b2 100644 --- a/backend/src/test/resources/tests/query/DATE_DISTANCE/expected-letzter.csv +++ b/backend/src/test/resources/tests/query/DATE_DISTANCE/expected-letzter.csv @@ -1,3 +1,3 @@ result,dates -3,{2011-12-31/2011-12-31} -5,{2011-12-31/2011-12-31} \ No newline at end of file +3,2011-12-31/2011-12-31 +5,2011-12-31/2011-12-31 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/DATE_DISTANCE/expected-versichertenzeit.csv b/backend/src/test/resources/tests/query/DATE_DISTANCE/expected-versichertenzeit.csv index 7b405e107f..17c77636ce 100644 --- a/backend/src/test/resources/tests/query/DATE_DISTANCE/expected-versichertenzeit.csv +++ b/backend/src/test/resources/tests/query/DATE_DISTANCE/expected-versichertenzeit.csv @@ -1,5 +1,5 @@ result,dates -1,{2011-01-01/2011-01-01} -2,{2011-01-01/2011-01-01} -3,{2011-01-01/2011-01-01} -5,{2011-01-01/2011-01-01} \ No newline at end of file +1,2011-01-01/2011-01-01 +2,2011-01-01/2011-01-01 +3,2011-01-01/2011-01-01 +5,2011-01-01/2011-01-01 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/DATE_DISTANCE_AGE_SPAN/expected-erster.csv b/backend/src/test/resources/tests/query/DATE_DISTANCE_AGE_SPAN/expected-erster.csv index 47be0a629d..c8f9763eb8 100644 --- a/backend/src/test/resources/tests/query/DATE_DISTANCE_AGE_SPAN/expected-erster.csv +++ b/backend/src/test/resources/tests/query/DATE_DISTANCE_AGE_SPAN/expected-erster.csv @@ -1,5 +1,5 @@ result,dates -2,{2011-01-01/2011-01-01} -5,{2011-01-01/2011-01-01} -32,{2011-01-01/2011-01-01} -35,{2011-01-01/2011-01-01} \ No newline at end of file +2,2011-01-01/2011-01-01 +5,2011-01-01/2011-01-01 +32,2011-01-01/2011-01-01 +35,2011-01-01/2011-01-01 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/DATE_DISTANCE_AGE_SPAN/expected-letzter.csv b/backend/src/test/resources/tests/query/DATE_DISTANCE_AGE_SPAN/expected-letzter.csv index dc8899feb3..b8c74b477d 100644 --- a/backend/src/test/resources/tests/query/DATE_DISTANCE_AGE_SPAN/expected-letzter.csv +++ b/backend/src/test/resources/tests/query/DATE_DISTANCE_AGE_SPAN/expected-letzter.csv @@ -1,5 +1,5 @@ result,dates -3,{2011-12-31/2011-12-31} -5,{2011-12-31/2011-12-31} -33,{2011-12-31/2011-12-31} -35,{2011-12-31/2011-12-31} \ No newline at end of file +3,2011-12-31/2011-12-31 +5,2011-12-31/2011-12-31 +33,2011-12-31/2011-12-31 +35,2011-12-31/2011-12-31 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/DATE_DISTANCE_AGE_SPAN/expected-versichertenzeit.csv b/backend/src/test/resources/tests/query/DATE_DISTANCE_AGE_SPAN/expected-versichertenzeit.csv index 2401164da9..2789096005 100644 --- a/backend/src/test/resources/tests/query/DATE_DISTANCE_AGE_SPAN/expected-versichertenzeit.csv +++ b/backend/src/test/resources/tests/query/DATE_DISTANCE_AGE_SPAN/expected-versichertenzeit.csv @@ -1,9 +1,9 @@ result,dates -1,{2011-01-01/2011-01-01} -2,{2011-01-01/2011-01-01} -3,{2011-01-01/2011-01-01} -5,{2011-01-01/2011-01-01} -31,{2011-01-01/2011-01-01} -32,{2011-01-01/2011-01-01} -33,{2011-01-01/2011-01-01} -35,{2011-01-01/2011-01-01} \ No newline at end of file +1,2011-01-01/2011-01-01 +2,2011-01-01/2011-01-01 +3,2011-01-01/2011-01-01 +5,2011-01-01/2011-01-01 +31,2011-01-01/2011-01-01 +32,2011-01-01/2011-01-01 +33,2011-01-01/2011-01-01 +35,2011-01-01/2011-01-01 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/DATE_DISTANCE_NEGATION/expected-erster.csv b/backend/src/test/resources/tests/query/DATE_DISTANCE_NEGATION/expected-erster.csv index b82e7abd18..e89be33c09 100644 --- a/backend/src/test/resources/tests/query/DATE_DISTANCE_NEGATION/expected-erster.csv +++ b/backend/src/test/resources/tests/query/DATE_DISTANCE_NEGATION/expected-erster.csv @@ -1,21 +1,21 @@ result,dates -1,{} -3,{} -4,{} -21,{} -22,{} -23,{} -24,{} -25,{} -31,{} -32,{} -33,{} -34,{} -35,{} -41,{} -42,{} -43,{} -44,{} -45,{} -51,{} -61,{} \ No newline at end of file +1, +3, +4, +21, +22, +23, +24, +25, +31, +32, +33, +34, +35, +41, +42, +43, +44, +45, +51, +61, \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/DATE_DISTANCE_NEGATION/expected-letzter.csv b/backend/src/test/resources/tests/query/DATE_DISTANCE_NEGATION/expected-letzter.csv index 98ae8c8a3f..c9d4a4e993 100644 --- a/backend/src/test/resources/tests/query/DATE_DISTANCE_NEGATION/expected-letzter.csv +++ b/backend/src/test/resources/tests/query/DATE_DISTANCE_NEGATION/expected-letzter.csv @@ -1,21 +1,21 @@ result,dates -1,{} -2,{} -4,{} -21,{} -22,{} -23,{} -24,{} -25,{} -31,{} -32,{} -33,{} -34,{} -35,{} -41,{} -42,{} -43,{} -44,{} -45,{} -51,{} -61,{} \ No newline at end of file +1, +2, +4, +21, +22, +23, +24, +25, +31, +32, +33, +34, +35, +41, +42, +43, +44, +45, +51, +61, \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/DATE_DISTANCE_NEGATION/expected-versichertenzeit.csv b/backend/src/test/resources/tests/query/DATE_DISTANCE_NEGATION/expected-versichertenzeit.csv index cd19af88da..ea3520b39a 100644 --- a/backend/src/test/resources/tests/query/DATE_DISTANCE_NEGATION/expected-versichertenzeit.csv +++ b/backend/src/test/resources/tests/query/DATE_DISTANCE_NEGATION/expected-versichertenzeit.csv @@ -1,19 +1,19 @@ result,dates -4,{} -21,{} -22,{} -23,{} -24,{} -25,{} -31,{} -32,{} -33,{} -34,{} -35,{} -41,{} -42,{} -43,{} -44,{} -45,{} -51,{} -61,{} \ No newline at end of file +4, +21, +22, +23, +24, +25, +31, +32, +33, +34, +35, +41, +42, +43, +44, +45, +51, +61, \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/DELETE_IMPORT_TESTS/expected.csv b/backend/src/test/resources/tests/query/DELETE_IMPORT_TESTS/expected.csv index f0f34f6300..443c6a4fd1 100644 --- a/backend/src/test/resources/tests/query/DELETE_IMPORT_TESTS/expected.csv +++ b/backend/src/test/resources/tests/query/DELETE_IMPORT_TESTS/expected.csv @@ -1,3 +1,3 @@ result,dates -1,{2012-01-01/2012-01-01} -3,{2013-11-10/2013-11-10} \ No newline at end of file +1,2012-01-01/2012-01-01 +3,2013-11-10/2013-11-10 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/DURATION_SUM_EMPTY_DATE_CONCEPT_QUERY/expected-zeitspanne.csv b/backend/src/test/resources/tests/query/DURATION_SUM_EMPTY_DATE_CONCEPT_QUERY/expected-zeitspanne.csv index 5aecef9055..711e624d73 100644 --- a/backend/src/test/resources/tests/query/DURATION_SUM_EMPTY_DATE_CONCEPT_QUERY/expected-zeitspanne.csv +++ b/backend/src/test/resources/tests/query/DURATION_SUM_EMPTY_DATE_CONCEPT_QUERY/expected-zeitspanne.csv @@ -1,3 +1,3 @@ result,dates -4,{2010-01-01/2010-01-02} -10,{2010-01-02/2010-01-07} \ No newline at end of file +4,2010-01-01/2010-01-02 +10,2010-01-02/2010-01-07 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/LOGICAL/AND/expected.csv b/backend/src/test/resources/tests/query/LOGICAL/AND/expected.csv index 99c7ef85cc..08750d360f 100644 --- a/backend/src/test/resources/tests/query/LOGICAL/AND/expected.csv +++ b/backend/src/test/resources/tests/query/LOGICAL/AND/expected.csv @@ -1,3 +1,3 @@ result,dates,tree - a and tree - b -3,{2012-01-01/2012-01-01},1 -4,{2012-01-01/2012-01-01},1 \ No newline at end of file +3,2012-01-01/2012-01-01,1 +4,2012-01-01/2012-01-01,1 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/LOGICAL/AND_DATE_LOGICAL/expected.csv b/backend/src/test/resources/tests/query/LOGICAL/AND_DATE_LOGICAL/expected.csv index 1b10b88cb3..8cbcef6ee8 100644 --- a/backend/src/test/resources/tests/query/LOGICAL/AND_DATE_LOGICAL/expected.csv +++ b/backend/src/test/resources/tests/query/LOGICAL/AND_DATE_LOGICAL/expected.csv @@ -1,5 +1,5 @@ result,dates,tree - a and tree - b -2,{2012-01-01/2012-01-01},1 -3,{},1 -4,{2012-01-02/2012-01-02},1 -5,{2012-01-03/2012-01-03},1 \ No newline at end of file +2,2012-01-01/2012-01-01,1 +3,,1 +4,2012-01-02/2012-01-02,1 +5,2012-01-03/2012-01-03,1 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/LOGICAL/AND_DURATION_SUM/expected.csv b/backend/src/test/resources/tests/query/LOGICAL/AND_DURATION_SUM/expected.csv index b6c5f4834e..0a78e4b8af 100644 --- a/backend/src/test/resources/tests/query/LOGICAL/AND_DURATION_SUM/expected.csv +++ b/backend/src/test/resources/tests/query/LOGICAL/AND_DURATION_SUM/expected.csv @@ -1,5 +1,5 @@ result,dates,tree - a - event_duration,tree - a and tree - b -2,{2012-01-01/2012-01-01},1,1 -3,{},0,1 -4,{2012-01-02/2012-01-02},1,1 -5,{2012-01-02/2012-01-04},3,1 \ No newline at end of file +2,2012-01-01/2012-01-01,1,1 +3,,0,1 +4,2012-01-02/2012-01-02,1,1 +5,2012-01-02/2012-01-04,3,1 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/LOGICAL/AND_DURATION_SUM_MULTI_CONCEPT/expected.csv b/backend/src/test/resources/tests/query/LOGICAL/AND_DURATION_SUM_MULTI_CONCEPT/expected.csv index 3f192c6956..905c0e449d 100644 --- a/backend/src/test/resources/tests/query/LOGICAL/AND_DURATION_SUM_MULTI_CONCEPT/expected.csv +++ b/backend/src/test/resources/tests/query/LOGICAL/AND_DURATION_SUM_MULTI_CONCEPT/expected.csv @@ -1,7 +1,7 @@ result,secondary,dates,tree1 - a - event_duration,tree1 - a and tree2 - b -1,1,{},0,1 -1,2,{},0,1 -1,3,{},0,1 -1,4,{2012-01-02/2012-01-02},1,1 -1,5,{2012-01-02/2012-01-04},3,1 -2,1,{},0,1 \ No newline at end of file +1,1,,0,1 +1,2,,0,1 +1,3,,0,1 +1,4,2012-01-02/2012-01-02,1,1 +1,5,2012-01-02/2012-01-04,3,1 +2,1,,0,1 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/LOGICAL/AND_NEGATION/expected.csv b/backend/src/test/resources/tests/query/LOGICAL/AND_NEGATION/expected.csv index 055b44a486..96d0a78d5e 100644 --- a/backend/src/test/resources/tests/query/LOGICAL/AND_NEGATION/expected.csv +++ b/backend/src/test/resources/tests/query/LOGICAL/AND_NEGATION/expected.csv @@ -1,3 +1,3 @@ result,dates,another-name-for-and -2,{2012-01-01/2012-01-01},1 -5,{2012-01-01/2012-01-01},1 \ No newline at end of file +2,2012-01-01/2012-01-01,1 +5,2012-01-01/2012-01-01,1 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/LOGICAL/AND_NEGATION_DATE_LOGICAL/expected.csv b/backend/src/test/resources/tests/query/LOGICAL/AND_NEGATION_DATE_LOGICAL/expected.csv index 1fdf185816..6240469bca 100644 --- a/backend/src/test/resources/tests/query/LOGICAL/AND_NEGATION_DATE_LOGICAL/expected.csv +++ b/backend/src/test/resources/tests/query/LOGICAL/AND_NEGATION_DATE_LOGICAL/expected.csv @@ -1,3 +1,3 @@ result,dates,NEGATION and tree - b -2,{2012-01-01/2012-01-01},1 -5,"{2012-01-03/2012-01-03, 2012-01-05/2012-01-05}",1 \ No newline at end of file +2,2012-01-01/2012-01-01,1 +5,"2012-01-03/2012-01-03, 2012-01-05/2012-01-05",1 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/LOGICAL/AND_NEGATION_RESTRICTION_DATE_LOGICAL/expected.csv b/backend/src/test/resources/tests/query/LOGICAL/AND_NEGATION_RESTRICTION_DATE_LOGICAL/expected.csv index ebb5c462b0..67c7c4644b 100644 --- a/backend/src/test/resources/tests/query/LOGICAL/AND_NEGATION_RESTRICTION_DATE_LOGICAL/expected.csv +++ b/backend/src/test/resources/tests/query/LOGICAL/AND_NEGATION_RESTRICTION_DATE_LOGICAL/expected.csv @@ -1,3 +1,3 @@ result,dates,NEGATION and tree - b -2,{2012-01-01/2012-01-01},1 -4,{2012-01-02/2012-01-02},1 \ No newline at end of file +2,2012-01-01/2012-01-01,1 +4,2012-01-02/2012-01-02,1 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/LOGICAL/OR/expected.csv b/backend/src/test/resources/tests/query/LOGICAL/OR/expected.csv index addf5b32a9..afb48f0c4b 100644 --- a/backend/src/test/resources/tests/query/LOGICAL/OR/expected.csv +++ b/backend/src/test/resources/tests/query/LOGICAL/OR/expected.csv @@ -1,5 +1,5 @@ result,dates,tree - a or tree - b -1,{2012-01-01/2012-01-01},1 -2,{2012-01-01/2012-01-01},1 -3,{2012-01-01/2012-01-01},1 -4,{2012-01-01/2012-01-01},1 \ No newline at end of file +1,2012-01-01/2012-01-01,1 +2,2012-01-01/2012-01-01,1 +3,2012-01-01/2012-01-01,1 +4,2012-01-01/2012-01-01,1 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/LOGICAL/OR_AND/expected.csv b/backend/src/test/resources/tests/query/LOGICAL/OR_AND/expected.csv index a3405c69aa..eaf41bc526 100644 --- a/backend/src/test/resources/tests/query/LOGICAL/OR_AND/expected.csv +++ b/backend/src/test/resources/tests/query/LOGICAL/OR_AND/expected.csv @@ -1,5 +1,5 @@ result,dates,tree - a and tree - b,tree - a or tree - b -1,{2012-01-01/2012-01-01},0,1 -2,{2012-01-01/2012-01-01},0,1 -3,{2012-01-01/2012-01-01},1,1 -4,{2012-01-01/2012-01-01},1,1 \ No newline at end of file +1,2012-01-01/2012-01-01,0,1 +2,2012-01-01/2012-01-01,0,1 +3,2012-01-01/2012-01-01,1,1 +4,2012-01-01/2012-01-01,1,1 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/LOGICAL/OR_DATE_LOGICAL/expected.csv b/backend/src/test/resources/tests/query/LOGICAL/OR_DATE_LOGICAL/expected.csv index 4a4c06cd47..64bc17cfeb 100644 --- a/backend/src/test/resources/tests/query/LOGICAL/OR_DATE_LOGICAL/expected.csv +++ b/backend/src/test/resources/tests/query/LOGICAL/OR_DATE_LOGICAL/expected.csv @@ -1,6 +1,6 @@ result,dates,tree - a or tree - b -1,{2012-01-01/2012-01-01},1 -2,{2012-01-01/2012-01-01},1 -3,{2012-01-01/2012-01-02},1 -4,{2012-01-01/2012-01-02},1 -5,"{2012-01-01/2012-01-01, 2012-01-03/2012-01-03, 2012-01-05/2012-01-05}",1 \ No newline at end of file +1,2012-01-01/2012-01-01,1 +2,2012-01-01/2012-01-01,1 +3,2012-01-01/2012-01-02,1 +4,2012-01-01/2012-01-02,1 +5,"2012-01-01/2012-01-01, 2012-01-03/2012-01-03, 2012-01-05/2012-01-05",1 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/MERGE/AND_DURATION_SUM/expected.csv b/backend/src/test/resources/tests/query/MERGE/AND_DURATION_SUM/expected.csv index 6c21ee6f85..47a39a3ef6 100644 --- a/backend/src/test/resources/tests/query/MERGE/AND_DURATION_SUM/expected.csv +++ b/backend/src/test/resources/tests/query/MERGE/AND_DURATION_SUM/expected.csv @@ -1,5 +1,5 @@ result,dates,tree - a - event_duration,tree - a and tree - b -2,{2012-01-01/2012-01-01},1,1 -3,{2012-01-01/2012-01-02},1,1 -4,{2012-01-01/2012-01-02},2,1 -5,{2012-01-01/2012-01-05},4,1 \ No newline at end of file +2,2012-01-01/2012-01-01,1,1 +3,2012-01-01/2012-01-02,1,1 +4,2012-01-01/2012-01-02,2,1 +5,2012-01-01/2012-01-05,4,1 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/MERGE/AND_EVENT_DATE_EXCLUDED/expected.csv b/backend/src/test/resources/tests/query/MERGE/AND_EVENT_DATE_EXCLUDED/expected.csv index 802b298718..37186d850c 100644 --- a/backend/src/test/resources/tests/query/MERGE/AND_EVENT_DATE_EXCLUDED/expected.csv +++ b/backend/src/test/resources/tests/query/MERGE/AND_EVENT_DATE_EXCLUDED/expected.csv @@ -1,5 +1,5 @@ result,dates,tree - a - dates -2,{2012-01-01/2012-01-01},{2012-01-01/2012-01-01} -3,{2012-01-02/2012-01-02},{2012-01-01/2012-01-01} -4,{2012-01-02/2012-01-02},{2012-01-01/2012-01-02} -5,{2012-01-02/2012-01-05},{2012-01-01/2012-01-04} \ No newline at end of file +2,2012-01-01/2012-01-01,2012-01-01/2012-01-01 +3,2012-01-02/2012-01-02,2012-01-01/2012-01-01 +4,2012-01-02/2012-01-02,2012-01-01/2012-01-02 +5,2012-01-02/2012-01-05,2012-01-01/2012-01-04 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/MERGE/OR_EVENT_DATE_EXCLUDED/expected.csv b/backend/src/test/resources/tests/query/MERGE/OR_EVENT_DATE_EXCLUDED/expected.csv index fb2aab296b..c38972f4af 100644 --- a/backend/src/test/resources/tests/query/MERGE/OR_EVENT_DATE_EXCLUDED/expected.csv +++ b/backend/src/test/resources/tests/query/MERGE/OR_EVENT_DATE_EXCLUDED/expected.csv @@ -1,6 +1,6 @@ result,dates,tree - a - dates -1,{},{2012-01-01/2012-01-01} -2,{2012-01-01/2012-01-01},{2012-01-01/2012-01-01} -3,{2012-01-02/2012-01-02},{2012-01-01/2012-01-01} -4,{2012-01-02/2012-01-02},{2012-01-01/2012-01-02} -5,{2012-01-02/2012-01-05},{2012-01-01/2012-01-04} \ No newline at end of file +1,,2012-01-01/2012-01-01 +2,2012-01-01/2012-01-01,2012-01-01/2012-01-01 +3,2012-01-02/2012-01-02,2012-01-01/2012-01-01 +4,2012-01-02/2012-01-02,2012-01-01/2012-01-02 +5,2012-01-02/2012-01-05,2012-01-01/2012-01-04 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/MULTIPLE_CONNECTORS_QUERY/expected.csv b/backend/src/test/resources/tests/query/MULTIPLE_CONNECTORS_QUERY/expected.csv index 587b3269ee..225e384fcd 100644 --- a/backend/src/test/resources/tests/query/MULTIPLE_CONNECTORS_QUERY/expected.csv +++ b/backend/src/test/resources/tests/query/MULTIPLE_CONNECTORS_QUERY/expected.csv @@ -1,2 +1,2 @@ result,dates -1,{2012-01-01/2012-01-01} \ No newline at end of file +1,2012-01-01/2012-01-01 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/MULTIPLE_TABLES_ICD_QUERY2/expected.csv b/backend/src/test/resources/tests/query/MULTIPLE_TABLES_ICD_QUERY2/expected.csv index 2149dd5c8e..f3afb60729 100644 --- a/backend/src/test/resources/tests/query/MULTIPLE_TABLES_ICD_QUERY2/expected.csv +++ b/backend/src/test/resources/tests/query/MULTIPLE_TABLES_ICD_QUERY2/expected.csv @@ -1,8 +1,8 @@ result,dates -2,{2017-06-01/2017-06-01} -3,{2017-03-31/2017-03-31} -4,"{2017-03-31/2017-03-31, 2017-06-01/2017-06-01}" -5,{2017-03-31/2017-03-31} -6,"{2017-03-31/2017-03-31, 2017-05-31/2017-05-31}" -7,"{2017-03-31/2017-03-31, 2017-05-15/2017-05-15, 2017-06-01/2017-06-01, 2017-09-01/2017-09-01}" -17,{2017-06-01/2017-06-01} \ No newline at end of file +2,2017-06-01/2017-06-01 +3,2017-03-31/2017-03-31 +4,"2017-03-31/2017-03-31, 2017-06-01/2017-06-01" +5,2017-03-31/2017-03-31 +6,"2017-03-31/2017-03-31, 2017-05-31/2017-05-31" +7,"2017-03-31/2017-03-31, 2017-05-15/2017-05-15, 2017-06-01/2017-06-01, 2017-09-01/2017-09-01" +17,2017-06-01/2017-06-01 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/MULTI_CONCEPT_QUERY_SEPARATE_DATES/expected.csv b/backend/src/test/resources/tests/query/MULTI_CONCEPT_QUERY_SEPARATE_DATES/expected.csv index 28b3c9346a..f1a509b449 100644 --- a/backend/src/test/resources/tests/query/MULTI_CONCEPT_QUERY_SEPARATE_DATES/expected.csv +++ b/backend/src/test/resources/tests/query/MULTI_CONCEPT_QUERY_SEPARATE_DATES/expected.csv @@ -1,3 +1,3 @@ result,dates,test_tree - test_child1 - dates,test_tree2 - test_child1 - dates -1,{2012-01-01/2012-01-02},{2012-01-01/2012-01-01},{2012-01-02/2012-01-02} -3,{2013-11-10/2013-11-10},{2013-11-10/2013-11-10},{2013-11-10/2013-11-10} \ No newline at end of file +1,2012-01-01/2012-01-02,2012-01-01/2012-01-01,2012-01-02/2012-01-02 +3,2013-11-10/2013-11-10,2013-11-10/2013-11-10,2013-11-10/2013-11-10 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/MULTI_CONNECTOR_QUERY_SEPARATE_DATES/expected.csv b/backend/src/test/resources/tests/query/MULTI_CONNECTOR_QUERY_SEPARATE_DATES/expected.csv index bc799d5786..83ee2d2796 100644 --- a/backend/src/test/resources/tests/query/MULTI_CONNECTOR_QUERY_SEPARATE_DATES/expected.csv +++ b/backend/src/test/resources/tests/query/MULTI_CONNECTOR_QUERY_SEPARATE_DATES/expected.csv @@ -1,3 +1,3 @@ result,dates,test_tree - test_child1 - tree_label dates -1,{2012-01-01/2012-01-02},{2012-01-01/2012-01-01} -3,{2013-11-10/2013-11-10},{2013-11-10/2013-11-10} \ No newline at end of file +1,2012-01-01/2012-01-02,2012-01-01/2012-01-01 +3,2013-11-10/2013-11-10,2013-11-10/2013-11-10 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/MULTI_SELECT_DATE_RESTRICTION_OR_CONCEPT_QUERY/expected.csv b/backend/src/test/resources/tests/query/MULTI_SELECT_DATE_RESTRICTION_OR_CONCEPT_QUERY/expected.csv index 587b3269ee..225e384fcd 100644 --- a/backend/src/test/resources/tests/query/MULTI_SELECT_DATE_RESTRICTION_OR_CONCEPT_QUERY/expected.csv +++ b/backend/src/test/resources/tests/query/MULTI_SELECT_DATE_RESTRICTION_OR_CONCEPT_QUERY/expected.csv @@ -1,2 +1,2 @@ result,dates -1,{2012-01-01/2012-01-01} \ No newline at end of file +1,2012-01-01/2012-01-01 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/MULTI_SELECT_NEGATION_DATE_RESTRICTION_OR_CONCEPT_QUERY/expected.csv b/backend/src/test/resources/tests/query/MULTI_SELECT_NEGATION_DATE_RESTRICTION_OR_CONCEPT_QUERY/expected.csv index 9a90f4af91..2b5597449f 100644 --- a/backend/src/test/resources/tests/query/MULTI_SELECT_NEGATION_DATE_RESTRICTION_OR_CONCEPT_QUERY/expected.csv +++ b/backend/src/test/resources/tests/query/MULTI_SELECT_NEGATION_DATE_RESTRICTION_OR_CONCEPT_QUERY/expected.csv @@ -1,7 +1,7 @@ result,dates -2,{} -3,{} -5,{} -6,{} -7,{} -8,{} \ No newline at end of file +2, +3, +5, +6, +7, +8, \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/MULTI_SELECT_NEGATION_DATE_RESTRICTION_OR_CONCEPT_QUERY2/expected.csv b/backend/src/test/resources/tests/query/MULTI_SELECT_NEGATION_DATE_RESTRICTION_OR_CONCEPT_QUERY2/expected.csv index fed99294c1..8ae78128e7 100644 --- a/backend/src/test/resources/tests/query/MULTI_SELECT_NEGATION_DATE_RESTRICTION_OR_CONCEPT_QUERY2/expected.csv +++ b/backend/src/test/resources/tests/query/MULTI_SELECT_NEGATION_DATE_RESTRICTION_OR_CONCEPT_QUERY2/expected.csv @@ -1,8 +1,8 @@ result,dates -2,{} -3,{} -4,{} -5,{} -6,{} -7,{} -8,{} \ No newline at end of file +2, +3, +4, +5, +6, +7, +8, \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/NUMBER/expected.csv b/backend/src/test/resources/tests/query/NUMBER/expected.csv index b48f310f48..3f0c675515 100644 --- a/backend/src/test/resources/tests/query/NUMBER/expected.csv +++ b/backend/src/test/resources/tests/query/NUMBER/expected.csv @@ -1,5 +1,5 @@ result,dates -11,{2014-06-30/2014-12-13} -51,{2014-06-30/2014-12-13} -61,{2014-06-30/2014-12-13} -71,"{2014-02-12/2014-02-20, 2014-04-30/2014-06-30}" \ No newline at end of file +11,2014-06-30/2014-12-13 +51,2014-06-30/2014-12-13 +61,2014-06-30/2014-12-13 +71,"2014-02-12/2014-02-20, 2014-04-30/2014-06-30" \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/NUMBER_MISSING/expected.csv b/backend/src/test/resources/tests/query/NUMBER_MISSING/expected.csv index 12f510e9fe..b7b471cf48 100644 --- a/backend/src/test/resources/tests/query/NUMBER_MISSING/expected.csv +++ b/backend/src/test/resources/tests/query/NUMBER_MISSING/expected.csv @@ -1,2 +1,2 @@ result,dates -11,{2014-06-30/2014-12-13} \ No newline at end of file +11,2014-06-30/2014-12-13 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/NUMBER_MISSING_NO_RESTRICTION/expected.csv b/backend/src/test/resources/tests/query/NUMBER_MISSING_NO_RESTRICTION/expected.csv index a2a659fbca..e45538857b 100644 --- a/backend/src/test/resources/tests/query/NUMBER_MISSING_NO_RESTRICTION/expected.csv +++ b/backend/src/test/resources/tests/query/NUMBER_MISSING_NO_RESTRICTION/expected.csv @@ -1,3 +1,3 @@ result,dates -11,{2014-06-30/2014-12-13} -20,{2014-06-30/2014-12-13} \ No newline at end of file +11,2014-06-30/2014-12-13 +20,2014-06-30/2014-12-13 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/NUMBER_MISSING_NO_RESTRICTION/expected2.csv b/backend/src/test/resources/tests/query/NUMBER_MISSING_NO_RESTRICTION/expected2.csv index c1f2429eaf..f05a1c66db 100644 --- a/backend/src/test/resources/tests/query/NUMBER_MISSING_NO_RESTRICTION/expected2.csv +++ b/backend/src/test/resources/tests/query/NUMBER_MISSING_NO_RESTRICTION/expected2.csv @@ -1,5 +1,5 @@ result,dates -11,{2014-06-30/2015-06-30} -20,{2014-06-30/2015-06-30} -30,{2015-04-30/2015-06-30} -40,{2015-04-30/2015-06-30} \ No newline at end of file +11,2014-06-30/2015-06-30 +20,2014-06-30/2015-06-30 +30,2015-04-30/2015-06-30 +40,2015-04-30/2015-06-30 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/NUMBER_NEGATION/expected.csv b/backend/src/test/resources/tests/query/NUMBER_NEGATION/expected.csv index 1d1870acf4..277684d341 100644 --- a/backend/src/test/resources/tests/query/NUMBER_NEGATION/expected.csv +++ b/backend/src/test/resources/tests/query/NUMBER_NEGATION/expected.csv @@ -1,5 +1,5 @@ result,dates -20,{} -30,{} -40,{} -80,{} \ No newline at end of file +20, +30, +40, +80, \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/QUERY_AND_NOT_TEMPORAL_SAME_CONCEPT_QUERY/expected.csv b/backend/src/test/resources/tests/query/QUERY_AND_NOT_TEMPORAL_SAME_CONCEPT_QUERY/expected.csv index c1d5ddee92..1a2f2973d8 100644 --- a/backend/src/test/resources/tests/query/QUERY_AND_NOT_TEMPORAL_SAME_CONCEPT_QUERY/expected.csv +++ b/backend/src/test/resources/tests/query/QUERY_AND_NOT_TEMPORAL_SAME_CONCEPT_QUERY/expected.csv @@ -1,3 +1,3 @@ result,dates -2,{} -4,{} +2, +4, diff --git a/backend/src/test/resources/tests/query/RESTART_TEST_DATA/expected.csv b/backend/src/test/resources/tests/query/RESTART_TEST_DATA/expected.csv index f0f34f6300..443c6a4fd1 100644 --- a/backend/src/test/resources/tests/query/RESTART_TEST_DATA/expected.csv +++ b/backend/src/test/resources/tests/query/RESTART_TEST_DATA/expected.csv @@ -1,3 +1,3 @@ result,dates -1,{2012-01-01/2012-01-01} -3,{2013-11-10/2013-11-10} \ No newline at end of file +1,2012-01-01/2012-01-01 +3,2013-11-10/2013-11-10 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/REUSED_QUERY/expected.csv b/backend/src/test/resources/tests/query/REUSED_QUERY/expected.csv index 43a95d3f06..0885ce7d18 100644 --- a/backend/src/test/resources/tests/query/REUSED_QUERY/expected.csv +++ b/backend/src/test/resources/tests/query/REUSED_QUERY/expected.csv @@ -1,12 +1,12 @@ result,dates -2,{2009-12-01/2009-12-01} -5,{2007-01-01/2007-01-01} -6,{2007-01-01/2007-01-01} -15,"{2012-01-01/2012-01-01, 2014-01-01/2014-01-01}" -27,{2014-01-01/2014-01-01} -31,{2006-01-01/2006-01-01} -32,{2014-01-01/2014-01-01} -40,{2012-04-01/2012-04-01} -58,{2010-01-01/2010-01-01} -64,{2014-01-01/2014-01-01} -68,{2006-01-01/2006-01-01} \ No newline at end of file +2,2009-12-01/2009-12-01 +5,2007-01-01/2007-01-01 +6,2007-01-01/2007-01-01 +15,"2012-01-01/2012-01-01, 2014-01-01/2014-01-01" +27,2014-01-01/2014-01-01 +31,2006-01-01/2006-01-01 +32,2014-01-01/2014-01-01 +40,2012-04-01/2012-04-01 +58,2010-01-01/2010-01-01 +64,2014-01-01/2014-01-01 +68,2006-01-01/2006-01-01 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/REUSED_QUERY/query_results_1.csv b/backend/src/test/resources/tests/query/REUSED_QUERY/query_results_1.csv index 98f151fc36..96f20ab474 100644 --- a/backend/src/test/resources/tests/query/REUSED_QUERY/query_results_1.csv +++ b/backend/src/test/resources/tests/query/REUSED_QUERY/query_results_1.csv @@ -1,12 +1,12 @@ "result","dates" -2,{2009-12-01/2009-12-01} -5,{2007-01-01/2007-01-01} -6,{2007-01-01/2007-01-01} -15,"{2012-01-01/2012-01-01, 2014-01-01/2014-01-01}" -27,{2014-01-01/2014-01-01} -31,{2006-01-01/2006-01-01} -32,{2014-01-01/2014-01-01} -40,{2012-04-01/2012-04-01} -58,{2010-01-01/2010-01-01} -64,{2014-01-01/2014-01-01} -68,{2006-01-01/2006-01-01} \ No newline at end of file +2,2009-12-01/2009-12-01 +5,2007-01-01/2007-01-01 +6,2007-01-01/2007-01-01 +15,"2012-01-01/2012-01-01, 2014-01-01/2014-01-01" +27,2014-01-01/2014-01-01 +31,2006-01-01/2006-01-01 +32,2014-01-01/2014-01-01 +40,2012-04-01/2012-04-01 +58,2010-01-01/2010-01-01 +64,2014-01-01/2014-01-01 +68,2006-01-01/2006-01-01 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/REUSED_QUERY/query_results_2.csv b/backend/src/test/resources/tests/query/REUSED_QUERY/query_results_2.csv index 94132d3386..835fb5472a 100644 --- a/backend/src/test/resources/tests/query/REUSED_QUERY/query_results_2.csv +++ b/backend/src/test/resources/tests/query/REUSED_QUERY/query_results_2.csv @@ -1,11 +1,11 @@ "result","dates" -9978,{2012-01-01/2012-01-01} -9977,{2010-01-01/2010-01-01} -9974,{2011-01-01/2011-01-01} -9969,{2007-01-01/2007-01-01} -9962,{2014-01-01/2014-01-01} -9944,{2008-08-01/2008-08-01} -9926,{2010-01-01/2010-01-01} -9896,{2013-04-01/2013-04-01} -9893,{2012-01-01/2012-01-01} -9886,{2011-01-01/2011-01-01} \ No newline at end of file +9978,2012-01-01/2012-01-01 +9977,2010-01-01/2010-01-01 +9974,2011-01-01/2011-01-01 +9969,2007-01-01/2007-01-01 +9962,2014-01-01/2014-01-01 +9944,2008-08-01/2008-08-01 +9926,2010-01-01/2010-01-01 +9896,2013-04-01/2013-04-01 +9893,2012-01-01/2012-01-01 +9886,2011-01-01/2011-01-01 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/SECONDARY_ID/expected.csv b/backend/src/test/resources/tests/query/SECONDARY_ID/expected.csv index 947b1d8944..b2be405b8a 100644 --- a/backend/src/test/resources/tests/query/SECONDARY_ID/expected.csv +++ b/backend/src/test/resources/tests/query/SECONDARY_ID/expected.csv @@ -1,6 +1,6 @@ result,secondary,dates -a,f_a1,"{2014-06-30/2015-06-30, 2016-06-30/2016-06-30}" -a,f_a2,{2014-06-30/2015-06-30} -b,f_b1,{2015-02-03/2015-06-30} -a,f_a4,"{2024-06-30/2025-06-30, 2026-06-30/2026-06-30}" -b,f_b6,{2025-02-03/2025-06-30} \ No newline at end of file +a,f_a1,"2014-06-30/2015-06-30, 2016-06-30/2016-06-30" +a,f_a2,2014-06-30/2015-06-30 +b,f_b1,2015-02-03/2015-06-30 +a,f_a4,"2024-06-30/2025-06-30, 2026-06-30/2026-06-30" +b,f_b6,2025-02-03/2025-06-30 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/SECONDARY_ID_DATEMODE_LOGICAL/expected.csv b/backend/src/test/resources/tests/query/SECONDARY_ID_DATEMODE_LOGICAL/expected.csv index ee43efebd4..60f5b83fc6 100644 --- a/backend/src/test/resources/tests/query/SECONDARY_ID_DATEMODE_LOGICAL/expected.csv +++ b/backend/src/test/resources/tests/query/SECONDARY_ID_DATEMODE_LOGICAL/expected.csv @@ -1,3 +1,3 @@ result,secondary,dates -a,f_a1,"{2024-06-30/2025-06-30, 2026-06-30/2026-06-30}" -a,f_a2,{} \ No newline at end of file +a,f_a1,"2024-06-30/2025-06-30, 2026-06-30/2026-06-30" +a,f_a2, \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/SECONDARY_ID_EXCLUDED/expected.csv b/backend/src/test/resources/tests/query/SECONDARY_ID_EXCLUDED/expected.csv index ff053bd400..24a7b15402 100644 --- a/backend/src/test/resources/tests/query/SECONDARY_ID_EXCLUDED/expected.csv +++ b/backend/src/test/resources/tests/query/SECONDARY_ID_EXCLUDED/expected.csv @@ -1,6 +1,6 @@ result,secondary,dates -p0,f0,{2020-01-01/2020-01-01} -p0,f1,{2020-01-01/2020-01-01} -p1,f1,{2020-01-01/2020-01-01} -p2,f0,{2020-01-01/2020-01-01} -p2,f1,{2020-01-01/2020-01-01} \ No newline at end of file +p0,f0,2020-01-01/2020-01-01 +p0,f1,2020-01-01/2020-01-01 +p1,f1,2020-01-01/2020-01-01 +p2,f0,2020-01-01/2020-01-01 +p2,f1,2020-01-01/2020-01-01 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/SECONDARY_ID_MIXED/expected.csv b/backend/src/test/resources/tests/query/SECONDARY_ID_MIXED/expected.csv index 4ca0b2a7f5..27b854b94b 100644 --- a/backend/src/test/resources/tests/query/SECONDARY_ID_MIXED/expected.csv +++ b/backend/src/test/resources/tests/query/SECONDARY_ID_MIXED/expected.csv @@ -1,5 +1,5 @@ result,secondary,dates -a,f_a3,"{2024-06-30/2025-06-30, 2026-06-30/2026-06-30}" -a,f_a2,"{2014-06-30/2015-06-30, 2024-06-30/2025-06-30, 2026-06-30/2026-06-30}" -a,f_a1,"{2014-06-30/2015-06-30, 2016-06-30/2016-06-30, 2024-06-30/2025-06-30, 2026-06-30/2026-06-30}" -b,f_b1,"{2015-02-03/2015-06-30, 2025-02-03/2025-06-30}" \ No newline at end of file +a,f_a3,"2024-06-30/2025-06-30, 2026-06-30/2026-06-30" +a,f_a2,"2014-06-30/2015-06-30, 2024-06-30/2025-06-30, 2026-06-30/2026-06-30" +a,f_a1,"2014-06-30/2015-06-30, 2016-06-30/2016-06-30, 2024-06-30/2025-06-30, 2026-06-30/2026-06-30" +b,f_b1,"2015-02-03/2015-06-30, 2025-02-03/2025-06-30" \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/SIMPLE_CQEXTERNAL_QUERY/expected.csv b/backend/src/test/resources/tests/query/SIMPLE_CQEXTERNAL_QUERY/expected.csv index f0f34f6300..443c6a4fd1 100644 --- a/backend/src/test/resources/tests/query/SIMPLE_CQEXTERNAL_QUERY/expected.csv +++ b/backend/src/test/resources/tests/query/SIMPLE_CQEXTERNAL_QUERY/expected.csv @@ -1,3 +1,3 @@ result,dates -1,{2012-01-01/2012-01-01} -3,{2013-11-10/2013-11-10} \ No newline at end of file +1,2012-01-01/2012-01-01 +3,2013-11-10/2013-11-10 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/SIMPLE_CQEXTERNAL_QUERY/expected_start_end.csv b/backend/src/test/resources/tests/query/SIMPLE_CQEXTERNAL_QUERY/expected_start_end.csv index b90197cfad..5695f5be28 100644 --- a/backend/src/test/resources/tests/query/SIMPLE_CQEXTERNAL_QUERY/expected_start_end.csv +++ b/backend/src/test/resources/tests/query/SIMPLE_CQEXTERNAL_QUERY/expected_start_end.csv @@ -1,3 +1,3 @@ result,dates -1,{2012-01-01/2012-01-10} -3,{2013-11-10/2013-11-20} \ No newline at end of file +1,2012-01-01/2012-01-10 +3,2013-11-10/2013-11-20 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/SIMPLE_TREECONCEPT_CONNECTOR_CONDITION_WITH_CHILDREN_QUERY/expected.csv b/backend/src/test/resources/tests/query/SIMPLE_TREECONCEPT_CONNECTOR_CONDITION_WITH_CHILDREN_QUERY/expected.csv index 8114ea686d..ce93de1599 100644 --- a/backend/src/test/resources/tests/query/SIMPLE_TREECONCEPT_CONNECTOR_CONDITION_WITH_CHILDREN_QUERY/expected.csv +++ b/backend/src/test/resources/tests/query/SIMPLE_TREECONCEPT_CONNECTOR_CONDITION_WITH_CHILDREN_QUERY/expected.csv @@ -1,3 +1,3 @@ result,dates -1,{2012-01-01/2012-01-01} -3,{2013-11-10/2013-11-10} +1,2012-01-01/2012-01-01 +3,2013-11-10/2013-11-10 diff --git a/backend/src/test/resources/tests/query/SIMPLE_TREECONCEPT_GROOVY_QUERY/expected.csv b/backend/src/test/resources/tests/query/SIMPLE_TREECONCEPT_GROOVY_QUERY/expected.csv index 864ec80505..8b14139022 100644 --- a/backend/src/test/resources/tests/query/SIMPLE_TREECONCEPT_GROOVY_QUERY/expected.csv +++ b/backend/src/test/resources/tests/query/SIMPLE_TREECONCEPT_GROOVY_QUERY/expected.csv @@ -1,2 +1,2 @@ result,dates -3,{2013-11-10/2013-11-10} \ No newline at end of file +3,2013-11-10/2013-11-10 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/SIMPLE_TREECONCEPT_PRESENT_CONDITION_QUERY/expected.csv b/backend/src/test/resources/tests/query/SIMPLE_TREECONCEPT_PRESENT_CONDITION_QUERY/expected.csv index f0f34f6300..443c6a4fd1 100644 --- a/backend/src/test/resources/tests/query/SIMPLE_TREECONCEPT_PRESENT_CONDITION_QUERY/expected.csv +++ b/backend/src/test/resources/tests/query/SIMPLE_TREECONCEPT_PRESENT_CONDITION_QUERY/expected.csv @@ -1,3 +1,3 @@ result,dates -1,{2012-01-01/2012-01-01} -3,{2013-11-10/2013-11-10} \ No newline at end of file +1,2012-01-01/2012-01-01 +3,2013-11-10/2013-11-10 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/SIMPLE_TREECONCEPT_QUERY/expected.csv b/backend/src/test/resources/tests/query/SIMPLE_TREECONCEPT_QUERY/expected.csv index f0f34f6300..443c6a4fd1 100644 --- a/backend/src/test/resources/tests/query/SIMPLE_TREECONCEPT_QUERY/expected.csv +++ b/backend/src/test/resources/tests/query/SIMPLE_TREECONCEPT_QUERY/expected.csv @@ -1,3 +1,3 @@ result,dates -1,{2012-01-01/2012-01-01} -3,{2013-11-10/2013-11-10} \ No newline at end of file +1,2012-01-01/2012-01-01 +3,2013-11-10/2013-11-10 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/SIMPLE_TREECONCEPT_QUERY_SPECIAL_CHAR_DATA/expected.csv b/backend/src/test/resources/tests/query/SIMPLE_TREECONCEPT_QUERY_SPECIAL_CHAR_DATA/expected.csv index f0f34f6300..443c6a4fd1 100644 --- a/backend/src/test/resources/tests/query/SIMPLE_TREECONCEPT_QUERY_SPECIAL_CHAR_DATA/expected.csv +++ b/backend/src/test/resources/tests/query/SIMPLE_TREECONCEPT_QUERY_SPECIAL_CHAR_DATA/expected.csv @@ -1,3 +1,3 @@ result,dates -1,{2012-01-01/2012-01-01} -3,{2013-11-10/2013-11-10} \ No newline at end of file +1,2012-01-01/2012-01-01 +3,2013-11-10/2013-11-10 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/SIMPLE_VIRTUAL_CONCEPT_QUERY/expected.csv b/backend/src/test/resources/tests/query/SIMPLE_VIRTUAL_CONCEPT_QUERY/expected.csv index f0f34f6300..443c6a4fd1 100644 --- a/backend/src/test/resources/tests/query/SIMPLE_VIRTUAL_CONCEPT_QUERY/expected.csv +++ b/backend/src/test/resources/tests/query/SIMPLE_VIRTUAL_CONCEPT_QUERY/expected.csv @@ -1,3 +1,3 @@ result,dates -1,{2012-01-01/2012-01-01} -3,{2013-11-10/2013-11-10} \ No newline at end of file +1,2012-01-01/2012-01-01 +3,2013-11-10/2013-11-10 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/SINGLE_SELECT_DATE_RESTRICTION_OR_CONCEPT_QUERY/expected.csv b/backend/src/test/resources/tests/query/SINGLE_SELECT_DATE_RESTRICTION_OR_CONCEPT_QUERY/expected.csv index 587b3269ee..225e384fcd 100644 --- a/backend/src/test/resources/tests/query/SINGLE_SELECT_DATE_RESTRICTION_OR_CONCEPT_QUERY/expected.csv +++ b/backend/src/test/resources/tests/query/SINGLE_SELECT_DATE_RESTRICTION_OR_CONCEPT_QUERY/expected.csv @@ -1,2 +1,2 @@ result,dates -1,{2012-01-01/2012-01-01} \ No newline at end of file +1,2012-01-01/2012-01-01 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/SINGLE_SELECT_NEGATION_DATE_RESTRICTION_OR_CONCEPT_QUERY/expected.csv b/backend/src/test/resources/tests/query/SINGLE_SELECT_NEGATION_DATE_RESTRICTION_OR_CONCEPT_QUERY/expected.csv index fed99294c1..8ae78128e7 100644 --- a/backend/src/test/resources/tests/query/SINGLE_SELECT_NEGATION_DATE_RESTRICTION_OR_CONCEPT_QUERY/expected.csv +++ b/backend/src/test/resources/tests/query/SINGLE_SELECT_NEGATION_DATE_RESTRICTION_OR_CONCEPT_QUERY/expected.csv @@ -1,8 +1,8 @@ result,dates -2,{} -3,{} -4,{} -5,{} -6,{} -7,{} -8,{} \ No newline at end of file +2, +3, +4, +5, +6, +7, +8, \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/SUM_EMPTY_DATE_CONCEPT_QUERY/expected.csv b/backend/src/test/resources/tests/query/SUM_EMPTY_DATE_CONCEPT_QUERY/expected.csv index be5fb4effe..4d50fe2b6b 100644 --- a/backend/src/test/resources/tests/query/SUM_EMPTY_DATE_CONCEPT_QUERY/expected.csv +++ b/backend/src/test/resources/tests/query/SUM_EMPTY_DATE_CONCEPT_QUERY/expected.csv @@ -1,3 +1,3 @@ result,dates -4,"{2010-01-02/2010-01-02, 2010-01-04/2010-01-04}" -44,"{2010-01-02/2010-01-02, 2010-01-04/2010-01-04}" \ No newline at end of file +4,"2010-01-02/2010-01-02, 2010-01-04/2010-01-04" +44,"2010-01-02/2010-01-02, 2010-01-04/2010-01-04" \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/TABLE_EXPORT/expected.csv b/backend/src/test/resources/tests/query/TABLE_EXPORT/expected.csv index 831f8f2ad3..6de95c07a7 100644 --- a/backend/src/test/resources/tests/query/TABLE_EXPORT/expected.csv +++ b/backend/src/test/resources/tests/query/TABLE_EXPORT/expected.csv @@ -1,8 +1,8 @@ result,dates,SecondaryId,table1 - value,table2 - value -a,{2014-06-30/2015-06-30},f_a1,1.0, -a,{2016-06-30/2016-06-30},f_a1,1.0, -a,{2014-06-30/2015-06-30},f_a2,1.0, -a,{2010-06-30/2010-06-30},,1.0, -a,{2014-06-30/2015-06-30},f_a3,1.01, -a,{2020-06-30/2020-06-30},,,13.0 -b,{2015-02-03/2015-06-30},f_b1,1.0, \ No newline at end of file +a,2014-06-30/2015-06-30,f_a1,1.0, +a,2016-06-30/2016-06-30,f_a1,1.0, +a,2014-06-30/2015-06-30,f_a2,1.0, +a,2010-06-30/2010-06-30,,1.0, +a,2014-06-30/2015-06-30,f_a3,1.01, +a,2020-06-30/2020-06-30,,,13.0 +b,2015-02-03/2015-06-30,f_b1,1.0, \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/TEMPORAL_BEFORE_CONCEPT_QUERY/expected.csv b/backend/src/test/resources/tests/query/TEMPORAL_BEFORE_CONCEPT_QUERY/expected.csv index 28de71d660..40acdeebbf 100644 --- a/backend/src/test/resources/tests/query/TEMPORAL_BEFORE_CONCEPT_QUERY/expected.csv +++ b/backend/src/test/resources/tests/query/TEMPORAL_BEFORE_CONCEPT_QUERY/expected.csv @@ -1,2 +1,2 @@ result,dates -4,{2011-01-06/2011-01-06} \ No newline at end of file +4,2011-01-06/2011-01-06 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/TEMPORAL_BEFORE_OR_SAME_CONCEPT_QUERY/expected.csv b/backend/src/test/resources/tests/query/TEMPORAL_BEFORE_OR_SAME_CONCEPT_QUERY/expected.csv index 2370e8637a..714702dca1 100644 --- a/backend/src/test/resources/tests/query/TEMPORAL_BEFORE_OR_SAME_CONCEPT_QUERY/expected.csv +++ b/backend/src/test/resources/tests/query/TEMPORAL_BEFORE_OR_SAME_CONCEPT_QUERY/expected.csv @@ -1,4 +1,4 @@ result,dates -1,{2011-01-02/2011-01-02} -3,{2011-01-01/2011-01-01} -5,"{2011-01-01/2011-01-01, 2011-02-02/2011-02-02}" \ No newline at end of file +1,2011-01-02/2011-01-02 +3,2011-01-01/2011-01-01 +5,"2011-01-01/2011-01-01, 2011-02-02/2011-02-02" \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/TEMPORAL_DAYS_BEFORE_CONCEPT_QUERY/expected.csv b/backend/src/test/resources/tests/query/TEMPORAL_DAYS_BEFORE_CONCEPT_QUERY/expected.csv index 28de71d660..40acdeebbf 100644 --- a/backend/src/test/resources/tests/query/TEMPORAL_DAYS_BEFORE_CONCEPT_QUERY/expected.csv +++ b/backend/src/test/resources/tests/query/TEMPORAL_DAYS_BEFORE_CONCEPT_QUERY/expected.csv @@ -1,2 +1,2 @@ result,dates -4,{2011-01-06/2011-01-06} \ No newline at end of file +4,2011-01-06/2011-01-06 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/TEMPORAL_DAYS_BEFORE_OR_NEVER_CONCEPT_QUERY/expected.csv b/backend/src/test/resources/tests/query/TEMPORAL_DAYS_BEFORE_OR_NEVER_CONCEPT_QUERY/expected.csv index 151df19e75..fc3fb6fe0f 100644 --- a/backend/src/test/resources/tests/query/TEMPORAL_DAYS_BEFORE_OR_NEVER_CONCEPT_QUERY/expected.csv +++ b/backend/src/test/resources/tests/query/TEMPORAL_DAYS_BEFORE_OR_NEVER_CONCEPT_QUERY/expected.csv @@ -1,3 +1,3 @@ result,dates -1,{2012-01-01/2012-01-01} -3,{2012-11-12/2012-11-12} \ No newline at end of file +1,2012-01-01/2012-01-01 +3,2012-11-12/2012-11-12 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/TEMPORAL_SAME_CONCEPT_QUERY/expected.csv b/backend/src/test/resources/tests/query/TEMPORAL_SAME_CONCEPT_QUERY/expected.csv index 6370f509d9..145fcd42b1 100644 --- a/backend/src/test/resources/tests/query/TEMPORAL_SAME_CONCEPT_QUERY/expected.csv +++ b/backend/src/test/resources/tests/query/TEMPORAL_SAME_CONCEPT_QUERY/expected.csv @@ -1,3 +1,3 @@ result,dates -1,{2011-01-01/2011-01-01} -3,{2011-01-01/2011-01-01} +1,2011-01-01/2011-01-01 +3,2011-01-01/2011-01-01 diff --git a/backend/src/test/resources/tests/query/UPDATE_CONCEPT_TESTS/expected.csv b/backend/src/test/resources/tests/query/UPDATE_CONCEPT_TESTS/expected.csv index d5d49c43e6..ede1879bc5 100644 --- a/backend/src/test/resources/tests/query/UPDATE_CONCEPT_TESTS/expected.csv +++ b/backend/src/test/resources/tests/query/UPDATE_CONCEPT_TESTS/expected.csv @@ -1,4 +1,4 @@ result,dates -1,{2012-01-01/2012-01-01} -3,{2013-11-10/2013-11-10} -4,{2013-11-10/2013-11-10} \ No newline at end of file +1,2012-01-01/2012-01-01 +3,2013-11-10/2013-11-10 +4,2013-11-10/2013-11-10 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/VALIDITY_DATE_QUERY/expected.csv b/backend/src/test/resources/tests/query/VALIDITY_DATE_QUERY/expected.csv index 9eda2dbd9c..484f5086e5 100644 --- a/backend/src/test/resources/tests/query/VALIDITY_DATE_QUERY/expected.csv +++ b/backend/src/test/resources/tests/query/VALIDITY_DATE_QUERY/expected.csv @@ -1,3 +1,3 @@ result,dates -1,{2013-01-01/2013-01-01} -3,{2014-11-10/2014-11-10} \ No newline at end of file +1,2013-01-01/2013-01-01 +3,2014-11-10/2014-11-10 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/VIRTUAL_CONCEPT_REUSED_QUERY/expected.csv b/backend/src/test/resources/tests/query/VIRTUAL_CONCEPT_REUSED_QUERY/expected.csv index 078241b18e..7b493b16cc 100644 --- a/backend/src/test/resources/tests/query/VIRTUAL_CONCEPT_REUSED_QUERY/expected.csv +++ b/backend/src/test/resources/tests/query/VIRTUAL_CONCEPT_REUSED_QUERY/expected.csv @@ -1,5 +1,5 @@ result,dates -5,{2007-01-01/2007-01-01} -19,{2012-01-01/2012-01-01} -32,{2014-01-01/2014-01-01} -64,{2014-01-01/2014-01-01} \ No newline at end of file +5,2007-01-01/2007-01-01 +19,2012-01-01/2012-01-01 +32,2014-01-01/2014-01-01 +64,2014-01-01/2014-01-01 \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/VIRTUAL_CONCEPT_REUSED_QUERY/query_results.csv b/backend/src/test/resources/tests/query/VIRTUAL_CONCEPT_REUSED_QUERY/query_results.csv index bbacbe5742..aab90cac40 100644 --- a/backend/src/test/resources/tests/query/VIRTUAL_CONCEPT_REUSED_QUERY/query_results.csv +++ b/backend/src/test/resources/tests/query/VIRTUAL_CONCEPT_REUSED_QUERY/query_results.csv @@ -1,5 +1,5 @@ result,"dates" -5,{2007-01-01/2007-01-01} -19,{2012-01-01/2012-01-01} -32,{2014-01-01/2014-01-01} -64,{2014-01-01/2014-01-01} \ No newline at end of file +5,2007-01-01/2007-01-01 +19,2012-01-01/2012-01-01 +32,2014-01-01/2014-01-01 +64,2014-01-01/2014-01-01 \ No newline at end of file From 634fabebad7bedf1e26ef5672af77a08883d36b2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 11 Aug 2021 16:40:19 +0000 Subject: [PATCH 72/82] Update AutoDoc --- docs/Config JSON.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/Config JSON.md b/docs/Config JSON.md index 5e50de37cf..1ad599e9ae 100644 --- a/docs/Config JSON.md +++ b/docs/Config JSON.md @@ -256,7 +256,7 @@ Supported Fields: | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L50) | version | `String` | `"0.0.0-SNAPSHOT"` | | |

-### Type LocaleConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L24) +### Type LocaleConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L16)
Details

@@ -267,11 +267,11 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L29) | dateFormatMapping | map from `Locale` to `String` | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L33) | dateParsingFormats | `@NotNull Set` | `["yyyy-MM-dd","yyyyMMdd","dd.MM.yyyy"]` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L43) | dateSetLayouts | list of `DateSetLayout` | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L26) | frontend | `@NotNull Locale` | `""` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L40) | rangeStartEndSeperators | list of `String` | `["-","/"]` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L21) | dateFormatMapping | map from `Locale` to `String` | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L25) | dateParsingFormats | `@NotNull Set` | `["dd.MM.yyyy","yyyy-MM-dd","yyyyMMdd"]` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L18) | frontend | `@NotNull Locale` | `""` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L35) | listFormats | list of `ListFormat` | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L32) | rangeStartEndSeperators | map from `Locale` to `String` | | | |

### Type MinaConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/MinaConfig.java#L13) From 1ad2f61a194ea3ed19dbcb1f5dbbc0f7cce11e0e Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Thu, 12 Aug 2021 14:03:45 +0200 Subject: [PATCH 73/82] adds missing date range separator to fix all tests --- .../conquery/models/config/LocaleConfig.java | 22 ++++++++++++++----- .../MULTI_SELECT_AGGREGATOR/expected.csv | 10 ++++----- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java index 26f3a25ab7..2d5b4daac5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java @@ -19,18 +19,26 @@ public class LocaleConfig { private Locale frontend = Locale.ROOT; @NotNull - private Map dateFormatMapping = Map.of(Locale.GERMAN, "dd.MM.yyyy"); + private Map dateFormatMapping = Map.of( + Locale.GERMAN, "dd.MM.yyyy", + Locale.ROOT, "yyyy-MM-dd" + ); @NotNull - private Set dateParsingFormats = Set.of( + private Set parsingDateFormats = Set.of( "yyyy-MM-dd", "yyyyMMdd", "dd.MM.yyyy" ); @NotEmpty - private Map rangeStartEndSeperators = Map.of(Locale.GERMAN, "-"); + private Map localeRangeStartEndSeperators = Map.of( + Locale.GERMAN, "-", + Locale.ROOT, "/" + ); + + private Set parsingRangeStartEndSeperators = Set.of("/"); @NotNull @NotEmpty @@ -55,9 +63,11 @@ public static class ListFormat { */ @JsonIgnore public DateReader getDateReader() { + final ArrayList rangeStartEndSeperators = new ArrayList<>(localeRangeStartEndSeperators.values()); + rangeStartEndSeperators.addAll(parsingRangeStartEndSeperators); return new DateReader( - Sets.union(dateParsingFormats, Set.copyOf(dateFormatMapping.values())), - new ArrayList<>(rangeStartEndSeperators.values()), + Sets.union(parsingDateFormats, Set.copyOf(dateFormatMapping.values())), + rangeStartEndSeperators, listFormats ); } @@ -67,7 +77,7 @@ public DateReader getDateReader() { * If there is no perfect match, the locale is abstracted, see findClosestMatch. */ public String findDateRangeSeparator(Locale locale) { - final String closestMatch = findClosestMatch(locale, rangeStartEndSeperators); + final String closestMatch = findClosestMatch(locale, localeRangeStartEndSeperators); return closestMatch != null ? closestMatch : "/"; } diff --git a/backend/src/test/resources/tests/aggregator/MULTI_SELECT_AGGREGATOR/expected.csv b/backend/src/test/resources/tests/aggregator/MULTI_SELECT_AGGREGATOR/expected.csv index c08081f3a7..218231cfac 100644 --- a/backend/src/test/resources/tests/aggregator/MULTI_SELECT_AGGREGATOR/expected.csv +++ b/backend/src/test/resources/tests/aggregator/MULTI_SELECT_AGGREGATOR/expected.csv @@ -1,7 +1,7 @@ result,dates,concept - select 2,2010-07-15/2010-07-15, -3,2013-11-10/2013-11-10,f=1 -6,2012-11-11/2012-11-11,"f=1, m=1" -1,2012-01-01/2012-01-02,f=2 -4,2012-11-11/2012-11-11,m=1 -5,2012-11-11/2012-11-11,f=1 \ No newline at end of file +3,2013-11-10/2013-11-10,{f=1} +6,2012-11-11/2012-11-11,"{f=1, m=1}" +1,2012-01-01/2012-01-02,{f=2} +4,2012-11-11/2012-11-11,{m=1} +5,2012-11-11/2012-11-11,{f=1} \ No newline at end of file From deaa33cd611045898054b9b8a8a11c36315f3243 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 12 Aug 2021 12:33:55 +0000 Subject: [PATCH 74/82] Update AutoDoc --- docs/Config JSON.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/Config JSON.md b/docs/Config JSON.md index 1ad599e9ae..4e6909793c 100644 --- a/docs/Config JSON.md +++ b/docs/Config JSON.md @@ -268,10 +268,11 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L21) | dateFormatMapping | map from `Locale` to `String` | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L25) | dateParsingFormats | `@NotNull Set` | `["dd.MM.yyyy","yyyy-MM-dd","yyyyMMdd"]` | | | | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L18) | frontend | `@NotNull Locale` | `""` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L35) | listFormats | list of `ListFormat` | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L32) | rangeStartEndSeperators | map from `Locale` to `String` | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L43) | listFormats | list of `ListFormat` | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L35) | localeRangeStartEndSeperators | map from `Locale` to `String` | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L28) | parsingDateFormats | `@NotNull Set` | `["dd.MM.yyyy","yyyyMMdd","yyyy-MM-dd"]` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L41) | parsingRangeStartEndSeperators | `Set` | `["/"]` | | |

### Type MinaConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/MinaConfig.java#L13) From 67ebd1d2b74fd0eeb6413ed9ce5b91ac98c434c5 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Thu, 12 Aug 2021 16:14:03 +0200 Subject: [PATCH 75/82] cleanup Locale config --- .../conquery/models/config/LocaleConfig.java | 85 +++++++++++++------ 1 file changed, 57 insertions(+), 28 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java index 2d5b4daac5..c0b9c0b76e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java @@ -18,6 +18,11 @@ public class LocaleConfig { @NotNull private Locale frontend = Locale.ROOT; + /** + * Mappings from user provided locale to date format which is used in the generation of result tables. + * The formats are also available for parsing dates using the {@link DateReader}. However, the locale is + * neglected there and the formats are tried until one that fits is found. + */ @NotNull private Map dateFormatMapping = Map.of( Locale.GERMAN, "dd.MM.yyyy", @@ -25,27 +30,43 @@ public class LocaleConfig { ); + /** + * Additional date formats that are available only for parsing. + */ @NotNull private Set parsingDateFormats = Set.of( - "yyyy-MM-dd", - "yyyyMMdd", - "dd.MM.yyyy" + "yyyyMMdd" ); + /** + * Mappings from user provided locale to date range format which is used in the generation of result tables. + * The formats are also available for parsing dates ranges using the {@link DateReader}. However, the locale is + * neglected there and the formats are tried until one that fits is found. + */ @NotEmpty - private Map localeRangeStartEndSeperators = Map.of( + private Map localeRangeStartEndSeparators = Map.of( Locale.GERMAN, "-", Locale.ROOT, "/" ); - private Set parsingRangeStartEndSeperators = Set.of("/"); + /** + * Additional date range formats that are available only for parsing. + */ + private Set parsingRangeStartEndSeparators = Set.of("/"); + /** + * List formats that are available for parsing inputs and (the first one) for rendering result tables. + */ @NotNull @NotEmpty private List listFormats = List.of( new ListFormat("", ", ", ""), - new ListFormat("{", ", ", "}")); + new ListFormat("{", ", ", "}"), + new ListFormat("[", ", ", "]")); + /** + * Container to describe the format of a list + */ @Data @AllArgsConstructor public static class ListFormat { @@ -63,8 +84,8 @@ public static class ListFormat { */ @JsonIgnore public DateReader getDateReader() { - final ArrayList rangeStartEndSeperators = new ArrayList<>(localeRangeStartEndSeperators.values()); - rangeStartEndSeperators.addAll(parsingRangeStartEndSeperators); + final ArrayList rangeStartEndSeperators = new ArrayList<>(localeRangeStartEndSeparators.values()); + rangeStartEndSeperators.addAll(parsingRangeStartEndSeparators); return new DateReader( Sets.union(parsingDateFormats, Set.copyOf(dateFormatMapping.values())), rangeStartEndSeperators, @@ -77,8 +98,7 @@ public DateReader getDateReader() { * If there is no perfect match, the locale is abstracted, see findClosestMatch. */ public String findDateRangeSeparator(Locale locale) { - final String closestMatch = findClosestMatch(locale, localeRangeStartEndSeperators); - return closestMatch != null ? closestMatch : "/"; + return findClosestMatch(locale, localeRangeStartEndSeparators, "/"); } /** @@ -86,34 +106,43 @@ public String findDateRangeSeparator(Locale locale) { * If there is no perfect match, the locale is abstracted, see findClosestMatch. */ public DateTimeFormatter findDateTimeFormater(Locale locale) { - final String closestMatch = findClosestMatch(locale, dateFormatMapping); - return closestMatch != null ? DateTimeFormatter.ofPattern(closestMatch) : DateTimeFormatter.ISO_DATE; + return DateTimeFormatter.ofPattern(findClosestMatch(locale, dateFormatMapping, "yyyy-MM-dd")); } /** - * Adapted from {@link c10n.share.DefaultLocaleMapping} + * Helper method to find the best match for a given locale using its abstractions. + * First the vanilla locale is checked, then abstractions to country and language. + * The last resort is the {@link Locale#ROOT}. If no match is found, the alternative is returned. + * + * @param forLocale + * @param options + * @param + * @return */ - private static T findClosestMatch(Locale forLocale, Map options) { - Set fromSet = options.keySet(); - String variant = forLocale.getDisplayVariant(); + private static T findClosestMatch(Locale forLocale, Map options, T alternative) { String country = forLocale.getCountry(); String language = forLocale.getLanguage(); - List c = new ArrayList<>(4); - if (null != variant && !variant.isEmpty()) { - c.add(forLocale); + + if (options.containsKey(forLocale)){ + return options.get(forLocale); } - if (null != country && !country.isEmpty()) { - c.add(new Locale(language, country)); + + Locale abstraction = new Locale(language, country); + if (!country.isEmpty() && options.containsKey(abstraction)) { + return options.get(abstraction); } - if (null != language && !language.isEmpty()) { - c.add(new Locale(language)); + + abstraction = new Locale(language); + if (!language.isEmpty() && options.containsKey(abstraction)) { + return options.get(abstraction); } - for (Locale candidateLocale : c) { - if (fromSet.contains(candidateLocale)) { - return options.get(candidateLocale); - } + + abstraction = Locale.ROOT; + if (options.containsKey(abstraction)) { + return options.get(abstraction); } - return null; + + return alternative; } From 910d85913d598fca7d53bc3878256f680a1810d2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 12 Aug 2021 14:15:52 +0000 Subject: [PATCH 76/82] Update AutoDoc --- docs/Config JSON.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/Config JSON.md b/docs/Config JSON.md index 4e6909793c..2c67e45893 100644 --- a/docs/Config JSON.md +++ b/docs/Config JSON.md @@ -267,12 +267,12 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L21) | dateFormatMapping | map from `Locale` to `String` | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L21-L25) | dateFormatMapping | map from `Locale` to `String` | | | Mappings from user provided locale to date format which is used in the generation of result tables. The formats are also available for parsing dates using the {@link DateReader}. However, the locale is neglected there and the formats are tried until one that fits is found. | | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L18) | frontend | `@NotNull Locale` | `""` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L43) | listFormats | list of `ListFormat` | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L35) | localeRangeStartEndSeperators | map from `Locale` to `String` | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L28) | parsingDateFormats | `@NotNull Set` | `["dd.MM.yyyy","yyyyMMdd","yyyy-MM-dd"]` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L41) | parsingRangeStartEndSeperators | `Set` | `["/"]` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L57-L59) | listFormats | list of `ListFormat` | | | List formats that are available for parsing inputs and (the first one) for rendering result tables. | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L41-L45) | localeRangeStartEndSeparators | map from `Locale` to `String` | | | Mappings from user provided locale to date range format which is used in the generation of result tables. The formats are also available for parsing dates ranges using the {@link DateReader}. However, the locale is neglected there and the formats are tried until one that fits is found. | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L33-L35) | parsingDateFormats | `@NotNull Set` | `["yyyyMMdd"]` | | Additional date formats that are available only for parsing. | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L52-L54) | parsingRangeStartEndSeparators | `Set` | `["/"]` | | Additional date range formats that are available only for parsing. |

### Type MinaConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/MinaConfig.java#L13) From f49972f1c6dcb606e1174c037d02255785f29f4b Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 17 Aug 2021 12:18:25 +0200 Subject: [PATCH 77/82] review changes --- .../conquery/models/common/CDateSet.java | 2 -- .../com/bakdata/conquery/util/DateReader.java | 21 ++++++++++++------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/common/CDateSet.java b/backend/src/main/java/com/bakdata/conquery/models/common/CDateSet.java index 257b3725be..9328a29c2b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/common/CDateSet.java +++ b/backend/src/main/java/com/bakdata/conquery/models/common/CDateSet.java @@ -30,8 +30,6 @@ @EqualsAndHashCode public class CDateSet { - private static final String PARSE_PATTERN_TEMPLATE = "(\\%1$c|%2$c\\s*)(([^%1$c%2$c%3$c%4$c]*+)/([^^%1$c%2$c%3$c%4$c]*+))"; // set_begin, range_sep, range_start_end_sep, set_end - private static final Pattern PARSE_PATTERN = Pattern.compile("(\\{|,\\s*)(([^/{}]*+)/([^/{}]*+))"); private final NavigableMap rangesByLowerBound; private transient Set asRanges; private transient Set asDescendingSetOfRanges; diff --git a/backend/src/main/java/com/bakdata/conquery/util/DateReader.java b/backend/src/main/java/com/bakdata/conquery/util/DateReader.java index 1dd2cf4474..9a16eafb1d 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/DateReader.java +++ b/backend/src/main/java/com/bakdata/conquery/util/DateReader.java @@ -107,24 +107,24 @@ public CDateRange parseToCDateRange(String value) { try{ return parseToCDateRange(value,lastSep); } catch (ParsingException e) { - log.trace("Parsing with last used config failed for date range: " + value, e); + log.trace("Parsing with last used config failed for date range: {}", value, e); } } for(String sep : rangeStartEndSeperators) { try { result = parseToCDateRange(value,sep); + lastRangeFormat.set(sep); } catch (ParsingException e) { - log.trace("Parsing failed for date range: " + value, e); + log.trace("Parsing failed for date range: {}", value, e); continue; } - lastRangeFormat.set(sep); break; } if (result != null) { return result; } - throw new ParsingException("Non of the configured formats allowed to parse the date range: " + value); + throw new ParsingException("None of the configured formats allowed to parse the date range: " + value); } private CDateRange parseToCDateRange(String value, String sep) { @@ -187,10 +187,17 @@ private static Pattern generateDateSetPattern(@NonNull String setBegin, @NonNull assert(rangeSep.length() >= 1); assert(setEnd.length() < 2); + /* + Create a matcher pattern, that captures the date ranges in group 1 (the only group that is captured and which + is not allowed to hold any of the set-delimiters) + + Groups starting with "?:" are not captured. The format parameters are reused in the format string by positional + reference "%X$s" where X refers to the position of the argument in String.format(...). + */ return Pattern.compile(String.format("(?:(?:%1$s)|(?:%2$s\\s*))([^%1$s%2$s%3$s]+)(?:%3$s)?", - setBegin.isEmpty() ? "" : Pattern.quote(setBegin), - Pattern.quote(rangeSep), - setEnd.isEmpty() ? "" : Pattern.quote(setEnd))); + setBegin.isEmpty() ? "" : Pattern.quote(setBegin), // referenced as: %1$s + Pattern.quote(rangeSep), // referenced as: %2$s + setEnd.isEmpty() ? "" : Pattern.quote(setEnd))); // referenced as: %3$s } /** From 81c43997f994c9092349a81613e010fa9c843251 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 17 Aug 2021 13:01:36 +0200 Subject: [PATCH 78/82] localize excel date rendering --- .../conquery/io/result/excel/ExcelRenderer.java | 7 ++++--- .../io/result/excel/ResultExcelProcessor.java | 3 +-- .../bakdata/conquery/models/config/ExcelConfig.java | 10 +++++++--- .../conquery/models/config/LocaleConfig.java | 13 +++++-------- .../conquery/models/externalservice/ResultType.java | 13 +++++-------- .../conquery/models/query/PrintSettings.java | 7 +++++-- .../io/result/excel/ExcelResultRenderTest.java | 5 +---- 7 files changed, 28 insertions(+), 30 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java index 35cd1faa6e..79207d3f41 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java @@ -44,13 +44,15 @@ public class ExcelRenderer { private final SXSSFWorkbook workbook; private final ExcelConfig config; + private final PrintSettings cfg; private final ImmutableMap styles; - public ExcelRenderer(ExcelConfig config) { + public ExcelRenderer(ExcelConfig config, PrintSettings cfg) { workbook = new SXSSFWorkbook(); this.config = config; - styles = config.generateStyles(workbook); + styles = config.generateStyles(workbook, cfg); + this.cfg = cfg; } @FunctionalInterface @@ -59,7 +61,6 @@ private interface TypeWriter { } public & SingleTableResult> void renderToStream( - PrintSettings cfg, List idHeaders, E exec, OutputStream outputStream) throws IOException { diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java index 76cb20600f..9964187f00 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java @@ -48,10 +48,9 @@ public & SingleTableResult> Response getExcelResu idPrinter::createId ); - ExcelRenderer excelRenderer = new ExcelRenderer(config.getExcel()); + ExcelRenderer excelRenderer = new ExcelRenderer(config.getExcel(), settings); StreamingOutput out = output -> excelRenderer.renderToStream( - settings, config.getFrontend().getQueryUpload().getPrintIdFields(), (ManagedExecution & SingleTableResult)exec, output diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ExcelConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/ExcelConfig.java index 3fd448f584..07e56496ef 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ExcelConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ExcelConfig.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.models.config; +import com.bakdata.conquery.models.query.PrintSettings; import com.google.common.collect.ImmutableMap; import lombok.AllArgsConstructor; import lombok.Data; @@ -34,8 +35,7 @@ public class ExcelConfig { BASIC_STYLE, new CellStyler(), CURRENCY_STYLE_PREFIX+"EUR", new CellStyler().withDataFormatString("#,##0.00 €"), NUMERIC_STYLE, new CellStyler().withDataFormatString("#,##0.00"), - INTEGER_STYLE, new CellStyler().withDataFormatString("#,###"), - DATE_STYLE, new CellStyler().withDataFormatString("yyyy-mm-dd") + INTEGER_STYLE, new CellStyler().withDataFormatString("#,###") ); /** @@ -52,14 +52,18 @@ DATE_STYLE, new CellStyler().withDataFormatString("yyyy-mm-dd") private int defaultColumnWidth = 30; - public ImmutableMap generateStyles(SXSSFWorkbook workbook){ + public ImmutableMap generateStyles(SXSSFWorkbook workbook, PrintSettings settings){ ImmutableMap.Builder styles = ImmutableMap.builder(); + // Add localized DateCell style + styles.put(DATE_STYLE, new CellStyler().withDataFormatString(settings.getDateFormat()).generateStyle(workbook)); + // Build configured styles for (Map.Entry entry : this.styles.entrySet()) { styles.put(entry.getKey(), entry.getValue().generateStyle(workbook)); } + // Add missing basic styles for (String s : FALLBACK_STYLES.keySet()) { if(this.styles.containsKey(s)){ diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java index c0b9c0b76e..0729f4be57 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java @@ -101,23 +101,20 @@ public String findDateRangeSeparator(Locale locale) { return findClosestMatch(locale, localeRangeStartEndSeparators, "/"); } + + /** - * Finds the best formatter according to the locale and mapped date formatters. + * Finds the best date format according to the locale and mapped date formatters. * If there is no perfect match, the locale is abstracted, see findClosestMatch. */ - public DateTimeFormatter findDateTimeFormater(Locale locale) { - return DateTimeFormatter.ofPattern(findClosestMatch(locale, dateFormatMapping, "yyyy-MM-dd")); + public String findDateFormat(Locale locale) { + return findClosestMatch(locale, dateFormatMapping, "yyyy-MM-dd"); } /** * Helper method to find the best match for a given locale using its abstractions. * First the vanilla locale is checked, then abstractions to country and language. * The last resort is the {@link Locale#ROOT}. If no match is found, the alternative is returned. - * - * @param forLocale - * @param options - * @param - * @return */ private static T findClosestMatch(Locale forLocale, Map options, T alternative) { String country = forLocale.getCountry(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/externalservice/ResultType.java b/backend/src/main/java/com/bakdata/conquery/models/externalservice/ResultType.java index 43c49f379d..34a1e2da3e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/externalservice/ResultType.java +++ b/backend/src/main/java/com/bakdata/conquery/models/externalservice/ResultType.java @@ -1,12 +1,10 @@ package com.bakdata.conquery.models.externalservice; import c10n.C10N; -import c10n.share.LocaleMapping; import com.bakdata.conquery.internationalization.Results; import com.bakdata.conquery.io.cps.CPSBase; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.common.CDate; -import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.events.MajorTypeId; import com.bakdata.conquery.models.forms.util.DateContext; import com.bakdata.conquery.models.query.PrintSettings; @@ -24,8 +22,6 @@ import org.apache.arrow.vector.types.pojo.FieldType; import java.math.BigDecimal; -import java.text.DateFormat; -import java.text.SimpleDateFormat; import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.List; @@ -204,10 +200,11 @@ public String print(PrintSettings cfg, @NonNull Object f) { if(!(f instanceof Number)) { throw new IllegalStateException("Expected an Number but got an '" + f.getClass().getName() + "' with the value: " + f); } + final Number number = (Number) f; if (cfg.isPrettyPrint()) { - return cfg.getDateFormat().format(CDate.toLocalDate(((Number) f).intValue())); + return print(number, cfg.getDateFormatter()); } - return print((Number) f, cfg.getDateFormat()); + return CDate.toLocalDate(number.intValue()).toString(); } @Override @@ -241,7 +238,7 @@ public String print(PrintSettings cfg, @NonNull Object f) { if(list.size() != 2) { throw new IllegalStateException("Expected a list with 2 elements, one min, one max. The list was: " + list); } - final DateTimeFormatter dateFormat = cfg.getDateFormat(); + final DateTimeFormatter dateFormat = cfg.getDateFormatter(); final Integer min = (Integer) list.get(0); final Integer max = (Integer) list.get(1); String minString = min == Integer.MIN_VALUE ? "-∞" : ResultType.DateT.print(min, dateFormat); @@ -315,7 +312,7 @@ public static class ListT extends ResultType { private final ResultType elementType; @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) - public ListT(ResultType elementType) { + public ListT(@NonNull ResultType elementType) { this.elementType = elementType; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/PrintSettings.java b/backend/src/main/java/com/bakdata/conquery/models/query/PrintSettings.java index 492b45c6b6..ef19a3c260 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/PrintSettings.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/PrintSettings.java @@ -3,6 +3,7 @@ import java.text.NumberFormat; import java.time.format.DateTimeFormatter; import java.util.Currency; +import java.util.Date; import java.util.Locale; import java.util.function.Function; @@ -30,7 +31,8 @@ public class PrintSettings { private final boolean prettyPrint; @ToString.Include private final Locale locale; - private DateTimeFormatter dateFormat; + private final String dateFormat; + private final DateTimeFormatter dateFormatter; private final NumberFormat decimalFormat; private final NumberFormat integerFormat; private final Currency currency; @@ -64,7 +66,8 @@ public PrintSettings(boolean prettyPrint, Locale locale, DatasetRegistry dataset this.listFormat = config.getLocale().getListFormats().get(0); this.dateRangeSeparator = config.getLocale().findDateRangeSeparator(locale); - this.dateFormat = config.getLocale().findDateTimeFormater(locale); + this.dateFormat = config.getLocale().findDateFormat(locale); + this.dateFormatter = DateTimeFormatter.ofPattern(dateFormat); } public PrintSettings(boolean prettyPrint, Locale locale, DatasetRegistry datasetRegistry, ConqueryConfig config, PrintIdMapper idMapper) { diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java index cf8a8eba18..59b7bb42a3 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java @@ -75,8 +75,6 @@ public List getResultInfo() { return coll.getInfos(); } - ; - @Override public Stream streamResults() { return results.stream(); @@ -86,10 +84,9 @@ public Stream streamResults() { // First we write to the buffer, than we read from it and parse it as TSV ByteArrayOutputStream output = new ByteArrayOutputStream(); - ExcelRenderer renderer = new ExcelRenderer(new ExcelConfig()); + ExcelRenderer renderer = new ExcelRenderer(new ExcelConfig(),printSettings); renderer.renderToStream( - printSettings, printIdFields, mquery, output); From 80cff192ca9db53bfc5fc749a69b9624c59f60b6 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Wed, 18 Aug 2021 16:25:42 +0200 Subject: [PATCH 79/82] make getPrintIdFields localized by localized label --- .../io/result/arrow/ResultArrowProcessor.java | 10 +++++----- .../io/result/csv/ResultCsvProcessor.java | 8 +++++--- .../io/result/excel/ResultExcelProcessor.java | 19 ++++++++++--------- .../models/config/FrontendConfig.java | 18 ++++++++++-------- .../conquery/models/query/ManagedQuery.java | 7 +++++-- .../conquery/integration/json/FormTest.java | 5 ++--- 6 files changed, 37 insertions(+), 30 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java index 39a1d4cda6..341b29b141 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java @@ -5,6 +5,7 @@ import static com.bakdata.conquery.models.auth.AuthorizationHelper.authorizeDownloadDatasets; import java.io.OutputStream; +import java.util.Locale; import java.util.function.Function; import javax.ws.rs.core.Response; @@ -17,12 +18,10 @@ import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.forms.managed.ManagedForm; import com.bakdata.conquery.models.i18n.I18n; - import com.bakdata.conquery.models.identifiable.mapping.IdPrinter; import com.bakdata.conquery.models.query.ManagedQuery; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.SingleTableResult; -import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.util.io.ConqueryMDC; @@ -67,10 +66,11 @@ public static & SingleTableResult> Response getAr // Get the locale extracted by the LocaleFilter - IdPrinter idPrinter = config.getFrontend().getQueryUpload().getIdPrinter(user,exec,namespace); + IdPrinter idPrinter = config.getFrontend().getQueryUpload().getIdPrinter(user, exec, namespace); + final Locale locale = I18n.LOCALE.get(); PrintSettings settings = new PrintSettings( pretty, - I18n.LOCALE.get(), + locale, datasetRegistry, config, idPrinter::createId @@ -81,7 +81,7 @@ public static & SingleTableResult> Response getAr writerProducer.apply(output), settings, config.getArrow().getBatchSize(), - config.getFrontend().getQueryUpload().getPrintIdFields(), + config.getFrontend().getQueryUpload().getPrintIdFields(locale), exec.getResultInfo(), exec.streamResults() ); diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java index cac11892ec..5f10fe2dc4 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java @@ -7,6 +7,7 @@ import java.io.BufferedWriter; import java.io.OutputStreamWriter; import java.nio.charset.Charset; +import java.util.Locale; import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.Response; @@ -48,13 +49,14 @@ public & SingleTableResult> Response getResult(Us // Check if user is permitted to download on all datasets that were referenced by the query authorizeDownloadDatasets(user, exec); - IdPrinter idPrinter = config.getFrontend().getQueryUpload().getIdPrinter(user,exec,namespace); + IdPrinter idPrinter = config.getFrontend().getQueryUpload().getIdPrinter(user, exec, namespace); // Get the locale extracted by the LocaleFilter + final Locale locale = I18n.LOCALE.get(); PrintSettings settings = new PrintSettings( pretty, - I18n.LOCALE.get(), + locale, datasetRegistry, config, idPrinter::createId @@ -65,7 +67,7 @@ public & SingleTableResult> Response getResult(Us StreamingOutput out = os -> { try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(os, charset))) { CsvRenderer renderer = new CsvRenderer(config.getCsv().createWriter(writer), settings); - renderer.toCSV(config.getFrontend().getQueryUpload().getPrintIdFields(), exec.getResultInfo(), exec.streamResults()); + renderer.toCSV(config.getFrontend().getQueryUpload().getPrintIdFields(locale), exec.getResultInfo(), exec.streamResults()); } catch (EofException e) { log.info("User canceled download"); diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java index 76cb20600f..19a767283e 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java @@ -1,5 +1,12 @@ package com.bakdata.conquery.io.result.excel; +import static com.bakdata.conquery.io.result.ResultUtil.makeResponseWithFileName; + +import java.util.Locale; + +import javax.ws.rs.core.Response; +import javax.ws.rs.core.StreamingOutput; + import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.auth.permissions.Ability; import com.bakdata.conquery.models.config.ConqueryConfig; @@ -7,21 +14,14 @@ import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.i18n.I18n; import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; - import com.bakdata.conquery.models.identifiable.mapping.IdPrinter; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.SingleTableResult; -import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.util.io.ConqueryMDC; import lombok.RequiredArgsConstructor; -import javax.ws.rs.core.Response; -import javax.ws.rs.core.StreamingOutput; - -import static com.bakdata.conquery.io.result.ResultUtil.makeResponseWithFileName; - @RequiredArgsConstructor public class ResultExcelProcessor { @@ -40,9 +40,10 @@ public & SingleTableResult> Response getExcelResu IdPrinter idPrinter = config.getFrontend().getQueryUpload().getIdPrinter(user,exec,namespace); + final Locale locale = I18n.LOCALE.get(); PrintSettings settings = new PrintSettings( pretty, - I18n.LOCALE.get(), + locale, datasetRegistry, config, idPrinter::createId @@ -52,7 +53,7 @@ public & SingleTableResult> Response getExcelResu StreamingOutput out = output -> excelRenderer.renderToStream( settings, - config.getFrontend().getQueryUpload().getPrintIdFields(), + config.getFrontend().getQueryUpload().getPrintIdFields(locale), (ManagedExecution & SingleTableResult)exec, output ); diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java index 144b5725d2..1c68ec2523 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java @@ -4,6 +4,7 @@ import java.util.Arrays; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -83,14 +84,14 @@ public static class UploadConfig { ); /** - * Headers for Output CSV. + * Localized header for output CSV. */ - @JsonIgnore - @Getter(lazy = true) - private final List printIdFields = ids.stream() - .filter(ColumnConfig::isPrint) - .map(ColumnConfig::getField) - .collect(Collectors.toUnmodifiableList()); + public List getPrintIdFields(Locale locale) { + return ids.stream() + .filter(ColumnConfig::isPrint) + .map(col -> col.getLabel().getOrDefault(locale.getDisplayName(), col.getField())) + .collect(Collectors.toUnmodifiableList()); + } @JsonIgnore @@ -175,7 +176,8 @@ public DateFormat resolveDateFormat(String name) { * Try to create a {@link FullIdPrinter} for user if they are allowed. If not allowed to read ids, they will receive a pseudomized result instead. */ public IdPrinter getIdPrinter(User owner, ManagedExecution execution, Namespace namespace) { - final int size = getPrintIdFields().size(); + final int size = (int) ids.stream().filter(ColumnConfig::isPrint).count(); + final int pos = IntStream.range(0, getIds().size()) .filter(idx -> getIds().get(idx).isFillAnon()) .findFirst() diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java b/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java index 7db69acc5f..cd53f6e1be 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java @@ -3,6 +3,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; @@ -162,8 +163,10 @@ public List generateColumnDescriptions(DatasetRegistry dataset Preconditions.checkArgument(isInitialized(), "The execution must have been initialized first"); List columnDescriptions = new ArrayList<>(); + final Locale locale = I18n.LOCALE.get(); + // First add the id columns to the descriptor list. The are the first columns - for (String header : config.getFrontend().getQueryUpload().getPrintIdFields()) { + for (String header : config.getFrontend().getQueryUpload().getPrintIdFields(locale)) { columnDescriptions.add(ColumnDescriptor.builder() .label(header) .type(ResultType.IdT.INSTANCE.typeInfo()) @@ -171,7 +174,7 @@ public List generateColumnDescriptions(DatasetRegistry dataset } // Then all columns that originate from selects and static aggregators - PrintSettings settings = new PrintSettings(true, I18n.LOCALE.get(), datasetRegistry, config, null); + PrintSettings settings = new PrintSettings(true, locale, datasetRegistry, config, null); getResultInfo().forEach(info -> columnDescriptions.add(info.asColumnDescriptor(settings))); return columnDescriptions; diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java b/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java index 80f1075c77..a6bc4a9457 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java @@ -23,14 +23,13 @@ import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.result.CsvLineStreamRenderer; import com.bakdata.conquery.models.auth.entities.User; -import com.bakdata.conquery.models.datasets.concepts.Concept; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.datasets.Dataset; +import com.bakdata.conquery.models.datasets.concepts.Concept; import com.bakdata.conquery.models.exceptions.JSONException; import com.bakdata.conquery.models.execution.ExecutionState; import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.forms.managed.ManagedForm; - import com.bakdata.conquery.models.identifiable.mapping.IdPrinter; import com.bakdata.conquery.models.query.ManagedQuery; import com.bakdata.conquery.models.query.PrintSettings; @@ -157,7 +156,7 @@ private void checkResults(StandaloneSupport standaloneSupport, ManagedForm manag log.info("{} CSV TESTING: {}", getLabel(), managed.getKey()); List actual = renderer.toStream( - config.getFrontend().getQueryUpload().getPrintIdFields(), + config.getFrontend().getQueryUpload().getPrintIdFields(Locale.ENGLISH), resultInfos, managed.getValue().stream().flatMap(ManagedQuery::streamResults) ) From d1cd2326e57ddcc3fc38fed9fab4fe5769fcebed Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Thu, 19 Aug 2021 16:30:27 +0200 Subject: [PATCH 80/82] post merge fixes --- .../conquery/io/result/arrow/ResultArrowProcessor.java | 1 + .../bakdata/conquery/io/result/csv/ResultCsvProcessor.java | 1 + .../conquery/resources/admin/rest/AdminDatasetProcessor.java | 2 +- .../conquery/resources/admin/rest/AdminDatasetResource.java | 5 ++--- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java index 451fa44c40..7f744da477 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java @@ -13,6 +13,7 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.StreamingOutput; +import com.bakdata.conquery.io.result.ResultUtil; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.auth.permissions.Ability; import com.bakdata.conquery.models.config.ConqueryConfig; diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java index f063e0997a..ea37e39604 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java @@ -14,6 +14,7 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.StreamingOutput; +import com.bakdata.conquery.io.result.ResultUtil; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.auth.permissions.Ability; import com.bakdata.conquery.models.config.ConqueryConfig; diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java index 6b625f8f47..4efda5e396 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java @@ -357,7 +357,7 @@ public void updateMatchingStats(Dataset dataset) { )); } - public PersistentIdMap getIdMapping(Namespace namespace) { + public EntityIdMap getIdMapping(Namespace namespace) { return namespace.getStorage().getIdMapping(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetResource.java index df0603ea42..25d1110e38 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetResource.java @@ -11,14 +11,13 @@ import com.bakdata.conquery.models.exceptions.JSONException; import com.bakdata.conquery.models.identifiable.ids.specific.ConceptId; import com.bakdata.conquery.models.identifiable.ids.specific.TableId; -import com.bakdata.conquery.models.identifiable.mapping.PersistentIdMap; +import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.resources.hierarchies.HAdmin; import io.dropwizard.auth.Auth; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; -import org.glassfish.jersey.media.multipart.FormDataParam; import javax.annotation.PostConstruct; import javax.inject.Inject; @@ -67,7 +66,7 @@ public void init() { @GET @Consumes(MediaType.WILDCARD) @Path("mapping") - public PersistentIdMap getIdMapping() { + public EntityIdMap getIdMapping() { return processor.getIdMapping(namespace); } From 7e1e1ac873967a5d8cbc12ef9b97519f5db6e480 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 19 Aug 2021 14:33:01 +0000 Subject: [PATCH 81/82] Update AutoDoc --- docs/Config JSON.md | 84 +++++++++++++++++++++--------------------- docs/REST API JSONs.md | 20 +++++----- 2 files changed, 53 insertions(+), 51 deletions(-) diff --git a/docs/Config JSON.md b/docs/Config JSON.md index 9bc446139e..992cd02c2c 100644 --- a/docs/Config JSON.md +++ b/docs/Config JSON.md @@ -41,10 +41,10 @@ No fields can be set for this type. --- -## Base AuthenticationConfig +## Base AuthenticationRealmFactory An `AuthenticationConfig` is used to define how specific realms for authentication are configured. -Different types of AuthenticationConfig can be used by setting `type` to one of the following values: +Different types of AuthenticationRealmFactory can be used by setting `type` to one of the following values: ### DEVELOPMENT [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/auth/develop/DevAuthConfig.java#L11-L14) @@ -58,7 +58,7 @@ No fields can be set for this type.

-### JWT_PKCE_REALM [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/auth/JwtPkceVerifyingRealmFactory.java#L33-L35) +### JWT_PKCE_REALM [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/auth/JwtPkceVerifyingRealmFactory.java#L41-L43) A realm that verifies oauth tokens using PKCE.
Details

@@ -69,14 +69,14 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/auth/JwtPkceVerifyingRealmFactory.java#L51) | additionalTokenChecks | list of `String` | `[]` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/auth/JwtPkceVerifyingRealmFactory.java#L47) | allowedAudiences | list of `String` | `[]` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/auth/JwtPkceVerifyingRealmFactory.java#L54-L57) | alternativeIdClaims | list of `String` | `[]` | | Which claims hold alternative Ids of the user in case the user name does not match a user. Pay attention, that the user must not be able to alter the value of any of these claims. | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/auth/JwtPkceVerifyingRealmFactory.java#L49) | issuer | `String` | `null` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/auth/JwtPkceVerifyingRealmFactory.java#L41-L44) | jwk | `@NotNull JWK` | `null` | | The public key information that is used to validate signed JWT. It can be retrieved from the IDP. | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/auth/JwtPkceVerifyingRealmFactory.java#L56) | additionalTokenChecks | list of `String` | `[]` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/auth/JwtPkceVerifyingRealmFactory.java#L71-L74) | alternativeIdClaims | list of `String` | `[]` | | Which claims hold alternative Ids of the user in case the user name does not match a user. Pay attention, that the user must not be able to alter the value of any of these claims. | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/auth/JwtPkceVerifyingRealmFactory.java#L50-L53) | client | `String` | `null` | | The client id is also used as the expected audience in the validated token. Ensure that the IDP is configured accordingly. | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/auth/JwtPkceVerifyingRealmFactory.java#L65-L67) | idpConfiguration | `IdpConfiguration` | `null` | | See wellKnownEndpoint. | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/auth/JwtPkceVerifyingRealmFactory.java#L59-L62) | wellKnownEndpoint | `URI` | `null` | | Either the wellKnownEndpoint from which an idpConfiguration can be obtained or the idpConfiguration must be supplied. If the idpConfiguration is given, the wellKnownEndpoint is ignored. |

-### LOCAL_AUTHENTICATION [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/auth/LocalAuthenticationConfig.java#L27) +### LOCAL_AUTHENTICATION [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/auth/LocalAuthenticationConfig.java#L32)
Details

@@ -87,10 +87,10 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/auth/LocalAuthenticationConfig.java#L48) | directory | `File` | `"./storage"` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/auth/LocalAuthenticationConfig.java#L38) | jwtDuration | `@MinDuration(value=1, unit=TimeUnit.MINUTES) Duration` | `"12 hours"` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/auth/LocalAuthenticationConfig.java#L32-L34) | passwordStoreConfig | [XodusConfig](#Type-XodusConfig) | | | Configuration for the password store. An encryption for the store it self might be set here. | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/auth/LocalAuthenticationConfig.java#L41-L43) | storeName | `String` | `"authenticationStore"` | | The name of the folder the store lives in. | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/auth/LocalAuthenticationConfig.java#L54) | directory | `File` | `"./storage"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/auth/LocalAuthenticationConfig.java#L44) | jwtDuration | `@MinDuration(value=1, unit=TimeUnit.MINUTES) Duration` | `"12 hours"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/auth/LocalAuthenticationConfig.java#L38-L40) | passwordStoreConfig | [XodusConfig](#Type-XodusConfig) | | | Configuration for the password store. An encryption for the store it self might be set here. | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/auth/LocalAuthenticationConfig.java#L47-L49) | storeName | `String` | `"authenticationStore"` | | The name of the folder the store lives in. |

### OIDC_AUTHORIZATION_CODE_FLOW [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/auth/OIDCAuthorizationCodeFlowRealmFactory.java#L11-L13) @@ -192,7 +192,7 @@ Supported Fields: | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ClusterConfig.java#L16) | port | `@io.dropwizard.validation.PortRange int` | `16170` | | |

-### Type ConqueryConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L35) +### Type ConqueryConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L37)
Details

@@ -203,27 +203,29 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L74) | api | [APIConfig](#Type-APIConfig) | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L50) | arrow | `@Valid @NotNull ArrowConfig` | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L83) | authentication | list of [AuthenticationConfig](#Base-AuthenticationConfig) | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L87) | authorization | [@Valid @NotNull AuthorizationConfig](#Base-AuthorizationConfig) | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L41) | cluster | [ClusterConfig](#Type-ClusterConfig) | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L47) | csv | [CSVConfig](#Type-CSVConfig) | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L96-L98) | debugMode | `boolean` or `null` | `null` | | null means here that we try to deduce from an attached agent | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L90) | excel | `@Valid @NotNull ExcelConfig` | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L101) | failOnError | `boolean` | `false` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L77) | frontend | [FrontendConfig](#Type-FrontendConfig) | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L63) | locale | [LocaleConfig](#Type-LocaleConfig) | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L81) | metricsConfig | `ConqueryMetricsConfig` | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L94) | plugins | list of `PluginConfig` | `[]` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L44) | preprocessor | [PreprocessingConfig](#Type-PreprocessingConfig) | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L71) | queries | [QueryConfig](#Type-QueryConfig) | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L53-L55) | resultProviders | list of `ResultRendererProvider` | | | The order of this lists determines the ordner of the generated result urls in a query status. | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L65) | standalone | [StandaloneConfig](#Type-StandaloneConfig) | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L68) | storage | `@Valid @NotNull StoreFactory` | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L76) | api | [APIConfig](#Type-APIConfig) | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L52) | arrow | `@Valid @NotNull ArrowConfig` | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L85) | authentication | `@Valid @NotNull AuthenticationConfig` | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L89) | authenticationRealms | list of [AuthenticationRealmFactory](#Base-AuthenticationRealmFactory) | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L93) | authorizationRealms | [@Valid @NotNull AuthorizationConfig](#Base-AuthorizationConfig) | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L43) | cluster | [ClusterConfig](#Type-ClusterConfig) | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L49) | csv | [CSVConfig](#Type-CSVConfig) | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L106-L108) | debugMode | `boolean` or `null` | `null` | | null means here that we try to deduce from an attached agent | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L96) | excel | `@Valid @NotNull ExcelConfig` | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L111) | failOnError | `boolean` | `false` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L79) | frontend | [FrontendConfig](#Type-FrontendConfig) | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L100) | jerseyClient | `@Valid @NotNull JerseyClientConfiguration` | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L65) | locale | [LocaleConfig](#Type-LocaleConfig) | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L83) | metricsConfig | `ConqueryMetricsConfig` | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L104) | plugins | list of `PluginConfig` | `[]` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L46) | preprocessor | [PreprocessingConfig](#Type-PreprocessingConfig) | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L73) | queries | [QueryConfig](#Type-QueryConfig) | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L55-L57) | resultProviders | list of `ResultRendererProvider` | | | The order of this lists determines the ordner of the generated result urls in a query status. | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L67) | standalone | [StandaloneConfig](#Type-StandaloneConfig) | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java#L70) | storage | `@Valid @NotNull StoreFactory` | | | |

-### Type CurrencyConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L227) +### Type CurrencyConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L195)
Details

@@ -234,13 +236,13 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L232) | decimalScale | `int` | `2` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L231) | decimalSeparator | `String` | `","` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L229) | prefix | `String` | `"€"` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L230) | thousandSeparator | `String` | `"."` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L200) | decimalScale | `int` | `2` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L199) | decimalSeparator | `String` | `","` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L197) | prefix | `String` | `"€"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L198) | thousandSeparator | `String` | `"."` | | |

-### Type FrontendConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L37) +### Type FrontendConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L42)
Details

@@ -251,9 +253,9 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L47) | currency | [CurrencyConfig](#Type-CurrencyConfig) | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L51) | queryUpload | `UploadConfig` | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L46) | version | `String` | `"0.0.0-SNAPSHOT"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L52) | currency | [CurrencyConfig](#Type-CurrencyConfig) | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L56) | queryUpload | `UploadConfig` | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L51) | version | `String` | `"0.0.0-SNAPSHOT"` | | |

### Type LocaleConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L11) diff --git a/docs/REST API JSONs.md b/docs/REST API JSONs.md index d8d152f2eb..7a503dc1e4 100644 --- a/docs/REST API JSONs.md +++ b/docs/REST API JSONs.md @@ -509,7 +509,7 @@ Supported Fields: | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQElement.java#L31-L33) | label | `String` | ? | | Allows the user to define labels. |

-### EXTERNAL [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java#L37-L39) +### EXTERNAL [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java#L38-L40) Allows uploading lists of entities.
Details

@@ -755,7 +755,7 @@ Supported Fields: | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptResource.java#L68) | concepts | list of `String` | `null` | | |

-### Type CurrencyConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L227) +### Type CurrencyConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L195)
Details

@@ -766,10 +766,10 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L232) | decimalScale | `int` | `2` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L231) | decimalSeparator | `String` | `","` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L229) | prefix | `String` | `"€"` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L230) | thousandSeparator | `String` | `"."` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L200) | decimalScale | `int` | `2` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L199) | decimalSeparator | `String` | `","` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L197) | prefix | `String` | `"€"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L198) | thousandSeparator | `String` | `"."` | | |

### Type ExecutionStatus [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/ExecutionStatus.java#L18) @@ -885,7 +885,7 @@ No fields can be set for this type.

-### Type FrontendConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L37) +### Type FrontendConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L42)
Details

@@ -896,9 +896,9 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L47) | currency | [CurrencyConfig](#Type-CurrencyConfig) | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L51) | queryUpload | `UploadConfig` | | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L46) | version | `String` | `"0.0.0-SNAPSHOT"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L52) | currency | [CurrencyConfig](#Type-CurrencyConfig) | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L56) | queryUpload | `UploadConfig` | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L51) | version | `String` | `"0.0.0-SNAPSHOT"` | | |

### Type FullExecutionStatus [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/FullExecutionStatus.java#L20-L24) From b8aabc2057820e9a5d20ba89991b89b19fc28400 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 19 Aug 2021 15:14:15 +0000 Subject: [PATCH 82/82] Update AutoDoc --- docs/Config JSON.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/Config JSON.md b/docs/Config JSON.md index 992cd02c2c..cd722c0e37 100644 --- a/docs/Config JSON.md +++ b/docs/Config JSON.md @@ -258,7 +258,7 @@ Supported Fields: | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java#L51) | version | `String` | `"0.0.0-SNAPSHOT"` | | |

-### Type LocaleConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L11) +### Type LocaleConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L16)
Details

@@ -269,7 +269,12 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L13) | frontend | `@NotNull Locale` | `""` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L21-L25) | dateFormatMapping | map from `Locale` to `String` | | | Mappings from user provided locale to date format which is used in the generation of result tables. The formats are also available for parsing dates using the {@link DateReader}. However, the locale is neglected there and the formats are tried until one that fits is found. | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L18) | frontend | `@NotNull Locale` | `""` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L57-L59) | listFormats | list of `ListFormat` | | | List formats that are available for parsing inputs and (the first one) for rendering result tables. | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L41-L45) | localeRangeStartEndSeparators | map from `Locale` to `String` | | | Mappings from user provided locale to date range format which is used in the generation of result tables. The formats are also available for parsing dates ranges using the {@link DateReader}. However, the locale is neglected there and the formats are tried until one that fits is found. | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L33-L35) | parsingDateFormats | `@NotNull Set` | `["yyyyMMdd"]` | | Additional date formats that are available only for parsing. | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/LocaleConfig.java#L52-L54) | parsingRangeStartEndSeparators | `Set` | `["/"]` | | Additional date range formats that are available only for parsing. |

### Type MinaConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/MinaConfig.java#L13)