Rework of IdMapping toward explicit Id-kinds

autodoc/src/main/java/com/bakdata/conquery/Constants.java

@@ -62,7 +62,6 @@
 import com.bakdata.conquery.models.forms.configs.FormConfig;
 import com.bakdata.conquery.models.forms.configs.FormConfig.FormConfigFullRepresentation;
 import com.bakdata.conquery.models.forms.configs.FormConfig.FormConfigOverviewRepresentation;
-import com.bakdata.conquery.models.identifiable.mapping.IdMappingConfig;
 import com.bakdata.conquery.models.preproc.TableImportDescriptor;
 import com.bakdata.conquery.models.preproc.TableInputDescriptor;
 import com.bakdata.conquery.models.preproc.outputs.OutputDescription;
@@ -117,7 +116,6 @@ public class Constants {
 				 .base(new Base(AuthorizationConfig.class, "An `AuthorizationConfig` defines the initial users that are created on application startup and other permission related options."))
 				 .base(new Base(AuthenticationConfig.class, "An `AuthenticationConfig` is used to define how specific realms for authentication are configured."))
 				 .base(new Base(PluginConfig.class, "A `PluginConfig` is used to define settings for Conquery plugins."))
-				 .base(new Base(IdMappingConfig.class, "An `IdMappingConfig` is used to define how multi column entity IDs are printed and parsed"))
 				 .otherClass(APIConfig.class)
 				 .otherClass(ConqueryConfig.class)
 				 .otherClass(ClusterConfig.class)

backend/src/main/java/com/bakdata/conquery/Conquery.java

@@ -12,14 +12,14 @@
 import com.bakdata.conquery.commands.RecodeStoreCommand;
 import com.bakdata.conquery.commands.ShardNode;
 import com.bakdata.conquery.commands.StandaloneCommand;
+import com.bakdata.conquery.io.jackson.InternalOnly;
 import com.bakdata.conquery.io.jackson.Jackson;
 import com.bakdata.conquery.io.jackson.MutableInjectableValues;
 import com.bakdata.conquery.models.config.ConqueryConfig;
-import com.bakdata.conquery.util.DateFormats;
 import com.bakdata.conquery.util.UrlRewriteBundle;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import io.dropwizard.Application;
 import io.dropwizard.ConfiguredBundle;
-import io.dropwizard.configuration.EnvironmentVariableSubstitutor;
 import io.dropwizard.configuration.JsonConfigurationFactory;
 import io.dropwizard.configuration.SubstitutingSourceProvider;
 import io.dropwizard.servlets.assets.AssetServlet;
@@ -46,7 +46,11 @@ public Conquery() {
 
 	@Override
 	public void initialize(Bootstrap<ConqueryConfig> bootstrap) {
-		Jackson.configure(bootstrap.getObjectMapper());
+		final ObjectMapper confMapper = bootstrap.getObjectMapper();
+		Jackson.configure(confMapper);
+
+		confMapper.setConfig(confMapper.getDeserializationConfig().withView(InternalOnly.class));
+
 		// check for java compiler, needed for the class generation
 		if (ToolProvider.getSystemJavaCompiler() == null) {
 			throw new IllegalStateException("Conquery requires to be run on either a JDK or a ServerJRE");
@@ -61,7 +65,7 @@ public void initialize(Bootstrap<ConqueryConfig> bootstrap) {
 		bootstrap.addCommand(new StandaloneCommand(this));
 		bootstrap.addCommand(new RecodeStoreCommand());
 
-		((MutableInjectableValues)bootstrap.getObjectMapper().getInjectableValues()).add(Validator.class, bootstrap.getValidatorFactory().getValidator());
+		((MutableInjectableValues) confMapper.getInjectableValues()).add(Validator.class, bootstrap.getValidatorFactory().getValidator());
 
 		// do some setup in other classes after initialization but before running a
 		// command

backend/src/main/java/com/bakdata/conquery/ConqueryConstants.java

@@ -30,12 +30,14 @@ public class ConqueryConstants {
 	public static final ResultInfo FEATURE_DATE_RANGE_INFO = new LocalizedDefaultResultInfo((l) -> C10N.get(ResultHeadersC10n.class, l).featureDateRange(), ResultType.DateRangeT.INSTANCE);
 	public static final ResultInfo OUTCOME_DATE_RANGE_INFO = new LocalizedDefaultResultInfo((l) -> C10N.get(ResultHeadersC10n.class, l).outcomeDateRange(), ResultType.DateRangeT.INSTANCE);
 
+	public static final String PRIMARY_DICTIONARY = "PRIMARY_DICTIONARY";
+
 	public static class AuthenticationUtil {
 		public static final String REALM_NAME = "CONQUERY";
 	}
 
 	public static DictionaryId getPrimaryDictionary(Dataset dataset) {
-		return DictionaryId.Parser.INSTANCE.parse(Arrays.asList(dataset.getName(), "primary_dictionary"));
+		return DictionaryId.Parser.INSTANCE.parse(Arrays.asList(dataset.getName(), PRIMARY_DICTIONARY));
 	}
 
 }

backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java

@@ -13,15 +13,19 @@
 import java.util.stream.Stream;
 
 import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.BadRequestException;
+import javax.ws.rs.core.Response;
 import javax.ws.rs.core.UriBuilder;
 
 import com.bakdata.conquery.apiv1.query.CQElement;
 import com.bakdata.conquery.apiv1.query.ConceptQuery;
+import com.bakdata.conquery.apiv1.query.ExternalUpload;
 import com.bakdata.conquery.apiv1.query.Query;
 import com.bakdata.conquery.apiv1.query.QueryDescription;
 import com.bakdata.conquery.apiv1.query.SecondaryIdQuery;
+import com.bakdata.conquery.apiv1.query.ExternalUploadResult;
 import com.bakdata.conquery.apiv1.query.concept.specific.CQAnd;
-import com.bakdata.conquery.apiv1.query.concept.specific.CQExternal;
+import com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal;
 import com.bakdata.conquery.io.result.ResultRender.ResultRendererProvider;
 import com.bakdata.conquery.io.storage.MetaStorage;
 import com.bakdata.conquery.metrics.ExecutionMetrics;
@@ -359,5 +363,37 @@ public FullExecutionStatus getQueryFullStatus(ManagedExecution<?> query, User us
 		return status;
 	}
 
+	/**
+	 * Try to resolve the external upload, if successful, create query for the user and return id and statistics for that.
+	 */
+	public ExternalUploadResult uploadEntities(User user, Dataset dataset, ExternalUpload upload) {
+
+		final CQExternal.ResolveStatistic statistic =
+				CQExternal.resolveEntities(upload.getValues(), upload.getFormat(),
+										   datasetRegistry.get(dataset.getId()).getStorage().getIdMapping(),
+										   config.getFrontend().getQueryUpload(),
+										   config.getPreprocessor().getParsers().getDateReader()
+				);
+
+		// Resolving nothing is a problem thus we fail.
+		if (statistic.getResolved().isEmpty()) {
+			throw new BadRequestException(Response.status(Response.Status.BAD_REQUEST)
+						   .entity(new ExternalUploadResult(null, 0, statistic.getUnresolvedId(), statistic.getUnreadableDate()))
+						   .build());
+		}
 
+		final ConceptQuery query = new ConceptQuery(new CQExternal(upload.getFormat(), upload.getValues()));
+
+		// We only create the Query, really no need to execute it as it's only useful for composition.
+		final ManagedExecution<?> execution =
+				datasetRegistry.get(dataset.getId()).getExecutionManager()
+							   .createExecution(datasetRegistry, query, user, dataset);
+
+		return new ExternalUploadResult(
+							   execution.getId(),
+							   statistic.getResolved().size(),
+							   statistic.getUnresolvedId(),
+							   statistic.getUnreadableDate()
+					   );
+	}
 }

backend/src/main/java/com/bakdata/conquery/apiv1/query/ExternalUpload.java

@@ -0,0 +1,31 @@
+package com.bakdata.conquery.apiv1.query;
+
+import java.util.Arrays;
+import java.util.List;
+
+import com.bakdata.conquery.models.auth.entities.User;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import io.dropwizard.validation.ValidationMethod;
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+
+
+/**
+ * Data Container for API call of {@link com.bakdata.conquery.resources.api.QueryResource#upload(User, ExternalUpload)}.
+ *
+ * This class acts as a wrapper for {@link com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal}.
+ */
+@RequiredArgsConstructor
+@Getter
+public class ExternalUpload {
+
+	private final List<String> format;
+	private final String[][] values;
+
+	@JsonIgnore
+	@ValidationMethod(message = "Values and Format are not of same width.")
+	public boolean isAllSameLength() {
+		final int expected = format.size();
+		return Arrays.stream(values).mapToInt(a -> a.length).allMatch(v -> expected == v);
+	}
+}

backend/src/main/java/com/bakdata/conquery/apiv1/query/ExternalUploadResult.java

@@ -0,0 +1,32 @@
+package com.bakdata.conquery.apiv1.query;
+
+import java.util.List;
+
+import com.bakdata.conquery.models.auth.entities.User;
+import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId;
+import lombok.Data;
+
+/**
+ * Result Container for responses of {@link com.bakdata.conquery.resources.api.QueryResource#upload(User, ExternalUpload)}.
+ */
+@Data
+public class ExternalUploadResult {
+	/**
+	 * Id of created execution of uploaded data.
+	 */
+	private final ManagedExecutionId execution;
+
+	/**
+	 * Number of successfully resolved rows.
+	 */
+	private final int resolved;
+
+	/**
+	 * Content of rows where no Id could be resolved.
+	 */
+	private final List<String[]> unresolvedId;
+	/**
+	 * Content of rows where dates could not be read.
+	 */
+	private final List<String[]> unreadableDate;
+}

backend/src/main/java/com/bakdata/conquery/apiv1/query/QueryDescription.java

@@ -3,7 +3,6 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 
-import com.bakdata.conquery.apiv1.query.concept.specific.CQExternal;
 import com.bakdata.conquery.io.cps.CPSBase;
 import com.bakdata.conquery.io.jackson.InternalOnly;
 import com.bakdata.conquery.models.auth.entities.User;
@@ -17,6 +16,7 @@
 import com.bakdata.conquery.models.identifiable.ids.NamespacedIdentifiable;
 import com.bakdata.conquery.models.query.QueryResolveContext;
 import com.bakdata.conquery.models.query.Visitable;
+import com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal;
 import com.bakdata.conquery.models.query.visitor.QueryVisitor;
 import com.bakdata.conquery.models.worker.DatasetRegistry;
 import com.bakdata.conquery.util.QueryUtils;