Feature/self inject storage into permission owners

backend/src/main/java/com/bakdata/conquery/apiv1/ApiV1.java

@@ -47,7 +47,7 @@ protected void configure() {
 				bind(new ConceptsProcessor(manager.getDatasetRegistry())).to(ConceptsProcessor.class);
 				bind(new MeProcessor(manager.getStorage(), datasets)).to(MeProcessor.class);
 				bind(new QueryProcessor(datasets, manager.getStorage(), manager.getConfig())).to(QueryProcessor.class);
-				bind(new FormConfigProcessor(manager.getValidator(), manager.getStorage())).to(FormConfigProcessor.class);
+				bind(new FormConfigProcessor(manager.getValidator(), manager.getStorage(), datasets)).to(FormConfigProcessor.class);
 			}
 		});
 

backend/src/main/java/com/bakdata/conquery/apiv1/FormConfigPatch.java

@@ -3,7 +3,6 @@
 import java.util.function.Consumer;
 
 import com.bakdata.conquery.io.storage.MetaStorage;
-import com.bakdata.conquery.models.auth.AuthorizationHelper;
 import com.bakdata.conquery.models.auth.entities.User;
 import com.bakdata.conquery.models.auth.permissions.Ability;
 import com.bakdata.conquery.models.forms.configs.FormConfig;

backend/src/main/java/com/bakdata/conquery/apiv1/MetaDataPatch.java

@@ -6,7 +6,6 @@
 import java.util.function.Consumer;
 
 import com.bakdata.conquery.io.storage.MetaStorage;
-import com.bakdata.conquery.models.auth.AuthorizationHelper;
 import com.bakdata.conquery.models.auth.entities.Group;
 import com.bakdata.conquery.models.auth.entities.User;
 import com.bakdata.conquery.models.auth.permissions.Ability;

backend/src/main/java/com/bakdata/conquery/apiv1/auth/ProtoUser.java

@@ -2,6 +2,7 @@
 
 import java.util.Collections;
 import java.util.List;
+import java.util.Optional;
 import java.util.Set;
 
 import javax.validation.Valid;
@@ -13,12 +14,15 @@
 import com.bakdata.conquery.models.auth.basic.LocalAuthenticationRealm;
 import com.bakdata.conquery.models.auth.entities.User;
 import com.bakdata.conquery.models.auth.permissions.WildcardPermission;
+import com.bakdata.conquery.models.identifiable.ids.specific.UserId;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import lombok.Builder;
 import lombok.Getter;
+import lombok.NonNull;
 
 /**
- * Container class for holding information about initial users.
+ * Factory class to create configured initial users.
+
  */
 @Getter
 @Builder
@@ -45,39 +49,21 @@ public class ProtoUser {
 	@NotNull
 	@Valid
 	private List<CredentialType> credentials = Collections.emptyList();
-
-	@JsonIgnore
-	// Let this be ignored by the builder
-	private User user = null;
 
-	public User getUser() {
-		if(user != null) {
-			return user;
-		}
+	public User createOrOverwriteUser(@NonNull MetaStorage storage) {
 		if (label == null) {
 			label = name;
 		}
-		user = new User(name, label);
-		return user;
-	}
-
-	public void registerForAuthorization(MetaStorage storage, boolean override) {
-		User user = this.getUser();
-		if(override) {			
-			storage.updateUser(user);
-		} else {
-			// Should throw an exception, if the user already existed
-			storage.addUser(user);
-		}
+		User user = new User(name, label, storage);
+		storage.updateUser(user);
 		for (String sPermission : permissions) {
-			user.addPermission(storage, new WildcardPermission(sPermission));
+			user.addPermission(new WildcardPermission(sPermission));
 		}
+		return user;
 	}
-
-	public boolean registerForAuthentication(UserManageable userManager, boolean override) {
-		if(override) {			
-			return userManager.updateUser(getUser(), credentials);
-		}
-		return userManager.addUser(getUser(), credentials);
+
+	@JsonIgnore
+	public UserId createId() {
+		return new UserId(name);
 	}
 }

backend/src/main/java/com/bakdata/conquery/apiv1/query/QueryDescription.java

@@ -3,6 +3,7 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal;
 import com.bakdata.conquery.io.cps.CPSBase;
 import com.bakdata.conquery.io.jackson.InternalOnly;
 import com.bakdata.conquery.models.auth.entities.User;
@@ -16,7 +17,6 @@
 import com.bakdata.conquery.models.identifiable.ids.NamespacedIdentifiable;
 import com.bakdata.conquery.models.query.QueryResolveContext;
 import com.bakdata.conquery.models.query.Visitable;
-import com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal;
 import com.bakdata.conquery.models.query.visitor.QueryVisitor;
 import com.bakdata.conquery.models.worker.DatasetRegistry;
 import com.bakdata.conquery.util.QueryUtils;

backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java

@@ -13,7 +13,6 @@
 import com.bakdata.conquery.io.cps.CPSTypeIdResolver;
 import com.bakdata.conquery.io.jackson.InternalOnly;
 import com.bakdata.conquery.io.jackson.Jackson;
-import com.bakdata.conquery.io.jackson.MutableInjectableValues;
 import com.bakdata.conquery.io.jersey.RESTServer;
 import com.bakdata.conquery.io.mina.BinaryJacksonCoder;
 import com.bakdata.conquery.io.mina.CQProtocolCodecFilter;
@@ -33,7 +32,6 @@
 import com.bakdata.conquery.models.messages.network.MessageToManagerNode;
 import com.bakdata.conquery.models.messages.network.NetworkMessageContext;
 import com.bakdata.conquery.models.worker.DatasetRegistry;
-import com.bakdata.conquery.models.worker.IdResolveContext;
 import com.bakdata.conquery.models.worker.Namespace;
 import com.bakdata.conquery.models.worker.Worker;
 import com.bakdata.conquery.resources.ResourcesProvider;
@@ -104,11 +102,13 @@ public ManagerNode(@NonNull String name) {
 	public void run(ConqueryConfig config, Environment environment) throws InterruptedException {
 		client = new JerseyClientBuilder(environment).using(config.getJerseyClient())
 				.build(getName());
+
+		// Instantiate DatasetRegistry and MetaStorage so they are ready for injection into the object mapper (API + Storage)
 		datasetRegistry = new DatasetRegistry(config.getCluster().getEntityBucketSize());
+		storage = new MetaStorage(datasetRegistry);
 
-		//inject datasets into the objectmapper
-		((MutableInjectableValues) environment.getObjectMapper().getInjectableValues())
-				.add(IdResolveContext.class, datasetRegistry);
+		datasetRegistry.injectInto(environment.getObjectMapper());
+		storage.injectInto(environment.getObjectMapper());
 
 
 		jobManager = new JobManager("ManagerNode", config.isFailOnError());
@@ -183,8 +183,9 @@ public void run(ConqueryConfig config, Environment environment) throws Interrupt
 	}
 
 	private void loadMetaStorage() {
-		log.info("Started meta storage");
-		storage = new MetaStorage(validator, config.getStorage(), datasetRegistry);
+		log.info("Opening MetaStorage");
+		storage.openStores(config.getStorage());
+		log.info("Loading MetaStorage");
 		storage.loadData();
 		log.info("MetaStorage loaded {}", storage);
 

backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java

@@ -8,6 +8,7 @@
 import javax.validation.Validator;
 
 import com.bakdata.conquery.io.jackson.Jackson;
+import com.bakdata.conquery.io.jackson.MutableInjectableValues;
 import com.bakdata.conquery.io.mina.BinaryJacksonCoder;
 import com.bakdata.conquery.io.mina.CQProtocolCodecFilter;
 import com.bakdata.conquery.io.mina.ChunkReader;
@@ -98,11 +99,14 @@ protected void run(Environment environment, Namespace namespace, ConqueryConfig
 
 		scheduler.scheduleAtFixedRate(this::reportJobManagerStatus, 30, 1, TimeUnit.SECONDS);
 
+
+		final ObjectMapper binaryMapper = config.configureObjectMapper(Jackson.BINARY_MAPPER);
+		((MutableInjectableValues) binaryMapper.getInjectableValues()).add(Validator.class, environment.getValidator());
+
 		workers = new Workers(
 				getConfig().getQueries().getExecutionPool(),
-				config.configureObjectMapper(Jackson.MAPPER),
-				config.configureObjectMapper(Jackson.BINARY_MAPPER)
-				, getConfig().getCluster().getEntityBucketSize()
+				binaryMapper,
+				getConfig().getCluster().getEntityBucketSize()
 		);
 
 		final Collection<WorkerStorage> workerStorages = config.getStorage().loadWorkerStorages();

backend/src/main/java/com/bakdata/conquery/commands/StandaloneCommand.java

@@ -18,6 +18,7 @@
 import io.dropwizard.setup.Bootstrap;
 import io.dropwizard.setup.Environment;
 import lombok.Getter;
+import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 import net.sourceforge.argparse4j.inf.Namespace;
 
@@ -130,6 +131,5 @@ protected void startStandalone(Environment environment, Namespace namespace, Con
 		log.debug("Starting REST Server");
 		ConqueryMDC.setLocation(null);
 		super.run(environment, namespace, config);
-		manager = conquery.getManager();
 	}
 }

backend/src/main/java/com/bakdata/conquery/io/jackson/Injectable.java

@@ -8,7 +8,10 @@
  */
 public interface Injectable {
 
-	default ObjectReader injectInto(ObjectReader reader) {
+	/**
+	 *	See {@link Injectable#injectIntoNew(ObjectMapper)} 
+	 */
+	default ObjectReader injectIntoNew(ObjectReader reader) {
 		// If is already MutableInjectable, add my values to other, else begin from scratch.
 		if (reader.getInjectableValues() instanceof MutableInjectableValues) {
 			return reader.with(inject(((MutableInjectableValues) reader.getInjectableValues()).copy()));
@@ -17,16 +20,35 @@ default ObjectReader injectInto(ObjectReader reader) {
 		return reader.with(inject(new MutableInjectableValues()));
 	}
 
-	default ObjectMapper injectInto(ObjectMapper mapper) {
+	/**
+	 * Creates a copy of the provided mapper and its injected values and adds the caller to the new copy 
+	 * @param mapper the blueprint mapper to use which remains untouched
+	 * @return a new mapper with this injected
+	 */
+	default ObjectMapper injectIntoNew(ObjectMapper mapper) {
 		// If is already MutableInjectable, add my values to other, else begin from scratch.
 
 		if (mapper.getInjectableValues() instanceof MutableInjectableValues) {
 			return mapper.copy()
 						 .setInjectableValues(inject(((MutableInjectableValues) mapper.getInjectableValues()).copy()));
 		}
+		// TODO unsure if overriding is expected here from the user
 		return mapper.copy()
 					 .setInjectableValues(inject(new MutableInjectableValues()));
 	}
 
+	/**
+	 * Injects this to the provided mapper and returns the mapper
+	 */
+	default ObjectMapper injectInto(ObjectMapper mapper) {
+		// If is already MutableInjectable, add my values to other, else begin from scratch.
+
+		if (mapper.getInjectableValues() instanceof MutableInjectableValues) {
+			mapper.setInjectableValues(inject(((MutableInjectableValues) mapper.getInjectableValues())));
+			return mapper;
+		}
+		throw new IllegalStateException("Cannot add additional injectables if the mapper does not provide MutableInjectableValues");
+	}
+
 	MutableInjectableValues inject(MutableInjectableValues values);
 }

backend/src/main/java/com/bakdata/conquery/io/jackson/PathParamInjector.java

@@ -30,7 +30,7 @@ public static class Modifier extends ObjectReaderModifier implements Injectable
 
 		@Override
 		public ObjectReader modify(EndpointConfigBase<?> endpoint, MultivaluedMap<String, String> httpHeaders, JavaType resultType, ObjectReader reader, JsonParser p) throws IOException {
-			return this.injectInto(reader);
+			return this.injectIntoNew(reader);
 		}
 
 		@Override

backend/src/main/java/com/bakdata/conquery/io/mina/BinaryJacksonCoder.java

@@ -31,7 +31,7 @@ public BinaryJacksonCoder(IdResolveContext datasets, Validator validator, Object
 			.writerFor(NetworkMessage.class)
 			.withView(InternalOnly.class);
 		this.reader = datasets
-				.injectInto(objectMapper.readerFor(NetworkMessage.class))
+				.injectIntoNew(objectMapper.readerFor(NetworkMessage.class))
 				.without(Feature.AUTO_CLOSE_SOURCE)
 				.withView(InternalOnly.class);
 	}

backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowFileProcessor.java

@@ -1,5 +1,13 @@
 package com.bakdata.conquery.io.result.arrow;
 
+import static com.bakdata.conquery.io.result.arrow.ResultArrowProcessor.getArrowResult;
+import static com.bakdata.conquery.resources.ResourceConstants.FILE_EXTENTION_ARROW_FILE;
+
+import java.nio.channels.Channels;
+
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+
 import com.bakdata.conquery.models.auth.entities.User;
 import com.bakdata.conquery.models.config.ConqueryConfig;
 import com.bakdata.conquery.models.datasets.Dataset;
@@ -10,13 +18,6 @@
 import org.apache.arrow.vector.dictionary.DictionaryProvider;
 import org.apache.arrow.vector.ipc.ArrowFileWriter;
 
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-import java.nio.channels.Channels;
-
-import static com.bakdata.conquery.io.result.arrow.ResultArrowProcessor.getArrowResult;
-import static com.bakdata.conquery.resources.ResourceConstants.FILE_EXTENTION_ARROW_FILE;
-
 @RequiredArgsConstructor
 public class ResultArrowFileProcessor {
 

backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java

@@ -8,7 +8,6 @@
 import java.util.Locale;
 import java.util.function.Function;
 
-import javax.ws.rs.WebApplicationException;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
 import javax.ws.rs.core.StreamingOutput;

backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowStreamProcessor.java

@@ -1,5 +1,11 @@
 package com.bakdata.conquery.io.result.arrow;
 
+import static com.bakdata.conquery.io.result.arrow.ResultArrowProcessor.getArrowResult;
+import static com.bakdata.conquery.resources.ResourceConstants.FILE_EXTENTION_ARROW_STREAM;
+
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+
 import com.bakdata.conquery.models.auth.entities.User;
 import com.bakdata.conquery.models.config.ConqueryConfig;
 import com.bakdata.conquery.models.datasets.Dataset;
@@ -10,12 +16,6 @@
 import org.apache.arrow.vector.dictionary.DictionaryProvider;
 import org.apache.arrow.vector.ipc.ArrowStreamWriter;
 
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-
-import static com.bakdata.conquery.io.result.arrow.ResultArrowProcessor.getArrowResult;
-import static com.bakdata.conquery.resources.ResourceConstants.FILE_EXTENTION_ARROW_STREAM;
-
 @RequiredArgsConstructor
 public class ResultArrowStreamProcessor {
 

backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java

@@ -1,13 +1,14 @@
 package com.bakdata.conquery.io.result.excel;
 
-import com.bakdata.conquery.io.result.ResultUtil;
 import static com.bakdata.conquery.io.result.ResultUtil.makeResponseWithFileName;
 
 import java.util.Locale;
 
+import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
 import javax.ws.rs.core.StreamingOutput;
 
+import com.bakdata.conquery.io.result.ResultUtil;
 import com.bakdata.conquery.models.auth.entities.User;
 import com.bakdata.conquery.models.auth.permissions.Ability;
 import com.bakdata.conquery.models.config.ConqueryConfig;
@@ -23,12 +24,6 @@
 import com.bakdata.conquery.util.io.ConqueryMDC;
 import lombok.RequiredArgsConstructor;
 
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.StreamingOutput;
-
-import static com.bakdata.conquery.io.result.ResultUtil.makeResponseWithFileName;
-
 @RequiredArgsConstructor
 public class ResultExcelProcessor {
 

backend/src/main/java/com/bakdata/conquery/io/storage/ConqueryStorage.java

@@ -5,12 +5,14 @@
 
 import javax.validation.Validator;
 
+import com.bakdata.conquery.models.config.StoreFactory;
 import com.bakdata.conquery.models.identifiable.CentralRegistry;
 
 public interface ConqueryStorage extends Closeable {
 
-	Validator getValidator();
 	CentralRegistry getCentralRegistry();
+
+	void openStores(StoreFactory storageFactory);
 
 	void loadData();
 

backend/src/main/java/com/bakdata/conquery/io/storage/IdentifiableCachedStore.java

@@ -18,11 +18,8 @@
 @Accessors(fluent=true) @Setter @Getter
 public class IdentifiableCachedStore<VALUE extends Identifiable<?>> extends IdentifiableStore<VALUE> {
 
-	public IdentifiableCachedStore(CentralRegistry centralRegistry, Store<IId<VALUE>, VALUE> store, Injectable... injectables) {
+	public IdentifiableCachedStore(CentralRegistry centralRegistry, Store<IId<VALUE>, VALUE> store) {
 		super(store, centralRegistry);
-		for(Injectable injectable : injectables) {
-			store.inject(injectable);
-		}
 	}
 
 	@Override

backend/src/main/java/com/bakdata/conquery/io/storage/IdentifiableStore.java

@@ -32,7 +32,6 @@ public abstract class IdentifiableStore<VALUE extends Identifiable<?>> extends K
 
     public IdentifiableStore(Store<IId<VALUE>, VALUE> store, CentralRegistry centralRegistry) {
         super(store);
-        store.inject(centralRegistry);
         this.centralRegistry = centralRegistry;
     }