From c48d1746f30b889d62957b465c51660f289d10c4 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 7 Sep 2021 18:51:20 +0200 Subject: [PATCH 01/35] inject storage into permission owner make store creators static dont double cache dictionaries inject meta storage into permission owners fixes tests Fix serialization test capsule storage for group and role handling fix compile errors remove usage of userish --- .../conquery/apiv1/FormConfigPatch.java | 1 - .../bakdata/conquery/apiv1/MetaDataPatch.java | 1 - .../conquery/apiv1/QueryProcessor.java | 8 +- .../conquery/apiv1/auth/ProtoUser.java | 38 +++--- .../apiv1/query/QueryDescription.java | 2 +- .../conquery/commands/StandaloneCommand.java | 1 - .../arrow/ResultArrowFileProcessor.java | 15 +-- .../io/result/arrow/ResultArrowProcessor.java | 1 - .../arrow/ResultArrowStreamProcessor.java | 12 +- .../io/result/excel/ResultExcelProcessor.java | 9 +- .../conquery/io/storage/IStoreInfo.java | 6 +- .../conquery/io/storage/MetaStorage.java | 16 ++- .../conquery/io/storage/StoreInfo.java | 20 +-- .../io/storage/xodus/stores/BigStore.java | 27 +---- .../xodus/stores/SerializingStore.java | 52 ++++---- .../storage/xodus/stores/SimpleStoreInfo.java | 6 +- .../io/storage/xodus/stores/XodusStore.java | 4 +- .../models/auth/AuthorizationController.java | 27 ++--- .../models/auth/AuthorizationHelper.java | 56 --------- ...UserAuthenticationManagementProcessor.java | 22 +++- .../models/auth/develop/DevAuthConfig.java | 4 +- .../conquery/models/auth/entities/Group.java | 16 +-- .../models/auth/entities/PermissionOwner.java | 61 +++++----- .../conquery/models/auth/entities/Role.java | 4 +- .../models/auth/entities/RoleOwner.java | 4 +- .../conquery/models/auth/entities/User.java | 10 +- .../oidc/IntrospectionDelegatingRealm.java | 8 +- .../auth/util/SinglePrincipalCollection.java | 2 +- .../conquery/models/config/StoreFactory.java | 7 +- .../models/config/XodusStoreFactory.java | 114 ++++++++---------- .../conquery/models/execution/Shareable.java | 5 +- .../models/forms/configs/FormConfig.java | 2 +- .../FormConfigProcessor.java | 10 +- .../frontendconfiguration/FormProcessor.java | 1 - .../models/query/ExecutionManager.java | 1 - .../admin/rest/AdminDatasetResource.java | 53 ++++---- .../resources/admin/rest/AdminProcessor.java | 38 +++--- .../resources/admin/rest/AdminResource.java | 1 + .../api/ResultArrowFileResource.java | 1 - .../api/ResultArrowStreamResource.java | 1 - .../resources/api/ResultExcelResource.java | 30 +++-- .../conquery/tasks/PermissionCleanupTask.java | 6 +- .../api/StoredQueriesProcessorTest.java | 9 +- .../api/form/config/FormConfigTest.java | 34 +++--- .../integration/DownloadLinkGeneration.java | 17 ++- .../integration/IntegrationTests.java | 2 + .../integration/common/IntegrationUtils.java | 2 +- .../integration/common/LoadingUtil.java | 4 +- .../tests/ConceptPermissionTest.java | 6 +- .../integration/tests/GroupHandlingTest.java | 24 ++-- .../tests/PermissionGroupHandlingTest.java | 20 +-- .../tests/PermissionRoleHandlingTest.java | 27 +++-- .../integration/tests/RestartTest.java | 21 ++-- .../integration/tests/ReusedQueryTest.java | 5 +- .../tests/RoleHandlingOnGroupTest.java | 18 +-- .../integration/tests/RoleHandlingTest.java | 20 +-- .../integration/tests/RoleUITest.java | 22 ++-- .../tests/SuperPermissionTest.java | 17 +-- .../conquery/integration/tests/TestUser.java | 5 +- .../io/jackson/serializer/IdRefrenceTest.java | 4 +- .../serializer/SerializationTestUtil.java | 14 ++- .../io/storage/xodus/stores/BigStoreTest.java | 4 +- .../stores/SerializingStoreDumpTest.java | 71 +++++------ .../conquery/models/SerializationTests.java | 42 ++++--- .../conquery/models/auth/CopyUserTest.java | 14 +-- .../IntrospectionDelegatingRealmTest.java | 14 +-- .../models/auth/LocalAuthRealmTest.java | 2 +- .../models/execution/DefaultLabelTest.java | 8 +- .../tasks/PermissionCleanupTaskTest.java | 21 ++-- .../util/NonPersistentStoreFactory.java | 6 +- .../conquery/util/support/TestConquery.java | 6 +- .../util/support/TestLoggingFactory.java | 1 + 72 files changed, 563 insertions(+), 600 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/FormConfigPatch.java b/backend/src/main/java/com/bakdata/conquery/apiv1/FormConfigPatch.java index 9ab3f8d5f7..a76dd99467 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/FormConfigPatch.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/FormConfigPatch.java @@ -3,7 +3,6 @@ import java.util.function.Consumer; import com.bakdata.conquery.io.storage.MetaStorage; -import com.bakdata.conquery.models.auth.AuthorizationHelper; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.auth.permissions.Ability; import com.bakdata.conquery.models.forms.configs.FormConfig; diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/MetaDataPatch.java b/backend/src/main/java/com/bakdata/conquery/apiv1/MetaDataPatch.java index 443700c627..8989e65509 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/MetaDataPatch.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/MetaDataPatch.java @@ -6,7 +6,6 @@ import java.util.function.Consumer; import com.bakdata.conquery.io.storage.MetaStorage; -import com.bakdata.conquery.models.auth.AuthorizationHelper; import com.bakdata.conquery.models.auth.entities.Group; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.auth.permissions.Ability; diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java index 5e43bc644a..2e73e0f955 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java @@ -112,7 +112,7 @@ public ManagedExecution postQuery(Dataset dataset, QueryDescription query, Us { final Optional executionId = visitors.getInstance(QueryUtils.OnlyReusingChecker.class).getOnlyReused(); - final Optional> execution = executionId.map(id -> tryReuse(query, id, datasetRegistry, config, executionManager, user)); + final Optional> execution = executionId.map(id -> tryReuse(query, id, datasetRegistry, config, executionManager, storage.getUser(user.getId()))); if (execution.isPresent()) { return execution.get(); @@ -121,7 +121,7 @@ public ManagedExecution postQuery(Dataset dataset, QueryDescription query, Us // Run the query on behalf of the user - ManagedExecution mq = executionManager.runQuery(datasetRegistry, query, user, dataset, config); + ManagedExecution mq = executionManager.runQuery(datasetRegistry, query, storage.getUser(user.getId()), dataset, config); if (query instanceof Query) { translateToOtherDatasets(dataset, query, user, mq); @@ -288,7 +288,7 @@ private void translateToOtherDatasets(Dataset dataset, QueryDescription query, U Query translated = QueryTranslator.replaceDataset(datasetRegistry, translateable, targetDataset); targetNamespace.getExecutionManager() - .createQuery(datasetRegistry, translated, mq.getQueryId(), user, targetDataset); + .createQuery(datasetRegistry, translated, mq.getQueryId(), storage.getUser(user.getId()), targetDataset); } catch (Exception e) { log.trace("Could not translate Query[{}] to Dataset[{}]", mq.getId(), targetDataset.getId(), e); @@ -396,7 +396,7 @@ public ExternalUploadResult uploadEntities(User user, Dataset dataset, ExternalU // We only create the Query, really no need to execute it as it's only useful for composition. final ManagedQuery execution = ((ManagedQuery) datasetRegistry.get(dataset.getId()).getExecutionManager() - .createExecution(datasetRegistry, query, user, dataset)); + .createExecution(datasetRegistry, query, storage.getUser(user.getId()), dataset)); execution.setLastResultCount((long) statistic.getResolved().size()); diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/auth/ProtoUser.java b/backend/src/main/java/com/bakdata/conquery/apiv1/auth/ProtoUser.java index 1b3cdba149..b8d343f65c 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/auth/ProtoUser.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/auth/ProtoUser.java @@ -2,6 +2,7 @@ import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.Set; import javax.validation.Valid; @@ -13,9 +14,10 @@ import com.bakdata.conquery.models.auth.basic.LocalAuthenticationRealm; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.auth.permissions.WildcardPermission; -import com.fasterxml.jackson.annotation.JsonIgnore; +import com.bakdata.conquery.models.identifiable.ids.specific.UserId; import lombok.Builder; import lombok.Getter; +import lombok.NonNull; /** * Container class for holding information about initial users. @@ -45,39 +47,27 @@ public class ProtoUser { @NotNull @Valid private List credentials = Collections.emptyList(); - - @JsonIgnore - // Let this be ignored by the builder - private User user = null; - public User getUser() { - if(user != null) { - return user; + public Optional getUser(@NonNull MetaStorage storage, boolean override) { + User user = storage.getUser(new UserId(name)); + if (!override) { + return Optional.ofNullable(user); } if (label == null) { label = name; } - user = new User(name, label); - return user; - } - - public void registerForAuthorization(MetaStorage storage, boolean override) { - User user = this.getUser(); - if(override) { - storage.updateUser(user); - } else { - // Should throw an exception, if the user already existed - storage.addUser(user); - } + user = new User(name, label, storage); + storage.updateUser(user); for (String sPermission : permissions) { - user.addPermission(storage, new WildcardPermission(sPermission)); + user.addPermission(new WildcardPermission(sPermission)); } + return Optional.of(user); } - public boolean registerForAuthentication(UserManageable userManager, boolean override) { + public static boolean registerForAuthentication(UserManageable userManager, User user, List credentials, boolean override) { if(override) { - return userManager.updateUser(getUser(), credentials); + return userManager.updateUser(user, credentials); } - return userManager.addUser(getUser(), credentials); + return userManager.addUser(user, credentials); } } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/QueryDescription.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/QueryDescription.java index 1a0749d09c..8d68074cce 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/QueryDescription.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/QueryDescription.java @@ -3,6 +3,7 @@ import java.util.Set; import java.util.stream.Collectors; +import com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal; import com.bakdata.conquery.io.cps.CPSBase; import com.bakdata.conquery.io.jackson.InternalOnly; import com.bakdata.conquery.models.auth.entities.User; @@ -16,7 +17,6 @@ import com.bakdata.conquery.models.identifiable.ids.NamespacedIdentifiable; import com.bakdata.conquery.models.query.QueryResolveContext; import com.bakdata.conquery.models.query.Visitable; -import com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal; import com.bakdata.conquery.models.query.visitor.QueryVisitor; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.util.QueryUtils; diff --git a/backend/src/main/java/com/bakdata/conquery/commands/StandaloneCommand.java b/backend/src/main/java/com/bakdata/conquery/commands/StandaloneCommand.java index a17233d244..54683dd211 100644 --- a/backend/src/main/java/com/bakdata/conquery/commands/StandaloneCommand.java +++ b/backend/src/main/java/com/bakdata/conquery/commands/StandaloneCommand.java @@ -130,6 +130,5 @@ protected void startStandalone(Environment environment, Namespace namespace, Con log.debug("Starting REST Server"); ConqueryMDC.setLocation(null); super.run(environment, namespace, config); - manager = conquery.getManager(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowFileProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowFileProcessor.java index 13fc959c31..0ed7142f2f 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowFileProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowFileProcessor.java @@ -1,5 +1,13 @@ package com.bakdata.conquery.io.result.arrow; +import static com.bakdata.conquery.io.result.arrow.ResultArrowProcessor.getArrowResult; +import static com.bakdata.conquery.resources.ResourceConstants.FILE_EXTENTION_ARROW_FILE; + +import java.nio.channels.Channels; + +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.datasets.Dataset; @@ -10,13 +18,6 @@ import org.apache.arrow.vector.dictionary.DictionaryProvider; import org.apache.arrow.vector.ipc.ArrowFileWriter; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import java.nio.channels.Channels; - -import static com.bakdata.conquery.io.result.arrow.ResultArrowProcessor.getArrowResult; -import static com.bakdata.conquery.resources.ResourceConstants.FILE_EXTENTION_ARROW_FILE; - @RequiredArgsConstructor public class ResultArrowFileProcessor { diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java index 7f744da477..f48a9400ed 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java @@ -8,7 +8,6 @@ import java.util.Locale; import java.util.function.Function; -import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import javax.ws.rs.core.StreamingOutput; diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowStreamProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowStreamProcessor.java index 359fbfcd71..c65cb5a770 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowStreamProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowStreamProcessor.java @@ -1,5 +1,11 @@ package com.bakdata.conquery.io.result.arrow; +import static com.bakdata.conquery.io.result.arrow.ResultArrowProcessor.getArrowResult; +import static com.bakdata.conquery.resources.ResourceConstants.FILE_EXTENTION_ARROW_STREAM; + +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.datasets.Dataset; @@ -10,12 +16,6 @@ import org.apache.arrow.vector.dictionary.DictionaryProvider; import org.apache.arrow.vector.ipc.ArrowStreamWriter; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; - -import static com.bakdata.conquery.io.result.arrow.ResultArrowProcessor.getArrowResult; -import static com.bakdata.conquery.resources.ResourceConstants.FILE_EXTENTION_ARROW_STREAM; - @RequiredArgsConstructor public class ResultArrowStreamProcessor { diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java index cb44e0024c..5fda394499 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java @@ -1,13 +1,14 @@ package com.bakdata.conquery.io.result.excel; -import com.bakdata.conquery.io.result.ResultUtil; import static com.bakdata.conquery.io.result.ResultUtil.makeResponseWithFileName; import java.util.Locale; +import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import javax.ws.rs.core.StreamingOutput; +import com.bakdata.conquery.io.result.ResultUtil; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.auth.permissions.Ability; import com.bakdata.conquery.models.config.ConqueryConfig; @@ -23,12 +24,6 @@ import com.bakdata.conquery.util.io.ConqueryMDC; import lombok.RequiredArgsConstructor; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import javax.ws.rs.core.StreamingOutput; - -import static com.bakdata.conquery.io.result.ResultUtil.makeResponseWithFileName; - @RequiredArgsConstructor public class ResultExcelProcessor { diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/IStoreInfo.java b/backend/src/main/java/com/bakdata/conquery/io/storage/IStoreInfo.java index 3a998c8567..45473dd58d 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/IStoreInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/IStoreInfo.java @@ -1,8 +1,8 @@ package com.bakdata.conquery.io.storage; -public interface IStoreInfo { +public interface IStoreInfo { String getName(); - Class getValueType(); - Class getKeyType(); + Class getKeyType(); + Class getValueType(); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java index 9e785faad2..d423a33367 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java @@ -2,10 +2,11 @@ import java.io.IOException; import java.util.Collection; -import java.util.List; import javax.validation.Validator; +import com.bakdata.conquery.io.jackson.Injectable; +import com.bakdata.conquery.io.jackson.MutableInjectableValues; import com.bakdata.conquery.models.auth.entities.Group; import com.bakdata.conquery.models.auth.entities.Role; import com.bakdata.conquery.models.auth.entities.User; @@ -24,7 +25,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -public class MetaStorage implements ConqueryStorage{ +public class MetaStorage implements ConqueryStorage, Injectable { private IdentifiableStore> executions; @@ -45,9 +46,9 @@ public MetaStorage(Validator validator, StoreFactory storageFactory, DatasetRegi this.validator = validator; - authUser = storageFactory.createUserStore(centralRegistry, "meta"); - authRole = storageFactory.createRoleStore(centralRegistry, "meta"); - authGroup = storageFactory.createGroupStore(centralRegistry, "meta"); + authUser = storageFactory.createUserStore(centralRegistry, "meta", this); + authRole = storageFactory.createRoleStore(centralRegistry, "meta", this); + authGroup = storageFactory.createGroupStore(centralRegistry, "meta", this); // Executions depend on users executions = storageFactory.createExecutionsStore(centralRegistry, datasetRegistry, "meta"); formConfigs = storageFactory.createFormConfigStore(centralRegistry, datasetRegistry, "meta"); @@ -195,4 +196,9 @@ public void close() throws IOException { authRole.close(); authGroup.close(); } + + @Override + public MutableInjectableValues inject(MutableInjectableValues values) { + return values.add(MetaStorage.class, this); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/StoreInfo.java b/backend/src/main/java/com/bakdata/conquery/io/storage/StoreInfo.java index 98a27ab30c..215257fc9b 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/StoreInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/StoreInfo.java @@ -2,6 +2,7 @@ import com.bakdata.conquery.io.jackson.Injectable; import com.bakdata.conquery.io.storage.xodus.stores.CachedStore; +import com.bakdata.conquery.io.storage.xodus.stores.SimpleStoreInfo; import com.bakdata.conquery.io.storage.xodus.stores.SingletonStore; import com.bakdata.conquery.models.auth.entities.Group; import com.bakdata.conquery.models.auth.entities.Role; @@ -50,7 +51,7 @@ @RequiredArgsConstructor @Getter @ToString(of = {"name", "keyType", "valueType"}) -public enum StoreInfo implements IStoreInfo { +public enum StoreInfo { DATASET(Dataset.class, Boolean.class), ID_MAPPING(EntityIdMap.class, Boolean.class), NAMESPACES(DatasetRegistry.class, Boolean.class), @@ -74,10 +75,14 @@ public enum StoreInfo implements IStoreInfo { private final Class valueType; private final Class keyType; + public , CLASS_V extends Class> IStoreInfo storeInfo(){ + return new SimpleStoreInfo(getName(),(CLASS_K) getKeyType(), (CLASS_V) getValueType()); + } + /** * Store for identifiable values, with injectors. Store is also cached. */ - public > DirectIdentifiableStore identifiable(Store, T> baseStore, CentralRegistry centralRegistry, Injectable... injectables) { + public static > DirectIdentifiableStore identifiable(Store, T> baseStore, CentralRegistry centralRegistry, Injectable... injectables) { for (Injectable injectable : injectables) { baseStore.inject(injectable); @@ -91,21 +96,21 @@ public > DirectIdentifiableStore identifiable(Store /** * Store for identifiable values, without injectors. Store is also cached. */ - public > DirectIdentifiableStore identifiable(Store, T> baseStore, CentralRegistry centralRegistry) { + public static > DirectIdentifiableStore identifiable(Store, T> baseStore, CentralRegistry centralRegistry) { return identifiable(baseStore, centralRegistry, new SingletonNamespaceCollection(centralRegistry)); } /** * General Key-Value store with caching. */ - public CachedStore cached(Store baseStore) { + public static CachedStore cached(Store baseStore) { return new CachedStore<>(baseStore); } /** * Identifiable store, that lazy registers items in the central registry. */ - public > IdentifiableCachedStore identifiableCachedStore(Store baseStore, CentralRegistry centralRegistry) { + public static > IdentifiableCachedStore identifiableCachedStore(Store,T> baseStore, CentralRegistry centralRegistry) { return new IdentifiableCachedStore(centralRegistry, baseStore); } @@ -113,12 +118,11 @@ public > IdentifiableCachedStore identifiableCached /** * Store holding a single value. */ - public SingletonStore singleton(Store baseStore, Injectable... injectables) { + public static SingletonStore singleton(Store baseStore, Injectable... injectables) { return new SingletonStore<>(baseStore, injectables); } - @Override - public String getName() { + private String getName() { return name(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/BigStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/BigStore.java index 8c7a9bfe1c..25ce8efafe 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/BigStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/BigStore.java @@ -22,8 +22,8 @@ import com.bakdata.conquery.io.jackson.Injectable; import com.bakdata.conquery.io.mina.ChunkingOutputStream; +import com.bakdata.conquery.io.storage.IStoreInfo; import com.bakdata.conquery.io.storage.Store; -import com.bakdata.conquery.io.storage.StoreInfo; import com.bakdata.conquery.io.storage.xodus.stores.SerializingStore.IterationStatistic; import com.bakdata.conquery.models.config.XodusStoreFactory; import com.fasterxml.jackson.annotation.JsonCreator; @@ -48,48 +48,33 @@ public class BigStore implements Store, Closeable { private final ObjectWriter valueWriter; private ObjectReader valueReader; - private final StoreInfo storeInfo; + private final IStoreInfo storeInfo; @Getter @Setter private int chunkByteSize; - public BigStore(XodusStoreFactory config, Validator validator, Environment env, StoreInfo storeInfo, Collection openStores, Consumer envCloseHook, Consumer envRemoveHook, ObjectMapper mapper) { + public BigStore(XodusStoreFactory config, Validator validator, Environment env, IStoreInfo storeInfo, Collection openStores, Consumer envCloseHook, Consumer envRemoveHook, ObjectMapper mapper) { this.storeInfo = storeInfo; // Recommendation by the author of Xodus is to have logFileSize at least be 4 times the biggest file size. this.chunkByteSize = Ints.checkedCast(config.getXodus().getLogFileSize().toBytes() / 4L); - final SimpleStoreInfo metaStoreInfo = new SimpleStoreInfo( - storeInfo.getName() + "_META", - storeInfo.getKeyType(), - BigStoreMetaKeys.class - ); - metaStore = new SerializingStore<>( - config, new XodusStore(env, storeInfo.getName() + "_META", openStores, envCloseHook, envRemoveHook), validator, - metaStoreInfo, mapper, - (Class) storeInfo.getKeyType(), - BigStoreMetaKeys.class - ); + storeInfo.getKeyType(), + BigStoreMetaKeys.class, config.isValidateOnWrite(), config.isRemoveUnreadableFromStore(), config.getUnreadableDataDumpDirectory() - final SimpleStoreInfo dataStoreInfo = new SimpleStoreInfo( - storeInfo.getName() + "_DATA", - UUID.class, - byte[].class ); dataStore = new SerializingStore<>( - config, new XodusStore(env, storeInfo.getName() + "_DATA", openStores, envCloseHook, envRemoveHook), validator, - dataStoreInfo, mapper, UUID.class, - byte[].class + byte[].class, config.isValidateOnWrite(), config.isRemoveUnreadableFromStore(), config.getUnreadableDataDumpDirectory() ); diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java index f1394e5861..cb74781f7e 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java @@ -9,13 +9,13 @@ import java.util.Optional; import java.util.function.Function; import java.util.function.Supplier; +import java.util.logging.Logger; import javax.validation.Validator; import com.bakdata.conquery.io.jackson.Injectable; import com.bakdata.conquery.io.jackson.Jackson; import com.bakdata.conquery.io.jackson.JacksonUtil; -import com.bakdata.conquery.io.storage.IStoreInfo; import com.bakdata.conquery.io.storage.Store; import com.bakdata.conquery.models.config.XodusStoreFactory; import com.bakdata.conquery.models.exceptions.ValidatorHelper; @@ -82,11 +82,6 @@ public class SerializingStore implements Store { */ private final Class valueType; - /** - * Description of the store. - */ - private final IStoreInfo storeInfo; - /** * Validate elements on write */ @@ -103,11 +98,18 @@ public class SerializingStore implements Store { private final ObjectMapper objectMapper; @SuppressWarnings("unchecked") - public , CLASS_V extends Class> SerializingStore(XodusStoreFactory config, XodusStore store, Validator validator, IStoreInfo storeInfo, ObjectMapper objectMapper, CLASS_K keyType, CLASS_V valueType) { - this.storeInfo = storeInfo; + public , CLASS_V extends Class> SerializingStore(XodusStore store, + Validator validator, + ObjectMapper objectMapper, + CLASS_K keyType, + CLASS_V valueType, + boolean validateOnWrite, + boolean removeUnreadableFromStore, + File unreadableDataDumpDirectory, + Injectable ... injectables) { this.store = store; this.validator = validator; - this.validateOnWrite = config.isValidateOnWrite(); + this.validateOnWrite = validateOnWrite; this.valueType = valueType; @@ -116,25 +118,25 @@ public , CLASS_V extends Class> SerializingSto valueWriter = objectMapper.writerFor(this.valueType); valueReader = objectMapper.readerFor(this.valueType); + for (Injectable injectable : injectables) { + valueReader = injectable.injectInto(valueReader); + } keyWriter = objectMapper.writerFor(keyType); keyReader = objectMapper.readerFor(keyType); - - removeUnreadablesFromUnderlyingStore = config.isRemoveUnreadableFromStore(); - + + removeUnreadablesFromUnderlyingStore = removeUnreadableFromStore; + + unreadableValuesDumpDir = unreadableDataDumpDirectory; // Prepare dump directory if there is one set in the config - Optional dumpUnreadable = config.getUnreadableDataDumpDirectory(); - if(dumpUnreadable.isPresent()) { - unreadableValuesDumpDir = dumpUnreadable.get(); - if(!unreadableValuesDumpDir.exists()) { - unreadableValuesDumpDir.mkdirs(); + if(unreadableValuesDumpDir != null ) { + if(!unreadableValuesDumpDir.exists() && unreadableValuesDumpDir.mkdirs()) { + log.info("Created dump directory for unreadable values: {}", unreadableValuesDumpDir.getAbsolutePath()); } else if(!unreadableValuesDumpDir.isDirectory()) { throw new IllegalArgumentException(String.format("The provided path points to an existing file which is not a directory. Was: %s", unreadableValuesDumpDir.getAbsolutePath())); } - } else { - unreadableValuesDumpDir = null; } } @@ -157,7 +159,7 @@ public VALUE get(KEY key) { return readValue(binValue); } catch (Exception e) { if(unreadableValuesDumpDir != null) { - dumpToFile(binValue, key.toString(), unreadableValuesDumpDir, storeInfo.getName(), objectMapper); + dumpToFile(binValue, key.toString(), unreadableValuesDumpDir, store.getName(), objectMapper); } if(removeUnreadablesFromUnderlyingStore) { remove(key); @@ -221,7 +223,7 @@ public IterationStatistic forEach(StoreEntryConsumer consumer) { log.debug( String.format( "While processing store %s:\n\tEntries processed:\t%d\n\tKey read failure:\t%d (%.2f%%)\n\tValue read failure:\t%d (%.2f%%)", - this.storeInfo.getName(), + store.getName(), total, result.getFailedKeys(), total > 0 ? (float) result.getFailedKeys() / total * 100 : 0, @@ -230,7 +232,7 @@ public IterationStatistic forEach(StoreEntryConsumer consumer) { // Remove corrupted entries from the store if configured so if (removeUnreadablesFromUnderlyingStore) { - log.warn("Removing {} unreadable elements from the store {}.", unreadables.size(), storeInfo.getName()); + log.warn("Removing {} unreadable elements from the store {}.", unreadables.size(), store.getName()); unreadables.forEach(store::remove); } return result; @@ -251,13 +253,13 @@ private TYPE getDeserializedAndDumpFailed(ByteIterable serial, Function implements IStoreInfo { private final String name; - private final Class keyType; - private final Class valueType; + private final Class keyType; + private final Class valueType; } diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/XodusStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/XodusStore.java index 31eefb4531..4107289dce 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/XodusStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/XodusStore.java @@ -13,6 +13,7 @@ import jetbrains.exodus.env.Environment; import jetbrains.exodus.env.Store; import jetbrains.exodus.env.StoreConfig; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -23,7 +24,8 @@ public class XodusStore { private final Collection openStores; private final Consumer envCloseHook; private final Consumer envRemoveHook; - private String name; + @Getter + private final String name; public XodusStore(Environment env, String name, Collection openStoresInEnv, Consumer envCloseHook, Consumer envRemoveHook) { // Arbitrary duration that is strictly shorter than the timeout to not get interrupted by StuckTxMonitor diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationController.java b/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationController.java index 15ef599c65..8ba051408d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationController.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationController.java @@ -1,10 +1,6 @@ package com.bakdata.conquery.models.auth; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.UUID; +import java.util.*; import java.util.stream.Collectors; import com.bakdata.conquery.apiv1.auth.ProtoUser; @@ -88,6 +84,9 @@ public AuthorizationController(MetaStorage storage, AuthorizationConfig authoriz authenticator.setAuthenticationStrategy(new FirstSuccessfulStrategy()); registerStaticSecurityManager(); + + // Register initial users for authorization and authentication (if the realm is able to) + initializeAuthConstellation(authorizationConfig, realms, storage); } public void externalInit(ManagerNode manager, List authenticationRealmFactories) { @@ -111,8 +110,6 @@ public void externalInit(ManagerNode manager, List a public void start() throws Exception { // Call Shiros init on all realms LifecycleUtils.init(realms); - // Register initial users for authorization and authentication (if the realm is able to) - initializeAuthConstellation(authorizationConfig, realms, storage); } @Override @@ -140,12 +137,14 @@ public void registerStaticSecurityManager() { */ private static void initializeAuthConstellation(@NonNull AuthorizationConfig config, @NonNull List realms, @NonNull MetaStorage storage) { for (ProtoUser pUser : config.getInitialUsers()) { - pUser.registerForAuthorization(storage, true); - for (Realm realm : realms) { - if (realm instanceof UserManageable) { - pUser.registerForAuthentication((UserManageable) realm, true); + final Optional user = pUser.getUser(storage, true); + user.ifPresent(u -> { + for (Realm realm : realms) { + if (realm instanceof UserManageable) { + ProtoUser.registerForAuthentication((UserManageable) realm, u, pUser.getCredentials(), true); + } } - } + }); } } @@ -194,9 +193,9 @@ public static User flatCopyUser(@NonNull User originUser, String namePrefix, @No ); // Create copied user - User copy = new User(name, originUser.getLabel()); + User copy = new User(name, originUser.getLabel(), storage); storage.addUser(copy); - copy.setPermissions(storage, copiedPermission); + copy.updatePermissions(copiedPermission); return copy; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationHelper.java b/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationHelper.java index c4db3b27d0..a0fa099ef7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationHelper.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationHelper.java @@ -1,9 +1,7 @@ package com.bakdata.conquery.models.auth; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; -import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -14,16 +12,12 @@ import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.models.auth.entities.Group; -import com.bakdata.conquery.models.auth.entities.PermissionOwner; import com.bakdata.conquery.models.auth.entities.Role; import com.bakdata.conquery.models.auth.entities.RoleOwner; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.auth.permissions.Ability; -import com.bakdata.conquery.models.auth.permissions.AdminPermission; -import com.bakdata.conquery.models.auth.permissions.Authorized; import com.bakdata.conquery.models.auth.permissions.ConqueryPermission; import com.bakdata.conquery.models.datasets.Dataset; -import com.bakdata.conquery.models.execution.Owned; import com.bakdata.conquery.models.identifiable.ids.NamespacedIdentifiable; import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; import com.bakdata.conquery.models.identifiable.ids.specific.RoleId; @@ -33,7 +27,6 @@ import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Multimap; import com.google.common.collect.Sets; -import it.unimi.dsi.fastutil.booleans.BooleanArrayList; import lombok.NonNull; import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; @@ -47,26 +40,6 @@ @UtilityClass public class AuthorizationHelper { - /** - * Utility function to add a permission to a subject (e.g {@link User}). - * @param owner The subject to own the new permission. - * @param permission The permission to add. - * @param storage A storage where the permission are added for persistence. - */ - public static void addPermission(@NonNull PermissionOwner owner, @NonNull ConqueryPermission permission, @NonNull MetaStorage storage) { - owner.addPermission(storage, permission); - } - - /** - * Utility function to remove a permission from a subject (e.g {@link User}). - * @param owner The subject to own the new permission. - * @param permission The permission to remove. - * @param storage A storage where the permission is removed from. - */ - public static void removePermission(@NonNull PermissionOwner owner, @NonNull Permission permission, @NonNull MetaStorage storage) { - owner.removePermission(storage, permission); - } - public static List getGroupsOf(@NonNull User user, @NonNull MetaStorage storage){ List userGroups = new ArrayList<>(); @@ -153,35 +126,6 @@ public static Multimap getEffectiveUserPermissions(U return mappedPerms; } - - public static void addRoleTo(MetaStorage storage, Role role, RoleOwner owner) { - owner.addRole(storage, role); - log.trace("Added role {} to {}", role, owner); - } - - public static void deleteRoleFrom(MetaStorage storage, RoleOwner owner, Role role) { - - owner.removeRole(storage, role); - - log.trace("Deleted role {} from {}", role, owner); - } - - public static void deleteRole(MetaStorage storage, Role role) { - log.info("Deleting {}", role); - - for (User user : storage.getAllUsers()) { - user.removeRole(storage, role); - } - - for (Group group : storage.getAllGroups()) { - group.removeRole(storage, role); - } - - storage.removeRole(role.getId()); - } - - - public static List getUsersByRole(MetaStorage storage, Role role) { return storage.getAllUsers().stream().filter(u -> u.getRoles().contains(role.getId())).collect(Collectors.toList()); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/basic/UserAuthenticationManagementProcessor.java b/backend/src/main/java/com/bakdata/conquery/models/auth/basic/UserAuthenticationManagementProcessor.java index 39eee40aa4..d326fc3d33 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/basic/UserAuthenticationManagementProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/basic/UserAuthenticationManagementProcessor.java @@ -3,10 +3,13 @@ import com.bakdata.conquery.apiv1.auth.ProtoUser; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.models.auth.entities.User; +import com.bakdata.conquery.models.identifiable.ids.specific.UserId; import com.bakdata.conquery.resources.admin.rest.UserAuthenticationManagementResource; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import java.util.Optional; + /** * Business logic for the {@link UserAuthenticationManagementResource}. */ @@ -19,18 +22,25 @@ public class UserAuthenticationManagementProcessor { public boolean addUser(ProtoUser pUser) { // Throws an exception if it would override the existing user - pUser.registerForAuthorization(storage, false); - log.trace("Added the user {} to the authorization storage", pUser.getUser().getId()); - if(pUser.registerForAuthentication(realm, false)) { - log.trace("Added the user {} to the realm {}", pUser.getUser().getId(), realm.getName()); + final Optional optUser = pUser.getUser(storage,false); + if (optUser.isEmpty()){ + log.warn("Unable to add new user {}. Probably already existed.", pUser); + return false; + } + final User user = optUser.get(); + final UserId id = user.getId(); + log.trace("Added the user {} to the authorization storage", id); + if(ProtoUser.registerForAuthentication(realm, user, pUser.getCredentials(),false)) { + log.trace("Added the user {} to the realm {}", id, realm.getName()); return true; } - log.trace("Failed to add added the user {} to the realm {}", pUser.getUser().getId(), realm.getName()); + log.trace("Failed to add added the user {} to the realm {}", id, realm.getName()); return false; } public boolean updateUser(ProtoUser pUser) { - return pUser.registerForAuthentication(realm, false); + final Optional user = pUser.getUser(storage, true); + return user.map(u -> ProtoUser.registerForAuthentication(realm, u,pUser.getCredentials(),false)).orElse(false); } public void remove(User user) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/develop/DevAuthConfig.java b/backend/src/main/java/com/bakdata/conquery/models/auth/develop/DevAuthConfig.java index c2d495617e..26899fedda 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/develop/DevAuthConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/develop/DevAuthConfig.java @@ -17,8 +17,8 @@ public class DevAuthConfig implements AuthenticationRealmFactory { @Override public ConqueryAuthenticationRealm createRealm(ManagerNode managerNode) { - User defaultUser = Objects.requireNonNull(managerNode.getConfig() - .getAuthorizationRealms().getInitialUsers().get(0).getUser(), "There must be at least one initial user configured."); + User defaultUser = managerNode.getConfig() + .getAuthorizationRealms().getInitialUsers().get(0).getUser(managerNode.getStorage(), true).orElseThrow(() -> new IllegalStateException("There must be at least one initial user configured.")); managerNode.getAuthController().getAuthenticationFilter().registerTokenExtractor(new UserIdTokenExtractor(defaultUser)); diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Group.java b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Group.java index 83f080fce7..b0d95b3e19 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Group.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Group.java @@ -25,8 +25,8 @@ public class Group extends PermissionOwner implements RoleOwner { @JsonProperty private Set roles = Collections.synchronizedSet(new HashSet<>()); - public Group(String name, String label) { - super(name, label); + public Group(String name, String label, MetaStorage storage) { + super(name, label, storage); } @Override @@ -39,16 +39,16 @@ public GroupId createId() { return new GroupId(name); } - public void addMember(MetaStorage storage, User user) { + public synchronized void addMember(User user) { if(members.add(user.getId())) { log.trace("Added user {} to group {}", user.getId(), getId()); updateStorage(storage); } } - public void removeMember(MetaStorage storage, User user) { - if(members.remove(user.getId())) { - log.trace("Removed user {} from group {}", user.getId(), getId()); + public synchronized void removeMember(User user) { + if (members.remove(user.getId())) { + log.trace("Removed user {} from group {}", user.getId(), getId()); updateStorage(storage); } } @@ -61,14 +61,14 @@ public Set getMembers() { return Collections.unmodifiableSet(members); } - public void addRole(MetaStorage storage, Role role) { + public synchronized void addRole(Role role) { if (roles.add(role.getId())) { log.trace("Added role {} to group {}", role.getId(), getId()); updateStorage(storage); } } - public void removeRole(MetaStorage storage, Role role) { + public synchronized void removeRole(Role role) { if (roles.remove(role.getId())) { log.trace("Removed role {} from group {}", role.getId(), getId()); updateStorage(storage); diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/PermissionOwner.java b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/PermissionOwner.java index 1c1019f4e0..052957a59b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/PermissionOwner.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/PermissionOwner.java @@ -12,6 +12,9 @@ import com.bakdata.conquery.models.auth.permissions.ConqueryPermission; import com.bakdata.conquery.models.identifiable.IdentifiableImpl; import com.bakdata.conquery.models.identifiable.ids.specific.PermissionOwnerId; +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.OptBoolean; import com.google.common.collect.ImmutableSet; import lombok.AccessLevel; import lombok.EqualsAndHashCode; @@ -57,71 +60,67 @@ public abstract class PermissionOwner permissions = new HashSet<>(); + @JacksonInject(useInput = OptBoolean.FALSE) + @NonNull + @EqualsAndHashCode.Exclude + protected MetaStorage storage; + - public PermissionOwner(String name, String label) { + public PermissionOwner(String name, String label, MetaStorage storage) { this.name = name; this.label = label; + this.storage = storage; } /** * Adds permissions to the owner object and to the persistent storage. * - * @param storage A storage where the permission are added for persistence. * @param permissions The permissions to add. * @return Returns the added Permission */ - public boolean addPermissions(MetaStorage storage, Set permissions) { - boolean ret = false; - synchronized (this) { - this.permissions = ImmutableSet - .builder() - .addAll(this.permissions) - .addAll(permissions) - .build(); - updateStorage(storage); - } - return ret; + public synchronized void addPermissions(Set permissions) { + this.permissions = ImmutableSet + .builder() + .addAll(this.permissions) + .addAll(permissions) + .build(); + updateStorage(this.storage); } - public boolean addPermission(MetaStorage storage, ConqueryPermission permission) { - boolean ret = false; - synchronized (this) { - this.permissions = ImmutableSet - .builder() - .addAll(this.permissions) - .add(permission) - .build(); - updateStorage(storage); - } - return ret; + public synchronized void addPermission(ConqueryPermission permission) { + this.permissions = ImmutableSet + .builder() + .addAll(this.permissions) + .add(permission) + .build(); + updateStorage(this.storage); } /** * Removes permissions from the owner object and from the persistent storage. * - * @param storage A storage where the permission are saved for persistence. * @param permissions The permission to remove. * @return Returns the added Permission */ - public boolean removePermissions(MetaStorage storage, Set permissions) { + public boolean removePermissions(Set permissions) { boolean ret = false; synchronized (this) { Set newSet = new HashSet<>(this.permissions); ret = newSet.removeAll(permissions); this.permissions = newSet; - updateStorage(storage); + updateStorage(this.storage); } return ret; } - public boolean removePermission(MetaStorage storage, Permission permission) { + public boolean removePermission(Permission permission) { boolean ret = false; synchronized (this) { Set newSet = new HashSet<>(this.permissions); ret = newSet.remove(permission); this.permissions = newSet; - updateStorage(storage); + updateStorage(this.storage); } return ret; } @@ -141,12 +140,12 @@ public Set getPermissions() { } - public void setPermissions(MetaStorage storage, Set permissionsNew) { + public void updatePermissions(Set permissionsNew) { synchronized (this) { Set newSet = new HashSet<>(permissionsNew.size()); newSet.addAll(permissionsNew); this.permissions = newSet; - updateStorage(storage); + updateStorage(this.storage); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Role.java b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Role.java index db07b661b9..1f73a79c24 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Role.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Role.java @@ -6,8 +6,8 @@ public class Role extends PermissionOwner { - public Role(String name, String label) { - super(name, label); + public Role(String name, String label, MetaStorage storage) { + super(name, label, storage); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/RoleOwner.java b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/RoleOwner.java index 1b915da5f9..bd8b366300 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/RoleOwner.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/RoleOwner.java @@ -8,9 +8,9 @@ public interface RoleOwner { - void addRole(MetaStorage storage, Role role); + void addRole(Role role); - void removeRole(MetaStorage storage, Role role); + void removeRole(Role role); /** * Return a copy of the roles hold by the owner. diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/User.java b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/User.java index 7227115d86..7d5485923c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/User.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/User.java @@ -33,7 +33,7 @@ public class User extends PermissionOwner implements Principal, RoleOwner { @JsonProperty - private Set roles = Collections.synchronizedSet(new HashSet<>()); + private final Set roles = Collections.synchronizedSet(new HashSet<>()); @Getter @Setter @@ -45,8 +45,8 @@ public class User extends PermissionOwner implements Principal, RoleOwne @Getter(AccessLevel.PROTECTED) private final transient ShiroUserAdapter shiroUserAdapter; - public User(String name, String label) { - super(name, label); + public User(String name, String label, MetaStorage storage) { + super(name, label, storage); this.shiroUserAdapter = new ShiroUserAdapter(); } @@ -55,7 +55,7 @@ public UserId createId() { return new UserId(name); } - public void addRole(MetaStorage storage, Role role) { + public synchronized void addRole(Role role) { if (roles.add(role.getId())) { log.trace("Added role {} to user {}", role.getId(), getId()); updateStorage(storage); @@ -63,7 +63,7 @@ public void addRole(MetaStorage storage, Role role) { } @Override - public void removeRole(MetaStorage storage, Role role) { + public synchronized void removeRole(Role role) { if (roles.remove(role.getId())) { log.trace("Removed role {} from user {}", role.getId(), getId()); updateStorage(storage); diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/IntrospectionDelegatingRealm.java b/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/IntrospectionDelegatingRealm.java index a5a71c409e..bbc2184346 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/IntrospectionDelegatingRealm.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/IntrospectionDelegatingRealm.java @@ -177,13 +177,13 @@ private void synchGroupMappings(User user, Set mappedGroupsToDo) { mappedGroupsToDo.remove(group); } else { // Mapping is not valid any more remove user from group - group.removeMember(storage, user); + group.removeMember(user); } } } for (Group group : mappedGroupsToDo) { - group.addMember(storage, user); + group.addMember(user); } } @@ -195,7 +195,7 @@ private synchronized User getOrCreateUser(TokenIntrospectionSuccessResponse succ } // try to construct a new User if none could be found in the storage String userLabel = successResponse.getStringParameter("name"); - user = new User(username, userLabel != null ? userLabel : username); + user = new User(username, userLabel != null ? userLabel : username, storage); storage.addUser(user); log.info("Created new user: {}", user); return user; @@ -213,7 +213,7 @@ private synchronized Group getOrCreateGroup(Pair groupNameId) { if (group != null) { return group; } - group = new Group(groupNameId.getValue().getGroup(), groupNameId.getKey()); + group = new Group(groupNameId.getValue().getGroup(), groupNameId.getKey(), storage); storage.addGroup(group); log.info("Created new group: {}", group); return group; diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/util/SinglePrincipalCollection.java b/backend/src/main/java/com/bakdata/conquery/models/auth/util/SinglePrincipalCollection.java index fd43125589..718a926a65 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/util/SinglePrincipalCollection.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/util/SinglePrincipalCollection.java @@ -15,7 +15,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; -public class SinglePrincipalCollection implements PrincipalCollection{ +public class SinglePrincipalCollection implements PrincipalCollection { private static final long serialVersionUID = -1801050265305362978L; diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java index 12544591ce..ebe79309b7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java @@ -6,6 +6,7 @@ import com.bakdata.conquery.commands.ShardNode; import com.bakdata.conquery.io.cps.CPSBase; import com.bakdata.conquery.io.storage.IdentifiableStore; +import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.io.storage.NamespaceStorage; import com.bakdata.conquery.io.storage.WorkerStorage; import com.bakdata.conquery.io.storage.xodus.stores.SingletonStore; @@ -63,9 +64,9 @@ public interface StoreFactory { // MetaStorage IdentifiableStore> createExecutionsStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName); IdentifiableStore createFormConfigStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName); - IdentifiableStore createUserStore(CentralRegistry centralRegistry, String pathName); - IdentifiableStore createRoleStore(CentralRegistry centralRegistry, String pathName); - IdentifiableStore createGroupStore(CentralRegistry centralRegistry, String pathName); + IdentifiableStore createUserStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage); + IdentifiableStore createRoleStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage); + IdentifiableStore createGroupStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage); SingletonStore createPrimaryDictionaryStore(String pathName, SingletonNamespaceCollection namespaceCollection); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java index 03cfe0c748..753908b1b0 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java @@ -5,11 +5,7 @@ import java.io.File; import java.io.IOException; import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Optional; -import java.util.Queue; +import java.util.*; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -24,14 +20,10 @@ import com.bakdata.conquery.commands.ManagerNode; import com.bakdata.conquery.commands.ShardNode; import com.bakdata.conquery.io.cps.CPSType; +import com.bakdata.conquery.io.jackson.Injectable; import com.bakdata.conquery.io.jackson.InternalOnly; import com.bakdata.conquery.io.jackson.Jackson; -import com.bakdata.conquery.io.storage.IdentifiableStore; -import com.bakdata.conquery.io.storage.NamespaceStorage; -import com.bakdata.conquery.io.storage.NamespacedStorage; -import com.bakdata.conquery.io.storage.Store; -import com.bakdata.conquery.io.storage.StoreInfo; -import com.bakdata.conquery.io.storage.WorkerStorage; +import com.bakdata.conquery.io.storage.*; import com.bakdata.conquery.io.storage.xodus.stores.BigStore; import com.bakdata.conquery.io.storage.xodus.stores.CachedStore; import com.bakdata.conquery.io.storage.xodus.stores.SerializingStore; @@ -113,7 +105,7 @@ public class XodusStoreFactory implements StoreFactory { /** * When set, all values that could not be deserialized from the persistent store, are dump into individual files. */ - private Optional unreadableDataDumpDirectory = Optional.empty(); + private File unreadableDataDumpDirectory = null; @JsonIgnore private transient Validator validator; @@ -122,7 +114,7 @@ public class XodusStoreFactory implements StoreFactory { private transient ObjectMapper objectMapper = Jackson.BINARY_MAPPER.copy(); @JsonIgnore - private BiMap activeEnvironments = HashBiMap.create(); + private final BiMap activeEnvironments = HashBiMap.create(); @JsonIgnore private final transient Multimap @@ -172,7 +164,7 @@ private Queue loadNamespacedStores(String prefi ExecutorService loaders = Executors.newFixedThreadPool(getNThreads()); - for (File directory : baseDir.listFiles((file, name) -> file.isDirectory() && name.startsWith(prefix))) { + for (File directory : Objects.requireNonNull(baseDir.listFiles((file, name) -> file.isDirectory() && name.startsWith(prefix)))) { final String name = directory.getName(); @@ -213,19 +205,10 @@ private Queue loadNamespacedStores(String prefi return storages; } - private List getRelativePathElements(Path path) { - ArrayList list = new ArrayList<>(); - Path relative = getDirectory().relativize(path); - for (int i = 0; i < relative.getNameCount(); i++) { - list.add(relative.getName(i).toString()); - } - return list; - } - private boolean environmentHasStores(File pathName) { Environment env = findEnvironment(pathName); - boolean exists = env.computeInTransaction(t -> env.storeExists(StoreInfo.DATASET.getName(), t)); - env.computeInTransaction(t -> env.getAllStoreNames(t)); + boolean exists = env.computeInTransaction(t -> env.storeExists(StoreInfo.DATASET.storeInfo().getName(), t)); + env.computeInTransaction(env::getAllStoreNames); if (!exists) { closeEnvironment(env); } @@ -234,17 +217,17 @@ private boolean environmentHasStores(File pathName) { @Override public SingletonStore createDatasetStore(String pathName) { - return DATASET.singleton(createStore(findEnvironment(pathName), validator, DATASET)); + return StoreInfo.singleton(createStore(findEnvironment(pathName), validator, DATASET)); } @Override public IdentifiableStore createSecondaryIdDescriptionStore(CentralRegistry centralRegistry, String pathName) { - return SECONDARY_IDS.identifiable(createStore(findEnvironment(pathName), validator, SECONDARY_IDS), centralRegistry); + return StoreInfo.identifiable(createStore(findEnvironment(pathName), validator, SECONDARY_IDS), centralRegistry); } @Override public IdentifiableStore createTableStore(CentralRegistry centralRegistry, String pathName) { - return TABLES.identifiable(createStore(findEnvironment(pathName), validator, TABLES), centralRegistry); + return StoreInfo.identifiable(createStore(findEnvironment(pathName), validator, TABLES), centralRegistry); } @Override @@ -257,46 +240,46 @@ public IdentifiableStore createDictionaryStore(CentralRegistry centr synchronized (openStoresInEnv) { bigStore = - new BigStore<>(this, validator, environment, DICTIONARIES, openStoresInEnv.get(environment), this::closeEnvironment, this::removeEnvironment, namespaceCollection - .injectInto(objectMapper)); + new BigStore<>( + this, + validator, + environment, + DICTIONARIES.storeInfo(), + openStoresInEnv.get(environment), + this::closeEnvironment, + XodusStoreFactory::removeEnvironmentHook, + namespaceCollection.injectInto(objectMapper)); } - final Store, Dictionary> result; - - // TODO this looks like dictionaries are double cached if (useWeakDictionaryCaching) { - result = new WeakCachedStore<>(bigStore, getWeakCacheDuration()); - } - else { - result = DICTIONARIES.cached(bigStore); + return StoreInfo.identifiableCachedStore(new WeakCachedStore<>(bigStore, getWeakCacheDuration()), centralRegistry); } - - return DICTIONARIES.identifiableCachedStore(result, centralRegistry); + return StoreInfo.identifiable(bigStore,centralRegistry); } @Override public IdentifiableStore> createConceptStore(CentralRegistry centralRegistry, String pathName) { - return CONCEPTS.identifiable(createStore(findEnvironment(pathName), validator, CONCEPTS), centralRegistry); + return StoreInfo.identifiable(createStore(findEnvironment(pathName), validator, CONCEPTS), centralRegistry); } @Override public IdentifiableStore createImportStore(CentralRegistry centralRegistry, String pathName) { - return IMPORTS.identifiable(createStore(findEnvironment(pathName), validator, IMPORTS), centralRegistry); + return StoreInfo.identifiable(createStore(findEnvironment(pathName), validator, IMPORTS), centralRegistry); } @Override public IdentifiableStore createCBlockStore(CentralRegistry centralRegistry, String pathName) { - return C_BLOCKS.identifiable(createStore(findEnvironment(pathName), validator, C_BLOCKS), centralRegistry); + return StoreInfo.identifiable(createStore(findEnvironment(pathName), validator, C_BLOCKS), centralRegistry); } @Override public IdentifiableStore createBucketStore(CentralRegistry centralRegistry, String pathName) { - return BUCKETS.identifiable(createStore(findEnvironment(pathName), validator, BUCKETS), centralRegistry); + return StoreInfo.identifiable(createStore(findEnvironment(pathName), validator, BUCKETS), centralRegistry); } @Override public SingletonStore createWorkerInformationStore(String pathName) { - return WORKER.singleton(createStore(findEnvironment(pathName), validator, WORKER)); + return StoreInfo.singleton(createStore(findEnvironment(pathName), validator, WORKER)); } @Override @@ -305,7 +288,7 @@ public SingletonStore createIdMappingStore(String pathName) { synchronized (openStoresInEnv) { final BigStore bigStore = - new BigStore<>(this, validator, environment, ID_MAPPING, openStoresInEnv.get(environment), this::closeEnvironment, this::removeEnvironment, objectMapper); + new BigStore<>(this, validator, environment, ID_MAPPING.storeInfo(), openStoresInEnv.get(environment), this::closeEnvironment, XodusStoreFactory::removeEnvironmentHook, objectMapper); return new SingletonStore<>(new CachedStore<>(bigStore)); } @@ -313,43 +296,43 @@ public SingletonStore createIdMappingStore(String pathName) { @Override public SingletonStore createWorkerToBucketsStore(String pathName) { - return WORKER_TO_BUCKETS.singleton(createStore(findEnvironment(pathName), validator, WORKER_TO_BUCKETS)); + return StoreInfo.singleton(createStore(findEnvironment(pathName), validator, WORKER_TO_BUCKETS)); } @Override public SingletonStore createStructureStore(String pathName, SingletonNamespaceCollection centralRegistry) { - return STRUCTURE.singleton(createStore(findEnvironment(pathName), validator, STRUCTURE), centralRegistry); + return StoreInfo.singleton(createStore(findEnvironment(pathName), validator, STRUCTURE), centralRegistry); } @Override public IdentifiableStore> createExecutionsStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName) { - return EXECUTIONS.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "executions")), validator, EXECUTIONS), centralRegistry, datasetRegistry); + return StoreInfo.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "executions")), validator, EXECUTIONS), centralRegistry, datasetRegistry); } @Override public IdentifiableStore createFormConfigStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName) { - return FORM_CONFIG.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "formConfigs")), validator, FORM_CONFIG), centralRegistry, datasetRegistry); + return StoreInfo.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "formConfigs")), validator, FORM_CONFIG), centralRegistry, datasetRegistry); } @Override - public IdentifiableStore createUserStore(CentralRegistry centralRegistry, String pathName) { - return AUTH_USER.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "users")), validator, AUTH_USER), centralRegistry); + public IdentifiableStore createUserStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage) { + return StoreInfo.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "users")), validator, AUTH_USER), centralRegistry, storage); } @Override - public IdentifiableStore createRoleStore(CentralRegistry centralRegistry, String pathName) { - return AUTH_ROLE.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "roles")), validator, AUTH_ROLE), centralRegistry); + public IdentifiableStore createRoleStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage) { + return StoreInfo.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "roles")), validator, AUTH_ROLE), centralRegistry, storage); } @Override - public IdentifiableStore createGroupStore(CentralRegistry centralRegistry, String pathName) { - return AUTH_GROUP.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "groups")), validator, AUTH_GROUP), centralRegistry); + public IdentifiableStore createGroupStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage) { + return StoreInfo.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "groups")), validator, AUTH_GROUP), centralRegistry, storage); } @Override public SingletonStore createPrimaryDictionaryStore(String pathName, SingletonNamespaceCollection namespaceCollection) { - return PRIMARY_DICTIONARY.singleton(createStore(findEnvironment(pathName), validator, PRIMARY_DICTIONARY), namespaceCollection); + return StoreInfo.singleton(createStore(findEnvironment(pathName), validator, PRIMARY_DICTIONARY), namespaceCollection); } private File resolveSubDir(String... subdirs) { @@ -397,27 +380,30 @@ private void closeEnvironment(Environment env) { } } - private void removeEnvironment(Environment env) { + public static void removeEnvironmentHook(Environment env) { log.info("Deleting Environment[{}]", env.getLocation()); try { FileUtil.deleteRecursive(Path.of(env.getLocation())); } catch (IOException e) { - log.error("Cannot delete directory of removed Environment[{}]", env.getLocation(), log.isDebugEnabled() ? e : null); + log.error("Cannot delete directory of removed Environment[{}]", env.getLocation(), e); } } - public Store createStore(Environment environment, Validator validator, StoreInfo storeId) { + public Store createStore(Environment environment, Validator validator, StoreInfo storeId, Injectable ... injectables) { + final IStoreInfo storeInfo = storeId.storeInfo(); synchronized (openStoresInEnv) { return new CachedStore<>( new SerializingStore<>( - this, - new XodusStore(environment, storeId.getName(), openStoresInEnv.get(environment), this::closeEnvironment, this::removeEnvironment), + new XodusStore(environment, storeInfo.getName(), openStoresInEnv.get(environment), this::closeEnvironment, XodusStoreFactory::removeEnvironmentHook), validator, - storeId, objectMapper, - (Class) storeId.getKeyType(), - (Class) storeId.getValueType() + storeInfo.getKeyType(), + storeInfo.getValueType(), + this.isValidateOnWrite(), + this.isRemoveUnreadableFromStore(), + this.getUnreadableDataDumpDirectory(), + injectables )); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/execution/Shareable.java b/backend/src/main/java/com/bakdata/conquery/models/execution/Shareable.java index 44f5914a09..aadb68e39a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/execution/Shareable.java +++ b/backend/src/main/java/com/bakdata/conquery/models/execution/Shareable.java @@ -7,7 +7,6 @@ import com.bakdata.conquery.apiv1.MetaDataPatch; import com.bakdata.conquery.io.storage.MetaStorage; -import com.bakdata.conquery.models.auth.AuthorizationHelper; import com.bakdata.conquery.models.auth.entities.Group; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.auth.permissions.AbilitySets; @@ -50,7 +49,7 @@ default , S extends Identifiable & Shareable & log.trace("User {} unshares instance {} ({}) from owner {}.", user, shareable.getClass().getSimpleName(), shareable.getId(), group1); - AuthorizationHelper.removePermission(group1, shareable.createPermission(AbilitySets.SHAREHOLDER), storage); + group1.removePermission(shareable.createPermission(AbilitySets.SHAREHOLDER)); } @@ -60,7 +59,7 @@ default , S extends Identifiable & Shareable & for(Group group : groups) { ConqueryPermission sharePermission = shareable.createPermission(AbilitySets.SHAREHOLDER); - AuthorizationHelper.addPermission(group, sharePermission, storage); + group.addPermission(sharePermission); log.trace("User {} shares instance {} ({}). Adding permission {} to owner {}.", user, shareable.getClass().getSimpleName(), shareable.getId(), sharePermission, group); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/configs/FormConfig.java b/backend/src/main/java/com/bakdata/conquery/models/forms/configs/FormConfig.java index 99079e4d7f..cc90d20659 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/configs/FormConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/configs/FormConfig.java @@ -101,7 +101,7 @@ public FormConfigOverviewRepresentation overview(User user) { .label(label) .tags(tags) .ownerName(ownerName) - .own(owner.equals(user)) + .own(user.isOwner(this)) .createdAt(getCreationTime().atZone(ZoneId.systemDefault())) .shared(shared) // system? diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormConfigProcessor.java b/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormConfigProcessor.java index 5f3b82faa6..7db632de33 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormConfigProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormConfigProcessor.java @@ -127,7 +127,7 @@ public FormConfig addConfig(User user, Dataset targetDataset, FormConfigAPI conf * @return */ public FormConfig addConfigAndTranslations(User user, Dataset targetDataset, FormConfigAPI config) { - FormConfig internalConfig = FormConfigAPI.intern(config, user, targetDataset); + FormConfig internalConfig = FormConfigAPI.intern(config, storage.getUser(user.getId()), targetDataset); // Add the config immediately to the submitted dataset addConfigToDataset(internalConfig); @@ -160,8 +160,8 @@ public FormConfigFullRepresentation patchConfig(User user, FormConfig config, Fo /** * Deletes a configuration from the storage and all permissions, that have this configuration as target. */ - public void deleteConfig(User user, FormConfig config) { - + public void deleteConfig(User User, FormConfig config) { + User user = storage.getUser(User.getId()); user.authorize( config, Ability.DELETE); storage.removeFormConfig(config.getId()); // Delete corresponding permissions (Maybe better to put it into a slow job) @@ -182,10 +182,10 @@ public void deleteConfig(User user, FormConfig config) { instancesCleared.remove(config.getId().toString()); WildcardPermission clearedPermission = new WildcardPermission(List.of(wpermission.getDomains(), wpermission.getAbilities(), instancesCleared), Instant.now()); - user.addPermission(storage, clearedPermission); + user.addPermission(clearedPermission); } - user.removePermission(storage, wpermission); + user.removePermission(wpermission); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormProcessor.java b/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormProcessor.java index 8b2938739c..41f15d6cf6 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormProcessor.java @@ -6,7 +6,6 @@ import java.util.Collection; import java.util.List; -import com.bakdata.conquery.models.auth.AuthorizationHelper; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.auth.permissions.Ability; import com.fasterxml.jackson.databind.JsonNode; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/ExecutionManager.java b/backend/src/main/java/com/bakdata/conquery/models/query/ExecutionManager.java index c62b789163..5e6a3480f6 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/ExecutionManager.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/ExecutionManager.java @@ -16,7 +16,6 @@ import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.execution.ExecutionState; import com.bakdata.conquery.models.execution.ManagedExecution; -import com.bakdata.conquery.models.messages.namespaces.WorkerMessage; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.query.results.ShardResult; import com.bakdata.conquery.models.worker.DatasetRegistry; diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetResource.java index 25d1110e38..63589b5b08 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetResource.java @@ -1,8 +1,36 @@ package com.bakdata.conquery.resources.admin.rest; +import static com.bakdata.conquery.resources.ResourceConstants.DATASET; +import static com.bakdata.conquery.resources.ResourceConstants.SECONDARY_ID; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import java.util.StringJoiner; +import java.util.stream.Collectors; +import java.util.zip.GZIPInputStream; + +import javax.annotation.PostConstruct; +import javax.inject.Inject; +import javax.validation.Valid; +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response.Status; + import com.bakdata.conquery.ConqueryConstants; import com.bakdata.conquery.io.jersey.ExtraMimeTypes; -import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.datasets.SecondaryIdDescription; import com.bakdata.conquery.models.datasets.Table; @@ -14,31 +42,10 @@ import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.resources.hierarchies.HAdmin; -import io.dropwizard.auth.Auth; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; -import javax.annotation.PostConstruct; -import javax.inject.Inject; -import javax.validation.Valid; -import javax.validation.constraints.Min; -import javax.validation.constraints.NotNull; -import javax.ws.rs.*; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response.Status; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.List; -import java.util.StringJoiner; -import java.util.stream.Collectors; -import java.util.zip.GZIPInputStream; - -import static com.bakdata.conquery.resources.ResourceConstants.DATASET; -import static com.bakdata.conquery.resources.ResourceConstants.SECONDARY_ID; - @Slf4j @Produces({ExtraMimeTypes.JSON_STRING, ExtraMimeTypes.SMILE_STRING}) @Consumes({ExtraMimeTypes.JSON_STRING, ExtraMimeTypes.SMILE_STRING}) @@ -190,7 +197,7 @@ public void delete() { @POST @Path("/update-matching-stats") @Consumes(MediaType.WILDCARD) - public void updateMatchingStats(@Auth User user, @PathParam(DATASET)Dataset dataset) { + public void updateMatchingStats(@PathParam(DATASET) Dataset dataset) { processor.updateMatchingStats(dataset); } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminProcessor.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminProcessor.java index c9ad84d1e1..ad73edf496 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminProcessor.java @@ -74,7 +74,17 @@ public void addRoles(List roles) { * @param role the role to delete */ public void deleteRole(Role role) { - AuthorizationHelper.deleteRole(storage, role); + log.info("Deleting {}", role); + + for (User user : storage.getAllUsers()) { + user.removeRole(role); + } + + for (Group group : storage.getAllGroups()) { + group.removeRole(role); + } + + storage.removeRole(role.getId()); } public SortedSet getAllRoles() { @@ -90,7 +100,7 @@ public SortedSet getAllRoles() { * @throws JSONException is thrown upon processing JSONs. */ public void createPermission(PermissionOwner owner, ConqueryPermission permission) throws JSONException { - AuthorizationHelper.addPermission(owner, permission, storage); + owner.addPermission(permission); } /** @@ -101,7 +111,7 @@ public void createPermission(PermissionOwner owner, ConqueryPermission permis * @param permission The permission to delete. */ public void deletePermission(PermissionOwner owner, ConqueryPermission permission) { - AuthorizationHelper.removePermission(owner, permission, storage); + owner.removePermission(permission); } @@ -111,7 +121,7 @@ public TreeSet getAllUsers() { public synchronized void deleteUser(User user) { for (Group group : storage.getAllGroups()) { - group.removeMember(storage, user); + group.removeMember(user); } storage.removeUser(user.getId()); log.trace("Removed user {} from the storage.", user.getId()); @@ -158,17 +168,12 @@ public void addGroups(List groups) { } public void addUserToGroup(Group group, User user) { - synchronized (storage) { - group.addMember(storage, user); - - log.trace("Added user {} to group {}", user, group); - } + group.addMember(user); + log.trace("Added user {} to group {}", user, group); } public void deleteUserFromGroup(Group group, User user) { - synchronized (storage) { - group.removeMember(storage,user); - } + group.removeMember(user); log.trace("Removed user {} from group {}", user, group); } @@ -178,14 +183,13 @@ public void deleteGroup(Group group) { } public void deleteRoleFrom(RoleOwner owner, Role role) { - synchronized (storage) { - AuthorizationHelper.deleteRoleFrom(storage, owner, role); - } - + owner.removeRole(role); + log.trace("Removed role {} from {}", role, owner); } public void addRoleTo(RoleOwner owner, Role role) { - AuthorizationHelper.addRoleTo(getStorage(), role, owner); + owner.addRole(role); + log.trace("Added role {} to {}", role, owner); } /** diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminResource.java index e0af8d4679..e59f5c1d9a 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminResource.java @@ -24,6 +24,7 @@ import com.bakdata.conquery.io.jersey.ExtraMimeTypes; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.models.auth.entities.User; +import com.bakdata.conquery.models.common.Range; import com.bakdata.conquery.models.config.auth.AuthenticationConfig; import com.bakdata.conquery.models.jobs.JobManagerStatus; import com.bakdata.conquery.models.messages.network.specific.CancelJobMessage; diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultArrowFileResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultArrowFileResource.java index d53507fd65..9871753114 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultArrowFileResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultArrowFileResource.java @@ -18,7 +18,6 @@ import com.bakdata.conquery.apiv1.AdditionalMediaTypes; import com.bakdata.conquery.io.result.arrow.ResultArrowFileProcessor; -import com.bakdata.conquery.io.result.arrow.ResultArrowProcessor; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.execution.ManagedExecution; diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultArrowStreamResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultArrowStreamResource.java index bdea5f96f1..4dc0deb93d 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultArrowStreamResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultArrowStreamResource.java @@ -17,7 +17,6 @@ import javax.ws.rs.core.UriBuilder; import com.bakdata.conquery.apiv1.AdditionalMediaTypes; -import com.bakdata.conquery.io.result.arrow.ResultArrowProcessor; import com.bakdata.conquery.io.result.arrow.ResultArrowStreamProcessor; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.datasets.Dataset; diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultExcelResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultExcelResource.java index 0ab93c3919..17d208ea34 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultExcelResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultExcelResource.java @@ -1,28 +1,32 @@ package com.bakdata.conquery.resources.api; +import static com.bakdata.conquery.io.result.ResultUtil.checkSingleTableResult; +import static com.bakdata.conquery.resources.ResourceConstants.DATASET; +import static com.bakdata.conquery.resources.ResourceConstants.QUERY; + +import java.net.MalformedURLException; +import java.net.URL; +import java.util.Optional; + +import javax.inject.Inject; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.UriBuilder; + import com.bakdata.conquery.apiv1.AdditionalMediaTypes; import com.bakdata.conquery.io.result.excel.ResultExcelProcessor; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; -import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.query.SingleTableResult; import com.bakdata.conquery.resources.ResourceConstants; -import com.bakdata.conquery.util.ResourceUtil; import io.dropwizard.auth.Auth; import lombok.extern.slf4j.Slf4j; -import javax.inject.Inject; -import javax.ws.rs.*; -import javax.ws.rs.core.Response; -import javax.ws.rs.core.UriBuilder; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.Optional; - -import static com.bakdata.conquery.io.result.ResultUtil.checkSingleTableResult; -import static com.bakdata.conquery.resources.ResourceConstants.*; - @Slf4j @Path("datasets/{" + DATASET + "}/result/") public class ResultExcelResource { diff --git a/backend/src/main/java/com/bakdata/conquery/tasks/PermissionCleanupTask.java b/backend/src/main/java/com/bakdata/conquery/tasks/PermissionCleanupTask.java index a1dba8c00d..d6bc5ac7db 100644 --- a/backend/src/main/java/com/bakdata/conquery/tasks/PermissionCleanupTask.java +++ b/backend/src/main/java/com/bakdata/conquery/tasks/PermissionCleanupTask.java @@ -78,11 +78,11 @@ public static int deleteQueryPermissionsWithMissingRef(MetaStorage storage, Iter // Create a new Permission that only contains valid references WildcardPermission reducedPermission = new WildcardPermission( List.of(wpermission.getDomains(), wpermission.getAbilities(), validRef), wpermission.getCreationTime()); - owner.addPermission(storage, reducedPermission); + owner.addPermission(reducedPermission); } // Delete the old permission that containes both valid and invalid references - owner.removePermission(storage, wpermission); + owner.removePermission(wpermission); countDeleted++; } @@ -148,7 +148,7 @@ public static & Owned, ID extends IId> int de } log.trace("User owns the instance. Deleting the permission"); - user.removePermission(storage, wpermission); + user.removePermission(wpermission); countDeleted++; diff --git a/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java b/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java index 92c3874af7..179860a07c 100644 --- a/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java +++ b/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java @@ -40,11 +40,6 @@ import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.identifiable.ids.specific.SecondaryIdDescriptionId; import com.bakdata.conquery.models.query.ManagedQuery; -import com.bakdata.conquery.apiv1.query.CQElement; -import com.bakdata.conquery.apiv1.query.ConceptQuery; -import com.bakdata.conquery.apiv1.query.SecondaryIdQuery; -import com.bakdata.conquery.apiv1.query.concept.specific.CQAnd; -import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; import com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.resources.api.ResultArrowFileResource; @@ -122,12 +117,12 @@ public void getQueriesFiltered() { } private static User mockUser(int id, List allowedQueryIds) { - final User user = new User("user" + id, null); + final User user = new User("user" + id, null, STORAGE); STORAGE.addUser(user); for (ManagedExecutionId queryId : allowedQueryIds) { - AuthorizationHelper.addPermission(user, ExecutionPermission.onInstance(AbilitySets.QUERY_CREATOR,queryId), STORAGE); + user.addPermission(ExecutionPermission.onInstance(AbilitySets.QUERY_CREATOR,queryId)); } return user; diff --git a/backend/src/test/java/com/bakdata/conquery/api/form/config/FormConfigTest.java b/backend/src/test/java/com/bakdata/conquery/api/form/config/FormConfigTest.java index 01f0e094f1..c0f6c8764e 100644 --- a/backend/src/test/java/com/bakdata/conquery/api/form/config/FormConfigTest.java +++ b/backend/src/test/java/com/bakdata/conquery/api/form/config/FormConfigTest.java @@ -147,9 +147,9 @@ public void cleanupTest() { @Test public void addConfigWithoutTranslation() { - User user = new User("test","test"); + User user = new User("test","test", storage); storage.addUser(user); - user.addPermission(storage, dataset.createPermission(Ability.READ.asSet())); + user.addPermission(dataset.createPermission(Ability.READ.asSet())); ObjectMapper mapper = FormConfigProcessor.getMAPPER(); FormConfigAPI formConfig = FormConfigAPI.builder() @@ -165,15 +165,15 @@ public void addConfigWithoutTranslation() { @Test public void deleteConfig() { // PREPARE - User user = new User("test","test"); + User user = new User("test","test", storage); storage.addUser(user); - user.addPermission(storage, DatasetPermission.onInstance(Ability.READ, datasetId)); + user.addPermission(DatasetPermission.onInstance(Ability.READ, datasetId)); ObjectMapper mapper = FormConfigProcessor.getMAPPER(); FormConfig formConfig = new FormConfig(form.getClass().getAnnotation(CPSType.class).id(), mapper.valueToTree(form)); formConfig.setDataset(dataset); - user.addPermission(storage, formConfig.createPermission(AbilitySets.FORM_CONFIG_CREATOR)); + user.addPermission(formConfig.createPermission(AbilitySets.FORM_CONFIG_CREATOR)); storage.addFormConfig(formConfig); // EXECUTE @@ -188,16 +188,16 @@ public void deleteConfig() { @Test public void getConfig() { // PREPARE - User user = new User("test","test"); + User user = new User("test","test", storage); storage.addUser(user); - user.addPermission(storage, dataset.createPermission(Ability.READ.asSet())); + user.addPermission(dataset.createPermission(Ability.READ.asSet())); ObjectMapper mapper = FormConfigProcessor.getMAPPER(); JsonNode values = mapper.valueToTree(form); FormConfig formConfig = new FormConfig(form.getClass().getAnnotation(CPSType.class).id(), values); formConfig.setDataset(dataset); formConfig.setOwner(user); - user.addPermission(storage, formConfig.createPermission(Ability.READ.asSet())); + user.addPermission(formConfig.createPermission(Ability.READ.asSet())); storage.addFormConfig(formConfig); // EXECUTE @@ -263,10 +263,10 @@ public void visit(Consumer visitor) { public void getConfigs() { // PREPARE - User user = new User("test","test"); + User user = new User("test","test", storage); storage.addUser(user); - user.addPermission(storage, DatasetPermission.onInstance(Ability.READ, datasetId)); - user.addPermission(storage, FormPermission.onInstance(Ability.CREATE, form.getFormType())); + user.addPermission(DatasetPermission.onInstance(Ability.READ, datasetId)); + user.addPermission(FormPermission.onInstance(Ability.CREATE, form.getFormType())); ExportForm form2 = new ExportForm(); RelativeMode mode3 = new RelativeMode(); @@ -330,16 +330,16 @@ public void getConfigs() { @Test public void patchConfig() { // PREPARE - User user = new User("test","test"); + User user = new User("test","test", storage); storage.addUser(user); - user.addPermission(storage, DatasetPermission.onInstance(Ability.READ, datasetId)); - Group group1 = new Group("test1","test1"); + user.addPermission(DatasetPermission.onInstance(Ability.READ, datasetId)); + Group group1 = new Group("test1","test1", storage); storage.addGroup(group1); - Group group2 = new Group("test2","test2"); + Group group2 = new Group("test2","test2", storage); storage.addGroup(group2); - group1.addMember(storage, user); - group2.addMember(storage, user); + group1.addMember(user); + group2.addMember(user); ObjectMapper mapper = FormConfigProcessor.getMAPPER(); JsonNode values = mapper.valueToTree(form); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/DownloadLinkGeneration.java b/backend/src/test/java/com/bakdata/conquery/integration/DownloadLinkGeneration.java index c994f1fb25..ded19ed71b 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/DownloadLinkGeneration.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/DownloadLinkGeneration.java @@ -9,6 +9,7 @@ import com.bakdata.conquery.integration.json.JsonIntegrationTest; import com.bakdata.conquery.integration.json.QueryTest; import com.bakdata.conquery.integration.tests.ProgrammaticIntegrationTest; +import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.auth.permissions.Ability; import com.bakdata.conquery.models.auth.permissions.DatasetPermission; @@ -25,13 +26,15 @@ public class DownloadLinkGeneration extends IntegrationTest.Simple implements Pr @Override public void execute(StandaloneSupport conquery) throws Exception { - final User user = new User("testU", "testU"); + final MetaStorage storage = conquery.getMetaStorage(); + + final User user = new User("testU", "testU", storage); final String testJson = In.resource("/tests/query/SIMPLE_TREECONCEPT_QUERY/SIMPLE_TREECONCEPT_Query.test.json").withUTF8() .readAll(); final QueryTest test = (QueryTest) JsonIntegrationTest.readJson(conquery.getDataset(), testJson); - conquery.getMetaStorage().updateUser(user); + storage.updateUser(user); // Manually import data ValidatorHelper.failOnError(log, conquery.getValidator().validate(test)); @@ -40,11 +43,9 @@ public void execute(StandaloneSupport conquery) throws Exception { // Create execution for download ManagedQuery exec = new ManagedQuery(test.getQuery(), user, conquery.getDataset()); - conquery.getMetaStorage().addExecution(exec); + storage.addExecution(exec); - user.addPermission( - conquery.getMetaStorage(), - DatasetPermission.onInstance(Set.of(Ability.READ), conquery.getDataset().getId())); + user.addPermission(DatasetPermission.onInstance(Set.of(Ability.READ), conquery.getDataset().getId())); { // Try to generate a download link: should not be possible, because the execution isn't run yet @@ -62,9 +63,7 @@ public void execute(StandaloneSupport conquery) throws Exception { { // Add permission to download: now it should be possible - user.addPermission( - conquery.getMetaStorage(), - DatasetPermission.onInstance(Set.of(Ability.DOWNLOAD), conquery.getDataset().getId())); + user.addPermission(DatasetPermission.onInstance(Set.of(Ability.DOWNLOAD), conquery.getDataset().getId())); FullExecutionStatus status = IntegrationUtils.getExecutionStatus(conquery, exec.getId(), user, 200); // This Url is missing the `/api` path part, because we use the standard UriBuilder here diff --git a/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java b/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java index 8bb624870a..4d885263ba 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java @@ -30,6 +30,8 @@ import com.bakdata.conquery.util.support.TestConquery; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.databind.node.TextNode; +import io.dropwizard.logging.DefaultLoggingFactory; import io.github.classgraph.Resource; import lombok.EqualsAndHashCode; import lombok.Getter; diff --git a/backend/src/test/java/com/bakdata/conquery/integration/common/IntegrationUtils.java b/backend/src/test/java/com/bakdata/conquery/integration/common/IntegrationUtils.java index cd6a6f14e1..23181a8c18 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/common/IntegrationUtils.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/common/IntegrationUtils.java @@ -51,7 +51,7 @@ public static void importPermissionConstellation(MetaStorage storage, Role[] rol RoleId[] rolesInjected = rUser.getRolesInjected(); for (RoleId mandatorId : rolesInjected) { - user.addRole(storage, storage.getRole(mandatorId)); + user.addRole(storage.getRole(mandatorId)); } } } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java b/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java index 964f470b14..cd6f2fb710 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java @@ -71,7 +71,7 @@ public static void importPreviousQueries(StandaloneSupport support, RequiredData ManagedExecution managed = support.getNamespace().getExecutionManager() .createQuery(support.getNamespace().getNamespaces(),q, queryId, user, support.getNamespace().getDataset()); - user.addPermission(support.getMetaStorage(), ExecutionPermission.onInstance(AbilitySets.QUERY_CREATOR, managed.getId())); + user.addPermission(ExecutionPermission.onInstance(AbilitySets.QUERY_CREATOR, managed.getId())); if (managed.getState() == ExecutionState.FAILED) { fail("Query failed"); @@ -85,7 +85,7 @@ public static void importPreviousQueries(StandaloneSupport support, RequiredData UUID queryId = new UUID(0L, id++); ManagedExecution managed = support.getNamespace().getExecutionManager().createQuery(support.getNamespace().getNamespaces(),query, queryId, user, support.getNamespace().getDataset()); - user.addPermission(support.getMetaStorage(), ExecutionPermission.onInstance(AbilitySets.QUERY_CREATOR, managed.getId())); + user.addPermission(ExecutionPermission.onInstance(AbilitySets.QUERY_CREATOR, managed.getId())); if (managed.getState() == ExecutionState.FAILED) { fail("Query failed"); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/ConceptPermissionTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/ConceptPermissionTest.java index 6aac31f38c..6698adca50 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/ConceptPermissionTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/ConceptPermissionTest.java @@ -33,7 +33,7 @@ public void execute(StandaloneSupport conquery) throws Exception { final String testJson = In.resource("/tests/query/SIMPLE_TREECONCEPT_QUERY/SIMPLE_TREECONCEPT_Query.test.json").withUTF8().readAll(); final QueryTest test = (QueryTest) JsonIntegrationTest.readJson(dataset.getId(), testJson); final QueryProcessor processor = new QueryProcessor(storage.getDatasetRegistry(), storage, conquery.getConfig()); - final User user = new User("testUser", "testUserLabel"); + final User user = new User("testUser", "testUserLabel", storage); // Manually import data, so we can do our own work. { @@ -52,7 +52,7 @@ public void execute(StandaloneSupport conquery) throws Exception { conquery.waitUntilWorkDone(); storage.addUser(user); - user.addPermission(storage, DatasetPermission.onInstance(Ability.READ, dataset.getId())); + user.addPermission(DatasetPermission.onInstance(Ability.READ, dataset.getId())); } // Query cannot be deserialized without Namespace set up @@ -68,7 +68,7 @@ public void execute(StandaloneSupport conquery) throws Exception { { final ConqueryPermission permission = conceptId.createPermission(Ability.READ.asSet()); log.info("Adding the Permission[{}] to User[{}]", permission, user); - AuthorizationHelper.addPermission(user, permission, storage); + user.addPermission(permission); } // Only assert permissions diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/GroupHandlingTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/GroupHandlingTest.java index 41db4f56e3..8c156caa29 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/GroupHandlingTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/GroupHandlingTest.java @@ -14,28 +14,30 @@ */ public class GroupHandlingTest extends IntegrationTest.Simple implements ProgrammaticIntegrationTest { - private final Group group1 = new Group("company", "company"); - private final User user1 = new User("user", "user"); - private final User user1copy = new User("user", "user"); - private final User user2 = new User("user2", "user2"); - + + @Override public void execute(StandaloneSupport conquery) throws Exception { MetaStorage storage = conquery.getMetaStorage(); - + + Group group1 = new Group("company", "company", storage); + User user1 = new User("user", "user", storage); + User user1copy = new User("user", "user", storage); + User user2 = new User("user2", "user2", storage); + try { storage.addGroup(group1); storage.addUser(user1); - - group1.addMember(storage, user1); - group1.addMember(storage, user1copy); + + group1.addMember(user1); + group1.addMember(user1copy); assertThat(group1.getMembers()).containsExactlyInAnyOrder(user1.getId()); - group1.addMember(storage, user2); + group1.addMember(user2); assertThat(group1.getMembers()).containsExactlyInAnyOrder(user1.getId(), user2.getId()); - group1.removeMember(storage, user2); + group1.removeMember(user2); assertThat(group1.getMembers()).containsExactlyInAnyOrder(user1.getId()); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/PermissionGroupHandlingTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/PermissionGroupHandlingTest.java index 6768477d2a..8edf25ea9e 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/PermissionGroupHandlingTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/PermissionGroupHandlingTest.java @@ -17,9 +17,6 @@ public class PermissionGroupHandlingTest extends IntegrationTest.Simple implements ProgrammaticIntegrationTest { - private final Role role1 = new Role("role", "role"); - private final TestUser user1 = new TestUser(); - private final Group group1 = new Group("company", "company"); /** * This is a longer test that plays through different scenarios of permission @@ -31,6 +28,11 @@ public void execute(StandaloneSupport conquery) throws Exception { Dataset dataset1 = new Dataset(); dataset1.setLabel("dataset1"); ManagedExecutionId query1 = new ManagedExecutionId(dataset1.getId(), UUID.randomUUID()); + + + Role role1 = new Role("role", "role", storage); + TestUser user1 = new TestUser(storage); + Group group1 = new Group("company", "company", storage); try { @@ -38,20 +40,20 @@ public void execute(StandaloneSupport conquery) throws Exception { storage.addUser(user1); storage.addGroup(group1); - AuthorizationHelper.addRoleTo(storage,role1,user1); + user1.addRole(role1); - group1.addMember(storage, user1); + group1.addMember(user1); - user1.addPermission(storage, ExecutionPermission.onInstance(Ability.READ, query1)); - role1.addPermission(storage, ExecutionPermission.onInstance(Ability.DELETE, query1)); - group1.addPermission(storage, ExecutionPermission.onInstance(Ability.SHARE, query1)); + user1.addPermission(ExecutionPermission.onInstance(Ability.READ, query1)); + role1.addPermission(ExecutionPermission.onInstance(Ability.DELETE, query1)); + group1.addPermission(ExecutionPermission.onInstance(Ability.SHARE, query1)); assertThat(user1.isPermitted(ExecutionPermission.onInstance(Ability.READ, query1))).isTrue(); assertThat(user1.isPermitted(ExecutionPermission.onInstance(Ability.DELETE, query1))).isTrue(); assertThat(user1.isPermitted(ExecutionPermission.onInstance(Ability.SHARE, query1))).isTrue(); // remove user from group - group1.removeMember(storage, user1); + group1.removeMember(user1); assertThat(user1.isPermitted(ExecutionPermission.onInstance(Ability.READ, query1))).isTrue(); assertThat(user1.isPermitted(ExecutionPermission.onInstance(Ability.DELETE, query1))).isTrue(); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/PermissionRoleHandlingTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/PermissionRoleHandlingTest.java index b8ecbf6062..105abea3f7 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/PermissionRoleHandlingTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/PermissionRoleHandlingTest.java @@ -11,8 +11,7 @@ public class PermissionRoleHandlingTest extends IntegrationTest.Simple implements ProgrammaticIntegrationTest { - private final Role mandator1 = new Role("company", "company"); - private final TestUser user1 = new TestUser(); + /** * This is a longer test that plays through different scenarios of permission @@ -21,6 +20,8 @@ public class PermissionRoleHandlingTest extends IntegrationTest.Simple implement @Override public void execute(StandaloneSupport conquery) throws Exception { MetaStorage storage = conquery.getMetaStorage(); + Role mandator1 = new Role("company", "company", storage); + TestUser user1 = new TestUser(storage); Dataset dataset = conquery.getDataset(); try { @@ -28,49 +29,49 @@ public void execute(StandaloneSupport conquery) throws Exception { storage.addRole(mandator1); storage.addUser(user1); - user1.addRole(storage, mandator1); + user1.addRole(mandator1); - user1.addPermission(storage, dataset.createPermission(Ability.READ.asSet())); - mandator1.addPermission(storage, dataset.createPermission(Ability.DOWNLOAD.asSet())); + user1.addPermission(dataset.createPermission(Ability.READ.asSet())); + mandator1.addPermission(dataset.createPermission(Ability.DOWNLOAD.asSet())); assertThat(user1.isPermitted(dataset.createPermission(Ability.READ.asSet()))).isTrue(); assertThat(user1.isPermitted(dataset.createPermission(Ability.DOWNLOAD.asSet()))).isTrue(); // Delete permission from mandator - mandator1.removePermission(storage, dataset.createPermission(Ability.DOWNLOAD.asSet())); + mandator1.removePermission(dataset.createPermission(Ability.DOWNLOAD.asSet())); assertThat(mandator1.getPermissions()).isEmpty(); assertThat(user1.isPermitted(dataset.createPermission(Ability.READ.asSet()))).isTrue(); assertThat(user1.isPermitted(dataset.createPermission(Ability.DOWNLOAD.asSet()))).isFalse(); // Add permission to user - user1.addPermission(storage, dataset.createPermission(Ability.DOWNLOAD.asSet())); + user1.addPermission(dataset.createPermission(Ability.DOWNLOAD.asSet())); assertThat(user1.isPermitted(dataset.createPermission(Ability.READ.asSet()))).isTrue(); assertThat(user1.isPermitted(dataset.createPermission(Ability.DOWNLOAD.asSet()))).isTrue(); // Delete permission from mandator - user1.removePermission(storage, dataset.createPermission(Ability.DOWNLOAD.asSet())); + user1.removePermission(dataset.createPermission(Ability.DOWNLOAD.asSet())); assertThat(user1.isPermitted(dataset.createPermission(Ability.READ.asSet()))).isTrue(); assertThat(user1.isPermitted(dataset.createPermission(Ability.DOWNLOAD.asSet()))).isFalse(); // Add permission to mandator, remove mandator from user - mandator1.addPermission(storage, dataset.createPermission(Ability.DOWNLOAD.asSet())); - user1.removeRole(storage, mandator1); + mandator1.addPermission(dataset.createPermission(Ability.DOWNLOAD.asSet())); + user1.removeRole(mandator1); assertThat(user1.isPermitted(dataset.createPermission(Ability.READ.asSet()))).isTrue(); assertThat(user1.isPermitted(dataset.createPermission(Ability.DOWNLOAD.asSet()))).isFalse(); // Add mandator back to user - user1.addRole(storage, mandator1); + user1.addRole(mandator1); assertThat(user1.isPermitted(dataset.createPermission(Ability.READ.asSet()))).isTrue(); assertThat(user1.isPermitted(dataset.createPermission(Ability.DOWNLOAD.asSet()))).isTrue(); // Delete all permissions from mandator and user - user1.removePermission(storage, dataset.createPermission(Ability.READ.asSet())); - mandator1.removePermission(storage, dataset.createPermission(Ability.DOWNLOAD.asSet())); + user1.removePermission(dataset.createPermission(Ability.READ.asSet())); + mandator1.removePermission(dataset.createPermission(Ability.DOWNLOAD.asSet())); assertThat(user1.isPermitted(dataset.createPermission(Ability.READ.asSet()))).isFalse(); assertThat(user1.isPermitted(dataset.createPermission(Ability.DOWNLOAD.asSet()))).isFalse(); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/RestartTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/RestartTest.java index 386024e5fb..878f1ba86b 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/RestartTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/RestartTest.java @@ -35,12 +35,6 @@ public class RestartTest implements ProgrammaticIntegrationTest { public static final Dataset TEST_DATASET_4 = new Dataset("testDataset4"); public static final Dataset TEST_DATASET_5 = new Dataset("testDataset5"); public static final Dataset TEST_DATASET_6 = new Dataset("testDataset6"); - private Role role = new Role("role", "ROLE"); - private Role roleToDelete = new Role("roleDelete", "ROLE_DELETE"); - private User user = new User("user@test.email", "USER"); - private User userToDelete = new User("userDelete@test.email", "USER_DELETE"); - private Group group = new Group("group", "GROUP"); - private Group groupToDelete = new Group("groupDelete", "GROUP_DELETE"); @Override public void execute(String name, TestConquery testConquery) throws Exception { @@ -79,6 +73,18 @@ public void execute(String name, TestConquery testConquery) throws Exception { final Dataset dataset5 = adminDatasetProcessor.addDataset(TEST_DATASET_5); final Dataset dataset6 = adminDatasetProcessor.addDataset(TEST_DATASET_6); + + + + MetaStorage storage = conquery.getMetaStorage(); + + Role role = new Role("role", "ROLE", storage); + Role roleToDelete = new Role("roleDelete", "ROLE_DELETE", storage); + User user = new User("user@test.email", "USER", storage); + User userToDelete = new User("userDelete@test.email", "USER_DELETE", storage); + Group group = new Group("group", "GROUP", storage); + Group groupToDelete = new Group("groupDelete", "GROUP_DELETE", storage); + {// Auth testing (deletion and permission grant) // build constellation //TODO USE APIS @@ -138,9 +144,6 @@ public void execute(String name, TestConquery testConquery) throws Exception { test.executeTest(support); - - MetaStorage storage = conquery.getMetaStorage(); - {// Auth actual tests User userStored = storage.getUser(user.getId()); assertThat(userStored).isEqualTo(user); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/ReusedQueryTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/ReusedQueryTest.java index 2753de8c76..a8efca5647 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/ReusedQueryTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/ReusedQueryTest.java @@ -49,7 +49,6 @@ import com.bakdata.conquery.util.support.TestConquery; import com.github.powerlibraries.io.In; import lombok.extern.slf4j.Slf4j; -import net.sourceforge.argparse4j.impl.Arguments; @Slf4j @@ -220,10 +219,10 @@ public void execute(String name, TestConquery testConquery) throws Exception { reused.setSecondaryId(query.getSecondaryId()); - User shareHolder = new User("shareholder", "ShareHolder"); + User shareHolder = new User("shareholder", "ShareHolder", conquery.getMetaStorage()); conquery.getMetaProcessor().addUser(shareHolder); - shareHolder.addPermissions(metaStorage, Set.of( + shareHolder.addPermissions(Set.of( dataset.createPermission(Set.of(Ability.READ)), execution.createPermission(Set.of(Ability.READ)) )); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/RoleHandlingOnGroupTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/RoleHandlingOnGroupTest.java index 4727ece7a3..205e39d5de 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/RoleHandlingOnGroupTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/RoleHandlingOnGroupTest.java @@ -18,9 +18,7 @@ */ public class RoleHandlingOnGroupTest extends IntegrationTest.Simple implements ProgrammaticIntegrationTest { - private final Group group1 = new Group("company", "company"); - private final Role role = new Role("role1", "role1"); - private final TestUser user1 = new TestUser(); + @Override @@ -28,25 +26,29 @@ public void execute(StandaloneSupport conquery) throws Exception { Dataset dataset1 = new Dataset(); dataset1.setLabel("dataset1"); MetaStorage storage = conquery.getMetaStorage(); - + + Group group1 = new Group("company", "company", storage); + Role role = new Role("role1", "role1", storage); + TestUser user1 = new TestUser(storage); + try { storage.addRole(role); storage.addUser(user1); storage.addGroup(group1); - role.addPermission(storage, new DatasetPermission().instancePermission(Ability.READ, new DatasetId("testDataset"))); + role.addPermission(new DatasetPermission().instancePermission(Ability.READ, new DatasetId("testDataset"))); //// Add user to group - group1.addMember(storage, user1); + group1.addMember(user1); assertThat(user1.isPermitted(new DatasetPermission().instancePermission(Ability.READ, new DatasetId("testDataset")))).isFalse(); //// Add role to group - group1.addRole(storage, role); + group1.addRole(role); assertThat(group1.getRoles()).containsExactlyInAnyOrder(role.getId()); assertThat(user1.isPermitted(new DatasetPermission().instancePermission(Ability.READ, new DatasetId("testDataset")))).isTrue(); //// Remove role from group - group1.removeRole(storage, role); + group1.removeRole(role); assertThat(group1.getRoles()).isEmpty(); assertThat(user1.isPermitted(new DatasetPermission().instancePermission(Ability.READ, new DatasetId("testDataset")))).isFalse(); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/RoleHandlingTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/RoleHandlingTest.java index 96d60e3c55..ba797e04be 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/RoleHandlingTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/RoleHandlingTest.java @@ -15,10 +15,7 @@ */ public class RoleHandlingTest extends IntegrationTest.Simple implements ProgrammaticIntegrationTest { - private final Role mandator1 = new Role("company", "company"); - private final Role mandator1Copy = new Role("company", "company"); - private final Role mandator2 = new Role("company2", "company2"); - private final User user1 = new User("user", "user"); + @Override @@ -26,6 +23,11 @@ public void execute(StandaloneSupport conquery) throws Exception { Dataset dataset1 = new Dataset(); dataset1.setLabel("dataset1"); MetaStorage storage = conquery.getMetaStorage(); + + Role mandator1 = new Role("company", "company", storage); + Role mandator1Copy = new Role("company", "company", storage); + Role mandator2 = new Role("company2", "company2", storage); + User user1 = new User("user", "user", storage); try { storage.addRole(mandator1); @@ -33,21 +35,21 @@ public void execute(StandaloneSupport conquery) throws Exception { storage.addUser(user1); //// ADDING - user1.addRole(storage, mandator1); + user1.addRole(mandator1); assertThat(user1.getRoles()).containsExactlyInAnyOrder(mandator1.getId()); - user1.addRole(storage, mandator1Copy); + user1.addRole(mandator1Copy); assertThat(user1.getRoles()).containsExactlyInAnyOrder(mandator1.getId()); - user1.addRole(storage, mandator2); + user1.addRole(mandator2); assertThat(user1.getRoles()).containsExactlyInAnyOrder(mandator1.getId(), mandator2.getId()); //// REMOVING - user1.removeRole(storage, mandator2); + user1.removeRole(mandator2); assertThat(user1.getRoles()).containsExactlyInAnyOrder(mandator1.getId()); - user1.removeRole(storage, mandator1); + user1.removeRole(mandator1); assertThat(user1.getRoles()).isEmpty(); } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/RoleUITest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/RoleUITest.java index 60e93e7172..7145165d96 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/RoleUITest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/RoleUITest.java @@ -31,25 +31,23 @@ */ public class RoleUITest extends IntegrationTest.Simple implements ProgrammaticIntegrationTest { - - private MetaStorage storage; - private Role mandator = new Role("testMandatorName", "testMandatorLabel"); - private RoleId mandatorId = mandator.getId(); - private User user = new User("testUser@test.de", "testUserName"); - private UserId userId = user.getId(); - private ConqueryPermission permission = DatasetPermission.onInstance(Ability.READ.asSet(), new DatasetId("testDatasetId")); - @Override public void execute(StandaloneSupport conquery) throws Exception { + MetaStorage storage = conquery.getMetaStorage(); + Role mandator = new Role("testMandatorName", "testMandatorLabel", storage); + RoleId mandatorId = mandator.getId(); + User user = new User("testUser@test.de", "testUserName", storage); + UserId userId = user.getId(); try { - - storage = conquery.getMetaStorage(); + + ConqueryPermission permission = DatasetPermission.onInstance(Ability.READ.asSet(), new DatasetId("testDatasetId")); + storage.addRole(mandator); storage.addUser(user); // override permission object, because it might have changed by the subject // owning the permission - mandator.addPermission(storage, permission); - user.addRole(storage, mandator); + mandator.addPermission(permission); + user.addRole(mandator); URI classBase = HierarchyHelper.hierarchicalPath(conquery.defaultAdminURIBuilder(), RoleUIResource.class, "getRole") diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/SuperPermissionTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/SuperPermissionTest.java index 92ee86d4ac..d83c1bd9e5 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/SuperPermissionTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/SuperPermissionTest.java @@ -13,8 +13,7 @@ public class SuperPermissionTest extends IntegrationTest.Simple implements ProgrammaticIntegrationTest { - private final Role role1 = new Role("company", "company"); - private final TestUser user1 = new TestUser(); + @Override @@ -22,25 +21,29 @@ public void execute(StandaloneSupport conquery) throws Exception { Dataset dataset1 = new Dataset(); dataset1.setLabel("dataset1"); MetaStorage storage = conquery.getMetaStorage(); + + Role role1 = new Role("company", "company", storage); + TestUser user1 = new TestUser(storage); + storage.addRole(role1); try { - user1.addRole(storage, role1); + user1.addRole(role1); // Add SuperPermission to User - user1.addPermission(storage, SuperPermission.onDomain()); + user1.addPermission(SuperPermission.onDomain()); assertThat(user1.isPermitted(DatasetPermission.onInstance(Ability.READ, dataset1.getId()))).isTrue(); assertThat(user1.isPermitted(DatasetPermission.onInstance(Ability.DOWNLOAD, dataset1.getId()))).isTrue(); // Add SuperPermission to mandator and remove from user - user1.removePermission(storage, SuperPermission.onDomain()); - role1.addPermission(storage, SuperPermission.onDomain()); + user1.removePermission(SuperPermission.onDomain()); + role1.addPermission(SuperPermission.onDomain()); assertThat(user1.isPermitted(DatasetPermission.onInstance(Ability.READ, dataset1.getId()))).isTrue(); assertThat(user1.isPermitted(DatasetPermission.onInstance(Ability.DOWNLOAD, dataset1.getId()))).isTrue(); // Add SuperPermission to mandator and remove from user - role1.removePermission(storage, SuperPermission.onDomain()); + role1.removePermission(SuperPermission.onDomain()); assertThat(user1.isPermitted(DatasetPermission.onInstance(Ability.READ, dataset1.getId()))).isFalse(); assertThat(user1.isPermitted(DatasetPermission.onInstance(Ability.DOWNLOAD, dataset1.getId()))).isFalse(); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/TestUser.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/TestUser.java index 40f1ceccfe..f07ccbca0b 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/TestUser.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/TestUser.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.integration.tests; +import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.models.auth.entities.User; import org.apache.shiro.authz.Permission; @@ -8,8 +9,8 @@ public class TestUser extends User { - public TestUser() { - super("user", "user"); + public TestUser(MetaStorage storage) { + super("user", "user", storage); } public boolean isPermitted(Permission permission) { diff --git a/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/IdRefrenceTest.java b/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/IdRefrenceTest.java index cf2471ea86..f8c5a44d38 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/IdRefrenceTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/IdRefrenceTest.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.io.jackson.serializer; +import static org.mockito.Mockito.*; import static org.assertj.core.api.Assertions.assertThat; import java.io.IOException; @@ -7,6 +8,7 @@ import java.util.List; import com.bakdata.conquery.io.jackson.Jackson; +import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.datasets.Table; @@ -31,7 +33,7 @@ public void testListReferences() throws IOException { registry.register(table); final CentralRegistry metaRegistry = new CentralRegistry(); - User user = new User("usermail", "userlabel"); + User user = new User("usermail", "userlabel", mock(MetaStorage.class)); metaRegistry.register(user); String json = Jackson.MAPPER.writeValueAsString( diff --git a/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/SerializationTestUtil.java b/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/SerializationTestUtil.java index f13abade17..5555abaeb7 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/SerializationTestUtil.java +++ b/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/SerializationTestUtil.java @@ -6,6 +6,7 @@ import javax.validation.Validator; +import com.bakdata.conquery.io.jackson.Injectable; import com.bakdata.conquery.io.jackson.InternalOnly; import com.bakdata.conquery.io.jackson.Jackson; import com.bakdata.conquery.models.exceptions.JSONException; @@ -19,6 +20,7 @@ import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.databind.ObjectWriter; import io.dropwizard.jersey.validation.Validators; +import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.Setter; import lombok.experimental.Accessors; @@ -34,6 +36,8 @@ public class SerializationTestUtil { private final Validator validator = Validators.newValidator(); @Setter private CentralRegistry registry; + @NonNull + private Injectable[] injectables = {}; public static SerializationTestUtil forType(TypeReference type) { return new SerializationTestUtil<>(Jackson.MAPPER.getTypeFactory().constructType(type)); @@ -43,6 +47,11 @@ public static SerializationTestUtil forType(Class type) { return new SerializationTestUtil<>(Jackson.MAPPER.getTypeFactory().constructType(type)); } + public SerializationTestUtil injectables(Injectable ... injectables) { + this.injectables = injectables; + return this; + } + public void test(T value, T expected) throws JSONException, IOException { test( value, @@ -60,10 +69,13 @@ public void test(T value) throws JSONException, IOException { test(value, value); } - private void test(T value, T expected, ObjectMapper mapper) throws JSONException, IOException { + private void test(T value, T expected, ObjectMapper mapper) throws IOException { if (registry != null) { mapper = new SingletonNamespaceCollection(registry, registry).injectInto(mapper); } + for (Injectable injectable : injectables) { + mapper = injectable.injectInto(mapper); + } ObjectWriter writer = mapper.writerFor(type).withView(InternalOnly.class); ObjectReader reader = mapper.readerFor(type).withView(InternalOnly.class); diff --git a/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/BigStoreTest.java b/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/BigStoreTest.java index 8e07c82b39..3d0b147dd9 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/BigStoreTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/BigStoreTest.java @@ -64,7 +64,7 @@ public void destroy() throws IOException { public void testFull() throws JSONException, IOException { BigStore store = new BigStore<>(new XodusStoreFactory(), Validators.newValidator(), env, - StoreInfo.DICTIONARIES, new ArrayList<>(), (e) -> {}, (e) -> {}, MAPPER + StoreInfo.DICTIONARIES.storeInfo(), new ArrayList<>(), (e) -> {}, (e) -> {}, MAPPER ); @@ -106,7 +106,7 @@ public void testFull() throws JSONException, IOException { @Test public void testEmpty() throws JSONException, IOException { BigStore store = new BigStore<>(new XodusStoreFactory(), Validators.newValidator(), env, - StoreInfo.DICTIONARIES, new ArrayList<>(), (e) -> {}, (e) -> {}, MAPPER + StoreInfo.DICTIONARIES.storeInfo(), new ArrayList<>(), (e) -> {}, (e) -> {}, MAPPER ); store.setChunkByteSize(Ints.checkedCast(DataSize.megabytes(1).toBytes())); diff --git a/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStoreDumpTest.java b/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStoreDumpTest.java index 090f4b885e..f9d112a010 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStoreDumpTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStoreDumpTest.java @@ -1,17 +1,11 @@ package com.bakdata.conquery.io.storage.xodus.stores; -import static org.assertj.core.api.Assertions.assertThat; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Optional; - -import javax.validation.Validator; - +import com.bakdata.conquery.apiv1.query.ConceptQuery; import com.bakdata.conquery.apiv1.query.QueryDescription; +import com.bakdata.conquery.apiv1.query.concept.specific.CQReusedQuery; import com.bakdata.conquery.io.jackson.Jackson; import com.bakdata.conquery.io.storage.IStoreInfo; +import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.io.storage.StoreInfo; import com.bakdata.conquery.io.storage.xodus.stores.SerializingStore.IterationStatistic; import com.bakdata.conquery.models.auth.entities.User; @@ -20,14 +14,12 @@ import com.bakdata.conquery.models.exceptions.JSONException; import com.bakdata.conquery.models.identifiable.ids.specific.UserId; import com.bakdata.conquery.models.query.ManagedQuery; -import com.bakdata.conquery.apiv1.query.ConceptQuery; -import com.bakdata.conquery.apiv1.query.concept.specific.CQReusedQuery; +import com.bakdata.conquery.models.worker.DatasetRegistry; +import com.bakdata.conquery.util.NonPersistentStoreFactory; import com.google.common.io.Files; import io.dropwizard.jersey.validation.Validators; import jetbrains.exodus.env.Environment; import jetbrains.exodus.env.Environments; -import lombok.Getter; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; import org.assertj.core.api.Condition; @@ -35,9 +27,19 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import javax.validation.Validator; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Objects; + +import static org.assertj.core.api.Assertions.assertThat; + @Slf4j public class SerializingStoreDumpTest { + private final static MetaStorage STORAGE = new MetaStorage(Validators.newValidator(), new NonPersistentStoreFactory(), new DatasetRegistry(2)); + public static final IStoreInfo USER_STORE_ID = StoreInfo.AUTH_USER.storeInfo(); private File tmpDir; private Environment env; private XodusStoreFactory config; @@ -46,7 +48,7 @@ public class SerializingStoreDumpTest { private final ManagedQuery managedQuery = new ManagedQuery(null, null, new Dataset("dataset")); private final ConceptQuery cQuery = new ConceptQuery( new CQReusedQuery(managedQuery.getId())); - private final User user = new User("username", "userlabel"); + private final User user = new User("username", "userlabel", STORAGE); @BeforeEach public void init() { @@ -61,8 +63,8 @@ public void destroy() throws IOException { FileUtils.deleteDirectory(tmpDir); } - private SerializingStore createSerializedStore(XodusStoreFactory config, Environment environment, Validator validator, IStoreInfo storeId) { - return new SerializingStore<>(config, new XodusStore(environment, storeId.getName(), new ArrayList<>(), (e) -> {}, (e) -> {}), validator, storeId, config.getObjectMapper(), (Class) storeId.getKeyType(), (Class) storeId.getValueType()); + private SerializingStore createSerializedStore(XodusStoreFactory config, Environment environment, Validator validator, IStoreInfo storeId) { + return new SerializingStore<>(new XodusStore(environment, storeId.getName(), new ArrayList<>(), (e) -> {}, (e) -> {}), validator, config.getObjectMapper(), storeId.getKeyType(), storeId.getValueType(), config.isValidateOnWrite(), config.isRemoveUnreadableFromStore(), config.getUnreadableDataDumpDirectory()); } /** @@ -71,11 +73,11 @@ private SerializingStore createSerializedStore(XodusSto @Test public void testCorruptValueDump() throws JSONException, IOException { // Set dump directory to this tests temp-dir - config.setUnreadableDataDumpDirectory(Optional.of(tmpDir)); + config.setUnreadableDataDumpDirectory(tmpDir); { // Open a store and insert a valid key-value pair (UserId & User) - SerializingStore store = createSerializedStore(config, env, Validators.newValidator(), StoreInfo.AUTH_USER); + SerializingStore store = createSerializedStore(config, env, Validators.newValidator(), USER_STORE_ID); store.add(user.getId(), user); } @@ -86,14 +88,14 @@ public void testCorruptValueDump() throws JSONException, IOException { config, env, Validators.newValidator(), - new CorruptableStoreInfo(StoreInfo.AUTH_USER.getName(), UserId.class, QueryDescription.class)); + new SimpleStoreInfo<>(USER_STORE_ID.getName(), UserId.class, QueryDescription.class)); store.add(new UserId("testU2"), cQuery); } { // Reopen the store with the initial value and try to iterate over all entries // (this triggers the dump or removal of invalid entries) - SerializingStore store = createSerializedStore(config, env, Validators.newValidator(), StoreInfo.AUTH_USER); + SerializingStore store = createSerializedStore(config, env, Validators.newValidator(), USER_STORE_ID); IterationStatistic expectedResult = new IterationStatistic(); expectedResult.setTotalProcessed(2); expectedResult.setFailedKeys(0); @@ -115,7 +117,7 @@ public void testCorruptValueDump() throws JSONException, IOException { } private File getDumpFile(Condition dumpFileCond) { - return tmpDir.listFiles((name) -> dumpFileCond.matches(name))[0]; + return Objects.requireNonNull(tmpDir.listFiles(dumpFileCond::matches))[0]; } /** @@ -124,11 +126,11 @@ private File getDumpFile(Condition dumpFileCond) { @Test public void testCorruptKeyDump() throws JSONException, IOException { // Set dump directory to this tests temp-dir - config.setUnreadableDataDumpDirectory(Optional.of(tmpDir)); + config.setUnreadableDataDumpDirectory(tmpDir); { // Open a store and insert a valid key-value pair (UserId & User) - SerializingStore store = createSerializedStore(config, env, Validators.newValidator(), StoreInfo.AUTH_USER); + SerializingStore store = createSerializedStore(config, env, Validators.newValidator(), USER_STORE_ID); store.add(new UserId("testU1"), user); } @@ -139,14 +141,14 @@ public void testCorruptKeyDump() throws JSONException, IOException { config, env, Validators.newValidator(), - new CorruptableStoreInfo(StoreInfo.AUTH_USER.getName(), String.class, QueryDescription.class)); + new SimpleStoreInfo<>(USER_STORE_ID.getName(), String.class, QueryDescription.class)); store.add("not a valid conquery Id", cQuery); } { // Reopen the store with the initial value and try to iterate over all entries // (this triggers the dump or removal of invalid entries) - SerializingStore store = createSerializedStore(config, env, Validators.newValidator(), StoreInfo.AUTH_USER); + SerializingStore store = createSerializedStore(config, env, Validators.newValidator(), USER_STORE_ID); IterationStatistic expectedResult = new IterationStatistic(); expectedResult.setTotalProcessed(2); expectedResult.setFailedKeys(1); @@ -179,7 +181,7 @@ public void testCorruptionRemoval() throws JSONException, IOException { { // Open a store and insert a valid key-value pair (UserId & User) - SerializingStore store = createSerializedStore(config, env, Validators.newValidator(), StoreInfo.AUTH_USER); + SerializingStore store = createSerializedStore(config, env, Validators.newValidator(), USER_STORE_ID); store.add(new UserId("testU1"), user); } @@ -190,7 +192,7 @@ public void testCorruptionRemoval() throws JSONException, IOException { config, env, Validators.newValidator(), - new CorruptableStoreInfo(StoreInfo.AUTH_USER.getName(), String.class, QueryDescription.class)); + new SimpleStoreInfo<>(USER_STORE_ID.getName(), String.class, QueryDescription.class)); store.add("not a valid conquery Id", cQuery); } @@ -199,7 +201,7 @@ public void testCorruptionRemoval() throws JSONException, IOException { config, env, Validators.newValidator(), - new CorruptableStoreInfo(StoreInfo.AUTH_USER.getName(), UserId.class, QueryDescription.class)); + new SimpleStoreInfo<>(USER_STORE_ID.getName(), UserId.class, QueryDescription.class)); store.add(new UserId("testU2"), cQuery); } } @@ -207,7 +209,7 @@ public void testCorruptionRemoval() throws JSONException, IOException { { // Reopen the store with correct configuration and try to iterate over all // entries (this triggers the dump or removal of invalid entries) - SerializingStore store = createSerializedStore(config, env, Validators.newValidator(), StoreInfo.AUTH_USER); + SerializingStore store = createSerializedStore(config, env, Validators.newValidator(), USER_STORE_ID); IterationStatistic expectedResult = new IterationStatistic(); expectedResult.setTotalProcessed(3); expectedResult.setFailedKeys(1); @@ -220,7 +222,7 @@ public void testCorruptionRemoval() throws JSONException, IOException { { // Reopen again to check that the corrupted values have been removed previously - SerializingStore store = createSerializedStore(config, env, Validators.newValidator(), StoreInfo.AUTH_USER); + SerializingStore store = createSerializedStore(config, env, Validators.newValidator(), USER_STORE_ID); IterationStatistic expectedResult = new IterationStatistic(); expectedResult.setTotalProcessed(1); expectedResult.setFailedKeys(0); @@ -231,13 +233,4 @@ public void testCorruptionRemoval() throws JSONException, IOException { assertThat(result).isEqualTo(expectedResult); } } - - @RequiredArgsConstructor - @Getter - private static class CorruptableStoreInfo implements IStoreInfo { - - private final String name; - private final Class keyType; - private final Class valueType; - } } diff --git a/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java b/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java index b9e13df8ce..1af1fca99c 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java +++ b/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java @@ -51,15 +51,19 @@ import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; import com.bakdata.conquery.models.query.ManagedQuery; import com.bakdata.conquery.models.query.entity.Entity; +import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.util.NonPersistentStoreFactory; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import io.dropwizard.jersey.validation.Validators; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Test; @Slf4j public class SerializationTests { + private final static MetaStorage STORAGE = new MetaStorage(Validators.newValidator(), new NonPersistentStoreFactory(), new DatasetRegistry(2)); + @Test public void dataset() throws IOException, JSONException { Dataset dataset = new Dataset(); @@ -81,10 +85,11 @@ public void passwordCredential() throws IOException, JSONException { @Test public void role() throws IOException, JSONException { - Role mandator = new Role("company", "company"); + Role mandator = new Role("company", "company", STORAGE); SerializationTestUtil .forType(Role.class) + .injectables(STORAGE) .test(mandator); } @@ -94,15 +99,14 @@ public void role() throws IOException, JSONException { @Test public void user() throws IOException, JSONException { MetaStorage storage = new MetaStorage(null, new NonPersistentStoreFactory(), null); - User user = new User("user", "user"); - user.addPermission(storage, DatasetPermission.onInstance(Ability.READ, new DatasetId("test"))); + User user = new User("user", "user", STORAGE); + user.addPermission(DatasetPermission.onInstance(Ability.READ, new DatasetId("test"))); user .addPermission( - storage, ExecutionPermission.onInstance(Ability.READ, new ManagedExecutionId(new DatasetId("dataset"), UUID.randomUUID())) ); - Role role = new Role("company", "company"); - user.addRole(storage, role); + Role role = new Role("company", "company", STORAGE); + user.addRole(role); CentralRegistry registry = new CentralRegistry(); registry.register(role); @@ -110,31 +114,35 @@ public void user() throws IOException, JSONException { SerializationTestUtil .forType(User.class) .registry(registry) + .injectables(STORAGE) .test(user); } @Test public void group() throws IOException, JSONException { MetaStorage storage = new MetaStorage(null, new NonPersistentStoreFactory(), null); - Group group = new Group("group", "group"); - group.addPermission(storage, DatasetPermission.onInstance(Ability.READ, new DatasetId("test"))); + Group group = new Group("group", "group", STORAGE); + group.addPermission(DatasetPermission.onInstance(Ability.READ, new DatasetId("test"))); group .addPermission( - storage, ExecutionPermission.onInstance(Ability.READ, new ManagedExecutionId(new DatasetId("dataset"), UUID.randomUUID())) ); - group.addRole(storage, new Role("company", "company")); + group.addRole(new Role("company", "company", STORAGE)); - Role role = new Role("company", "company"); - group.addRole(storage, role); - User user = new User("userName", "userLabel"); - group.addMember(storage, user); + Role role = new Role("company", "company", STORAGE); + group.addRole(role); + User user = new User("userName", "userLabel", STORAGE); + group.addMember(user); CentralRegistry registry = new CentralRegistry(); registry.register(role); registry.register(user); - SerializationTestUtil.forType(Group.class).registry(registry).test(group); + SerializationTestUtil + .forType(Group.class) + .injectables(STORAGE) + .registry(registry) + .test(group); } @Test @@ -239,7 +247,7 @@ public void managedQuery() throws JSONException, IOException { final Dataset dataset = new Dataset("test-dataset"); - final User user = new User("test-user", "test-user"); + final User user = new User("test-user", "test-user", STORAGE); registry.register(dataset); registry.register(user); @@ -319,7 +327,7 @@ public void executionQueryJobError() throws JSONException, IOException { @Test public void meInformation() throws IOException, JSONException { - User user = new User("name", "labe"); + User user = new User("name", "labe", STORAGE); MeProcessor.FEMeInformation info = MeProcessor.FEMeInformation.builder() .userName(user.getLabel()) diff --git a/backend/src/test/java/com/bakdata/conquery/models/auth/CopyUserTest.java b/backend/src/test/java/com/bakdata/conquery/models/auth/CopyUserTest.java index 607f8f0591..ca96c6de74 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/auth/CopyUserTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/auth/CopyUserTest.java @@ -25,20 +25,20 @@ void testUserCopy(){ registry.setMetaStorage(storage); // Create test role - Role role = new Role("role", "role"); + Role role = new Role("role", "role", storage); storage.addRole(role); - role.addPermission(storage, DatasetPermission.onInstance(Ability.READ, new DatasetId("dataset0"))); + role.addPermission(DatasetPermission.onInstance(Ability.READ, new DatasetId("dataset0"))); // Create test group - Group group = new Group("group", "group"); + Group group = new Group("group", "group", storage); storage.addGroup(group); - group.addPermission(storage, DatasetPermission.onInstance(Ability.READ, new DatasetId("dataset1"))); + group.addPermission(DatasetPermission.onInstance(Ability.READ, new DatasetId("dataset1"))); // Create original user with role and group mapping - User originUser = new User("user", "user"); + User originUser = new User("user", "user", storage); storage.addUser(originUser); - originUser.addRole(storage, role); - group.addMember(storage, originUser); + originUser.addRole(role); + group.addMember(originUser); // Do copy User copy = AuthorizationController.flatCopyUser(originUser, "copytest", storage); diff --git a/backend/src/test/java/com/bakdata/conquery/models/auth/IntrospectionDelegatingRealmTest.java b/backend/src/test/java/com/bakdata/conquery/models/auth/IntrospectionDelegatingRealmTest.java index bd8ce01680..f17a44bca6 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/auth/IntrospectionDelegatingRealmTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/auth/IntrospectionDelegatingRealmTest.java @@ -60,7 +60,7 @@ public class IntrospectionDelegatingRealmTest { private static final BearerToken USER_3_TOKEN_WRAPPED = new BearerToken(USER_3_TOKEN); // Groups private static final String GROUPNAME_1 = "group1"; - private static final Group GROUP_1_EXISTING = new Group(GROUPNAME_1, GROUPNAME_1); + private static final Group GROUP_1_EXISTING = new Group(GROUPNAME_1, GROUPNAME_1, STORAGE); private static final String GROUPNAME_2 = "group2"; // Group is created during test private static OIDCMockServer OIDC_SERVER; @@ -157,12 +157,12 @@ public void tokenIntrospectionSimpleUserNew() { .usingRecursiveComparison() .ignoringFields(ConqueryAuthenticationInfo.Fields.credentials) .isEqualTo(new ConqueryAuthenticationInfo(new UserId(USER_1_NAME), USER1_TOKEN_WRAPPED, REALM, true)); - assertThat(STORAGE.getAllUsers()).containsOnly(new User(USER_1_NAME, USER_1_NAME)); + assertThat(STORAGE.getAllUsers()).containsOnly(new User(USER_1_NAME, USER_1_NAME, STORAGE)); } @Test public void tokenIntrospectionSimpleUserExisting() { - User existingUser = new User(USER_1_NAME, USER_1_NAME); + User existingUser = new User(USER_1_NAME, USER_1_NAME, STORAGE); STORAGE.addUser(existingUser); AuthenticationInfo info = REALM.doGetAuthenticationInfo(USER1_TOKEN_WRAPPED); @@ -182,7 +182,7 @@ public void tokenIntrospectionGroupedUser() { .usingRecursiveComparison() .ignoringFields(ConqueryAuthenticationInfo.Fields.credentials) .isEqualTo(new ConqueryAuthenticationInfo(new UserId(USER_2_NAME), USER_2_TOKEN_WRAPPED, REALM, true)); - assertThat(STORAGE.getAllUsers()).containsOnly(new User(USER_2_NAME, USER_2_LABEL)); + assertThat(STORAGE.getAllUsers()).containsOnly(new User(USER_2_NAME, USER_2_LABEL, STORAGE)); assertThat(STORAGE.getAllGroups()).hasSize(2); // Pre-existing group and a second group that has been added in the process assertThat(STORAGE.getGroup(new GroupId(GROUPNAME_1)).getMembers()).contains(new UserId(USER_2_NAME)); assertThat(STORAGE.getGroup(new GroupId(GROUPNAME_2)).getMembers()).contains(new UserId(USER_2_NAME)); @@ -190,9 +190,9 @@ public void tokenIntrospectionGroupedUser() { @Test public void tokenIntrospectionGroupedUserRemoveGroupMapping() { - User user = new User(USER_3_NAME, USER_3_LABEL); + User user = new User(USER_3_NAME, USER_3_LABEL, STORAGE); STORAGE.addUser(user); - GROUP_1_EXISTING.addMember(STORAGE, user); + GROUP_1_EXISTING.addMember(user); assertThat(STORAGE.getGroup(new GroupId(GROUPNAME_1)).getMembers()).contains(new UserId(USER_3_NAME)); @@ -202,7 +202,7 @@ public void tokenIntrospectionGroupedUserRemoveGroupMapping() { .usingRecursiveComparison() .ignoringFields(ConqueryAuthenticationInfo.Fields.credentials) .isEqualTo(new ConqueryAuthenticationInfo(new UserId(USER_3_NAME), USER_3_TOKEN_WRAPPED, REALM, true)); - assertThat(STORAGE.getAllUsers()).containsOnly(new User(USER_3_NAME, USER_3_LABEL)); + assertThat(STORAGE.getAllUsers()).containsOnly(new User(USER_3_NAME, USER_3_LABEL, STORAGE)); assertThat(STORAGE.getAllGroups()).hasSize(1); // Pre-existing group assertThat(STORAGE.getGroup(new GroupId(GROUPNAME_1)).getMembers()).doesNotContain(new UserId(USER_3_NAME)); } diff --git a/backend/src/test/java/com/bakdata/conquery/models/auth/LocalAuthRealmTest.java b/backend/src/test/java/com/bakdata/conquery/models/auth/LocalAuthRealmTest.java index 063934e6e9..670b0eff8e 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/auth/LocalAuthRealmTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/auth/LocalAuthRealmTest.java @@ -59,7 +59,7 @@ public void setupAll() throws Exception { @BeforeEach public void setupEach() { // Create User in Realm - user1 = new User("TestUser", "Test User"); + user1 = new User("TestUser", "Test User", storage); PasswordCredential user1Password = new PasswordCredential("testPassword".toCharArray()); storage.addUser(user1); realm.addUser(user1, List.of(user1Password)); diff --git a/backend/src/test/java/com/bakdata/conquery/models/execution/DefaultLabelTest.java b/backend/src/test/java/com/bakdata/conquery/models/execution/DefaultLabelTest.java index 29c35b8775..906b75bf2c 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/execution/DefaultLabelTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/execution/DefaultLabelTest.java @@ -10,6 +10,7 @@ import java.util.UUID; import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; +import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; import com.bakdata.conquery.models.config.ConqueryConfig; @@ -24,6 +25,8 @@ import com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal; import com.bakdata.conquery.apiv1.query.concept.specific.CQReusedQuery; import com.bakdata.conquery.models.worker.DatasetRegistry; +import com.bakdata.conquery.util.NonPersistentStoreFactory; +import io.dropwizard.jersey.validation.Validators; import org.jetbrains.annotations.NotNull; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.params.ParameterizedTest; @@ -31,9 +34,12 @@ import org.mockito.Mockito; public class DefaultLabelTest { + + private final static MetaStorage STORAGE = new MetaStorage(Validators.newValidator(), new NonPersistentStoreFactory(), new DatasetRegistry(2)); + private static final DatasetRegistry DATASET_REGISTRY = Mockito.mock(DatasetRegistry.class); private static final Dataset DATASET = new Dataset("dataset"); - private static final User user = new User("user","user"); + private static final User user = new User("user","user", STORAGE); private static final TreeConcept CONCEPT = new TreeConcept() { { diff --git a/backend/src/test/java/com/bakdata/conquery/tasks/PermissionCleanupTaskTest.java b/backend/src/test/java/com/bakdata/conquery/tasks/PermissionCleanupTaskTest.java index b4e6ab9a6c..debbc14e33 100644 --- a/backend/src/test/java/com/bakdata/conquery/tasks/PermissionCleanupTaskTest.java +++ b/backend/src/test/java/com/bakdata/conquery/tasks/PermissionCleanupTaskTest.java @@ -56,9 +56,9 @@ void doNotDeletePermissionValidReference() { final ManagedQuery managedQuery = createManagedQuery(); // Saving the Execution - User user = new User("test", "test"); + User user = new User("test", "test", storage); storage.updateUser(user); - user.addPermission(storage, ExecutionPermission.onInstance(AbilitySets.QUERY_CREATOR, managedQuery.getId())); + user.addPermission(ExecutionPermission.onInstance(AbilitySets.QUERY_CREATOR, managedQuery.getId())); deleteQueryPermissionsWithMissingRef(storage, storage.getAllUsers()); @@ -73,9 +73,9 @@ void doDeletePermissionInvalidReference() { final ManagedQuery managedQuery = createManagedQuery(); // Removing the execution storage.removeExecution(managedQuery.getId()); - User user = new User("test", "test"); + User user = new User("test", "test", storage); storage.updateUser(user); - user.addPermission(storage, ExecutionPermission.onInstance(AbilitySets.QUERY_CREATOR, managedQuery.getId())); + user.addPermission(ExecutionPermission.onInstance(AbilitySets.QUERY_CREATOR, managedQuery.getId())); deleteQueryPermissionsWithMissingRef(storage, storage.getAllUsers()); @@ -91,11 +91,10 @@ void doDeletePartialPermissionWithInvalidReference() { final ManagedQuery managedQuery2 = createManagedQuery(); // Removing the second execution storage.removeExecution(managedQuery2.getId()); - User user = new User("test", "test"); + User user = new User("test", "test", storage); storage.updateUser(user); user.addPermission( - storage, - // Build a permission with multiple instances + // Build a permission with multiple instances new WildcardPermission(List.of( Set.of(ExecutionPermission.DOMAIN), Set.of(Ability.READ.toString().toLowerCase()), @@ -115,11 +114,11 @@ void doDeletePermissionsOfOwnedReference() { // Created owned execution final ManagedQuery managedQueryOwned = createManagedQuery(); // Setup user - User user = new User("test", "test"); - User user2 = new User("test2", "test2"); + User user = new User("test", "test", storage); + User user2 = new User("test2", "test2", storage); storage.updateUser(user); - user.addPermission(storage, ExecutionPermission.onInstance(AbilitySets.QUERY_CREATOR, managedQueryOwned.getId())); + user.addPermission(ExecutionPermission.onInstance(AbilitySets.QUERY_CREATOR, managedQueryOwned.getId())); managedQueryOwned.setOwner(user); storage.updateExecution(managedQueryOwned); @@ -127,7 +126,7 @@ void doDeletePermissionsOfOwnedReference() { // Created not owned execution final ManagedQuery managedQueryNotOwned = createManagedQuery(); // Setup user - user.addPermission(storage, ExecutionPermission.onInstance(Ability.READ, managedQueryNotOwned.getId())); + user.addPermission(ExecutionPermission.onInstance(Ability.READ, managedQueryNotOwned.getId())); // Set owner managedQueryNotOwned.setOwner(user2); diff --git a/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java b/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java index a88c4f8c30..d5aff76281 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java +++ b/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java @@ -117,17 +117,17 @@ public IdentifiableStore createFormConfigStore(CentralRegistry centr } @Override - public IdentifiableStore createUserStore(CentralRegistry centralRegistry, String pathName) { + public IdentifiableStore createUserStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage) { return AUTH_USER.identifiable(new NonPersistentStore(), centralRegistry); } @Override - public IdentifiableStore createRoleStore(CentralRegistry centralRegistry, String pathName) { + public IdentifiableStore createRoleStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage) { return AUTH_ROLE.identifiable(new NonPersistentStore(), centralRegistry); } @Override - public IdentifiableStore createGroupStore(CentralRegistry centralRegistry, String pathName) { + public IdentifiableStore createGroupStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage) { return AUTH_GROUP.identifiable(new NonPersistentStore(), centralRegistry); } diff --git a/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java b/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java index b0eb91901a..b85a4983b9 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java +++ b/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java @@ -22,6 +22,7 @@ import com.bakdata.conquery.integration.IntegrationTests; import com.bakdata.conquery.integration.json.JsonIntegrationTest; import com.bakdata.conquery.io.jackson.Jackson; +import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.config.XodusStoreFactory; @@ -287,7 +288,8 @@ private boolean isBusy() { } public void beforeEach() { - testUser = standaloneCommand.getManager().getConfig().getAuthorizationRealms().getInitialUsers().get(0).getUser(); - standaloneCommand.getManager().getStorage().updateUser(testUser); + final MetaStorage storage = standaloneCommand.getManager().getStorage(); + testUser = standaloneCommand.getManager().getConfig().getAuthorizationRealms().getInitialUsers().get(0).getUser(storage, true).orElseThrow(); + storage.updateUser(testUser); } } diff --git a/backend/src/test/java/com/bakdata/conquery/util/support/TestLoggingFactory.java b/backend/src/test/java/com/bakdata/conquery/util/support/TestLoggingFactory.java index 7e13bda4da..983494c0d7 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/support/TestLoggingFactory.java +++ b/backend/src/test/java/com/bakdata/conquery/util/support/TestLoggingFactory.java @@ -176,6 +176,7 @@ private Logger configureLoggers() { root.setLevel(Level.WARN); loggerContext.getLogger("com.bakdata").setLevel(Level.DEBUG); + loggerContext.getLogger("com.bakdata.conquery.io.storage.xodus.stores").setLevel(Level.DEBUG); return root; } From 8904ff938d531546e07b23a32bf15df5dfa1ba3d Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 21 Sep 2021 14:02:07 +0200 Subject: [PATCH 02/35] use an injectable storageUpdater instead of the storage it self --- .../conquery/apiv1/auth/ProtoUser.java | 2 +- .../conquery/io/storage/MetaStorage.java | 3 ++ .../models/auth/AuthorizationController.java | 2 +- .../conquery/models/auth/entities/Group.java | 49 ++++++++++++++++--- .../models/auth/entities/PermissionOwner.java | 23 +++------ .../conquery/models/auth/entities/Role.java | 49 ++++++++++++++++--- .../conquery/models/auth/entities/User.java | 42 +++++++++++++--- .../oidc/IntrospectionDelegatingRealm.java | 4 +- .../models/config/XodusStoreFactory.java | 6 +-- .../api/StoredQueriesProcessorTest.java | 2 +- .../api/form/config/FormConfigTest.java | 20 +++----- .../integration/DownloadLinkGeneration.java | 2 +- .../tests/ConceptPermissionTest.java | 2 +- .../integration/tests/GroupHandlingTest.java | 8 +-- .../tests/PermissionGroupHandlingTest.java | 4 +- .../tests/PermissionRoleHandlingTest.java | 2 +- .../integration/tests/RestartTest.java | 12 ++--- .../integration/tests/ReusedQueryTest.java | 2 +- .../tests/RoleHandlingOnGroupTest.java | 4 +- .../integration/tests/RoleHandlingTest.java | 8 +-- .../integration/tests/RoleUITest.java | 4 +- .../tests/SuperPermissionTest.java | 2 +- .../conquery/integration/tests/TestUser.java | 2 +- .../io/jackson/serializer/IdRefrenceTest.java | 2 +- .../stores/SerializingStoreDumpTest.java | 7 ++- .../conquery/models/SerializationTests.java | 18 +++---- .../conquery/models/auth/CopyUserTest.java | 6 +-- .../IntrospectionDelegatingRealmTest.java | 12 ++--- .../models/auth/LocalAuthRealmTest.java | 2 +- .../models/execution/DefaultLabelTest.java | 2 +- .../tasks/PermissionCleanupTaskTest.java | 10 ++-- 31 files changed, 200 insertions(+), 113 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/auth/ProtoUser.java b/backend/src/main/java/com/bakdata/conquery/apiv1/auth/ProtoUser.java index b8d343f65c..0effb88e0a 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/auth/ProtoUser.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/auth/ProtoUser.java @@ -56,7 +56,7 @@ public Optional getUser(@NonNull MetaStorage storage, boolean override) { if (label == null) { label = name; } - user = new User(name, label, storage); + user = new User(name, label, storage::updateUser); storage.updateUser(user); for (String sPermission : permissions) { user.addPermission(new WildcardPermission(sPermission)); diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java index d423a33367..7eef215a98 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java @@ -2,6 +2,7 @@ import java.io.IOException; import java.util.Collection; +import java.util.function.Consumer; import javax.validation.Validator; @@ -201,4 +202,6 @@ public void close() throws IOException { public MutableInjectableValues inject(MutableInjectableValues values) { return values.add(MetaStorage.class, this); } + + public interface StorageUpdater extends Consumer, Injectable{}; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationController.java b/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationController.java index 8ba051408d..5959fc6a3d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationController.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationController.java @@ -193,7 +193,7 @@ public static User flatCopyUser(@NonNull User originUser, String namePrefix, @No ); // Create copied user - User copy = new User(name, originUser.getLabel(), storage); + User copy = new User(name, originUser.getLabel(), storage::updateUser); storage.addUser(copy); copy.updatePermissions(copiedPermission); diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Group.java b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Group.java index b0d95b3e19..1c3f29f19f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Group.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Group.java @@ -3,12 +3,19 @@ import java.util.Collections; import java.util.HashSet; import java.util.Set; +import java.util.function.Consumer; +import com.bakdata.conquery.io.jackson.MutableInjectableValues; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.models.identifiable.ids.specific.GroupId; import com.bakdata.conquery.models.identifiable.ids.specific.RoleId; import com.bakdata.conquery.models.identifiable.ids.specific.UserId; +import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.OptBoolean; +import lombok.EqualsAndHashCode; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; /** @@ -25,13 +32,20 @@ public class Group extends PermissionOwner implements RoleOwner { @JsonProperty private Set roles = Collections.synchronizedSet(new HashSet<>()); - public Group(String name, String label, MetaStorage storage) { - super(name, label, storage); + + @JacksonInject(useInput = OptBoolean.FALSE) + @NonNull + @EqualsAndHashCode.Exclude + protected StorageUpdater storageUpdater; + + public Group(String name, String label, Consumer storageUpdater) { + super(name, label); + this.storageUpdater = new StorageUpdater(storageUpdater); } @Override - protected void updateStorage(MetaStorage storage) { - storage.updateGroup(this); + protected void updateStorage() { + storageUpdater.accept(this); } @Override @@ -42,14 +56,14 @@ public GroupId createId() { public synchronized void addMember(User user) { if(members.add(user.getId())) { log.trace("Added user {} to group {}", user.getId(), getId()); - updateStorage(storage); + updateStorage(); } } public synchronized void removeMember(User user) { if (members.remove(user.getId())) { log.trace("Removed user {} from group {}", user.getId(), getId()); - updateStorage(storage); + updateStorage(); } } @@ -64,18 +78,37 @@ public Set getMembers() { public synchronized void addRole(Role role) { if (roles.add(role.getId())) { log.trace("Added role {} to group {}", role.getId(), getId()); - updateStorage(storage); + updateStorage(); } } public synchronized void removeRole(Role role) { if (roles.remove(role.getId())) { log.trace("Removed role {} from group {}", role.getId(), getId()); - updateStorage(storage); + updateStorage(); } } public Set getRoles() { return Collections.unmodifiableSet(roles); } + + + + @RequiredArgsConstructor + public static class StorageUpdater implements MetaStorage.StorageUpdater { + + private final Consumer storageUpdater; + + @Override + public void accept(Group group) { + storageUpdater.accept(group); + } + + @Override + public MutableInjectableValues inject(MutableInjectableValues values) { + values.add(Group.StorageUpdater.class, this); + return values; + } + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/PermissionOwner.java b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/PermissionOwner.java index 052957a59b..e072f503bb 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/PermissionOwner.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/PermissionOwner.java @@ -12,9 +12,6 @@ import com.bakdata.conquery.models.auth.permissions.ConqueryPermission; import com.bakdata.conquery.models.identifiable.IdentifiableImpl; import com.bakdata.conquery.models.identifiable.ids.specific.PermissionOwnerId; -import com.fasterxml.jackson.annotation.JacksonInject; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.OptBoolean; import com.google.common.collect.ImmutableSet; import lombok.AccessLevel; import lombok.EqualsAndHashCode; @@ -60,16 +57,10 @@ public abstract class PermissionOwner permissions = new HashSet<>(); - @JacksonInject(useInput = OptBoolean.FALSE) - @NonNull - @EqualsAndHashCode.Exclude - protected MetaStorage storage; - - public PermissionOwner(String name, String label, MetaStorage storage) { + public PermissionOwner(String name, String label) { this.name = name; this.label = label; - this.storage = storage; } @@ -85,7 +76,7 @@ public synchronized void addPermissions(Set permissions) { .addAll(this.permissions) .addAll(permissions) .build(); - updateStorage(this.storage); + updateStorage(); } public synchronized void addPermission(ConqueryPermission permission) { @@ -94,7 +85,7 @@ public synchronized void addPermission(ConqueryPermission permission) { .addAll(this.permissions) .add(permission) .build(); - updateStorage(this.storage); + updateStorage(); } /** @@ -109,7 +100,7 @@ public boolean removePermissions(Set permissions) { Set newSet = new HashSet<>(this.permissions); ret = newSet.removeAll(permissions); this.permissions = newSet; - updateStorage(this.storage); + updateStorage(); } return ret; } @@ -120,7 +111,7 @@ public boolean removePermission(Permission permission) { Set newSet = new HashSet<>(this.permissions); ret = newSet.remove(permission); this.permissions = newSet; - updateStorage(this.storage); + updateStorage(); } return ret; } @@ -145,14 +136,14 @@ public void updatePermissions(Set permissionsNew) { Set newSet = new HashSet<>(permissionsNew.size()); newSet.addAll(permissionsNew); this.permissions = newSet; - updateStorage(this.storage); + updateStorage(); } } /** * Update this instance in the {@link MetaStorage}. */ - protected abstract void updateStorage(MetaStorage storage); + protected abstract void updateStorage(); @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Role.java b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Role.java index 1f73a79c24..7f4d68a368 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Role.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Role.java @@ -1,24 +1,57 @@ package com.bakdata.conquery.models.auth.entities; +import java.util.function.Consumer; + +import com.bakdata.conquery.io.jackson.MutableInjectableValues; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.models.identifiable.ids.specific.RoleId; +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.annotation.OptBoolean; +import lombok.EqualsAndHashCode; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; public class Role extends PermissionOwner { - public Role(String name, String label, MetaStorage storage) { - super(name, label, storage); - } - @Override public RoleId createId() { return new RoleId(name); } - + + @JacksonInject(useInput = OptBoolean.FALSE) + @NonNull + @EqualsAndHashCode.Exclude + protected StorageUpdater storageUpdater; + + public Role(String name, String label, Consumer storageUpdater) { + super(name, label); + this.storageUpdater = new StorageUpdater(storageUpdater); + } + @Override - protected void updateStorage(MetaStorage storage) { - storage.updateRole(this); - + protected void updateStorage() { + storageUpdater.accept(this); + + } + + + + @RequiredArgsConstructor + public static class StorageUpdater implements MetaStorage.StorageUpdater { + + private final Consumer storageUpdater; + + @Override + public void accept(Role role) { + storageUpdater.accept(role); + } + + @Override + public MutableInjectableValues inject(MutableInjectableValues values) { + values.add(StorageUpdater.class, this); + return values; + } } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/User.java b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/User.java index 7d5485923c..0c8d8aac54 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/User.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/User.java @@ -7,8 +7,10 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.function.Consumer; import java.util.stream.Collectors; +import com.bakdata.conquery.io.jackson.MutableInjectableValues; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.models.auth.permissions.Ability; import com.bakdata.conquery.models.auth.permissions.Authorized; @@ -16,12 +18,16 @@ import com.bakdata.conquery.models.execution.Owned; import com.bakdata.conquery.models.identifiable.ids.specific.RoleId; import com.bakdata.conquery.models.identifiable.ids.specific.UserId; +import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.OptBoolean; import it.unimi.dsi.fastutil.booleans.BooleanArrayList; import lombok.AccessLevel; +import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.NonNull; +import lombok.RequiredArgsConstructor; import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.apache.shiro.SecurityUtils; @@ -45,9 +51,16 @@ public class User extends PermissionOwner implements Principal, RoleOwne @Getter(AccessLevel.PROTECTED) private final transient ShiroUserAdapter shiroUserAdapter; - public User(String name, String label, MetaStorage storage) { - super(name, label, storage); + + @JacksonInject(useInput = OptBoolean.FALSE) + @NonNull + @EqualsAndHashCode.Exclude + protected User.StorageUpdater storageUpdater; + + public User(String name, String label, Consumer storageUpdater) { + super(name, label); this.shiroUserAdapter = new ShiroUserAdapter(); + this.storageUpdater = new StorageUpdater(storageUpdater); } @Override @@ -58,7 +71,7 @@ public UserId createId() { public synchronized void addRole(Role role) { if (roles.add(role.getId())) { log.trace("Added role {} to user {}", role.getId(), getId()); - updateStorage(storage); + updateStorage(); } } @@ -66,7 +79,7 @@ public synchronized void addRole(Role role) { public synchronized void removeRole(Role role) { if (roles.remove(role.getId())) { log.trace("Removed role {} from user {}", role.getId(), getId()); - updateStorage(storage); + updateStorage(); } } @@ -75,8 +88,8 @@ public Set getRoles() { } @Override - protected void updateStorage(MetaStorage storage) { - storage.updateUser(this); + protected void updateStorage() { + storageUpdater.accept(this); } public void authorize(@NonNull Authorized object, @NonNull Ability ability) { @@ -164,4 +177,21 @@ public Object getPrincipal() { } + + @RequiredArgsConstructor + public static class StorageUpdater implements MetaStorage.StorageUpdater { + + private final Consumer storageUpdater; + + @Override + public void accept(User user) { + storageUpdater.accept(user); + } + + @Override + public MutableInjectableValues inject(MutableInjectableValues values) { + values.add(StorageUpdater.class, this); + return values; + } + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/IntrospectionDelegatingRealm.java b/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/IntrospectionDelegatingRealm.java index bbc2184346..e8af2d6ad5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/IntrospectionDelegatingRealm.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/IntrospectionDelegatingRealm.java @@ -195,7 +195,7 @@ private synchronized User getOrCreateUser(TokenIntrospectionSuccessResponse succ } // try to construct a new User if none could be found in the storage String userLabel = successResponse.getStringParameter("name"); - user = new User(username, userLabel != null ? userLabel : username, storage); + user = new User(username, userLabel != null ? userLabel : username, storage::updateUser); storage.addUser(user); log.info("Created new user: {}", user); return user; @@ -213,7 +213,7 @@ private synchronized Group getOrCreateGroup(Pair groupNameId) { if (group != null) { return group; } - group = new Group(groupNameId.getValue().getGroup(), groupNameId.getKey(), storage); + group = new Group(groupNameId.getValue().getGroup(), groupNameId.getKey(), storage::updateGroup); storage.addGroup(group); log.info("Created new group: {}", group); return group; diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java index 753908b1b0..53a9f4ab8a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java @@ -316,18 +316,18 @@ public IdentifiableStore createFormConfigStore(CentralRegistry centr @Override public IdentifiableStore createUserStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage) { - return StoreInfo.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "users")), validator, AUTH_USER), centralRegistry, storage); + return StoreInfo.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "users")), validator, AUTH_USER), centralRegistry, new User.StorageUpdater(storage::updateUser)); } @Override public IdentifiableStore createRoleStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage) { - return StoreInfo.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "roles")), validator, AUTH_ROLE), centralRegistry, storage); + return StoreInfo.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "roles")), validator, AUTH_ROLE), centralRegistry, new Role.StorageUpdater(storage::updateRole)); } @Override public IdentifiableStore createGroupStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage) { - return StoreInfo.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "groups")), validator, AUTH_GROUP), centralRegistry, storage); + return StoreInfo.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "groups")), validator, AUTH_GROUP), centralRegistry, new Group.StorageUpdater(storage::updateGroup)); } @Override diff --git a/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java b/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java index 179860a07c..b21f1828a8 100644 --- a/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java +++ b/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java @@ -117,7 +117,7 @@ public void getQueriesFiltered() { } private static User mockUser(int id, List allowedQueryIds) { - final User user = new User("user" + id, null, STORAGE); + final User user = new User("user" + id, null, STORAGE::updateUser); STORAGE.addUser(user); diff --git a/backend/src/test/java/com/bakdata/conquery/api/form/config/FormConfigTest.java b/backend/src/test/java/com/bakdata/conquery/api/form/config/FormConfigTest.java index c0f6c8764e..7df060731f 100644 --- a/backend/src/test/java/com/bakdata/conquery/api/form/config/FormConfigTest.java +++ b/backend/src/test/java/com/bakdata/conquery/api/form/config/FormConfigTest.java @@ -86,6 +86,7 @@ public class FormConfigTest { private DatasetId datasetId; private DatasetId datasetId1; private ExportForm form; + private User user; @BeforeAll public void setupTestClass() throws Exception{ @@ -138,6 +139,10 @@ public void setupTest(){ form.setQueryGroupId(managedQuery.getId()); mode.setForm(form); mode.setFeatures(List.of(new CQConcept())); + + + user = new User("test","test", storage::updateUser); + storage.addUser(user); } @AfterEach @@ -147,8 +152,6 @@ public void cleanupTest() { @Test public void addConfigWithoutTranslation() { - User user = new User("test","test", storage); - storage.addUser(user); user.addPermission(dataset.createPermission(Ability.READ.asSet())); ObjectMapper mapper = FormConfigProcessor.getMAPPER(); @@ -165,8 +168,6 @@ public void addConfigWithoutTranslation() { @Test public void deleteConfig() { // PREPARE - User user = new User("test","test", storage); - storage.addUser(user); user.addPermission(DatasetPermission.onInstance(Ability.READ, datasetId)); ObjectMapper mapper = FormConfigProcessor.getMAPPER(); @@ -188,8 +189,6 @@ public void deleteConfig() { @Test public void getConfig() { // PREPARE - User user = new User("test","test", storage); - storage.addUser(user); user.addPermission(dataset.createPermission(Ability.READ.asSet())); ObjectMapper mapper = FormConfigProcessor.getMAPPER(); @@ -262,9 +261,6 @@ public void visit(Consumer visitor) { @Test public void getConfigs() { // PREPARE - - User user = new User("test","test", storage); - storage.addUser(user); user.addPermission(DatasetPermission.onInstance(Ability.READ, datasetId)); user.addPermission(FormPermission.onInstance(Ability.CREATE, form.getFormType())); @@ -330,12 +326,10 @@ public void getConfigs() { @Test public void patchConfig() { // PREPARE - User user = new User("test","test", storage); - storage.addUser(user); user.addPermission(DatasetPermission.onInstance(Ability.READ, datasetId)); - Group group1 = new Group("test1","test1", storage); + Group group1 = new Group("test1","test1", storage::updateGroup); storage.addGroup(group1); - Group group2 = new Group("test2","test2", storage); + Group group2 = new Group("test2","test2", storage::updateGroup); storage.addGroup(group2); group1.addMember(user); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/DownloadLinkGeneration.java b/backend/src/test/java/com/bakdata/conquery/integration/DownloadLinkGeneration.java index ded19ed71b..391e9aa0b7 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/DownloadLinkGeneration.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/DownloadLinkGeneration.java @@ -28,7 +28,7 @@ public class DownloadLinkGeneration extends IntegrationTest.Simple implements Pr public void execute(StandaloneSupport conquery) throws Exception { final MetaStorage storage = conquery.getMetaStorage(); - final User user = new User("testU", "testU", storage); + final User user = new User("testU", "testU", storage::updateUser); final String testJson = In.resource("/tests/query/SIMPLE_TREECONCEPT_QUERY/SIMPLE_TREECONCEPT_Query.test.json").withUTF8() .readAll(); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/ConceptPermissionTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/ConceptPermissionTest.java index 6698adca50..13c8e3968b 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/ConceptPermissionTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/ConceptPermissionTest.java @@ -33,7 +33,7 @@ public void execute(StandaloneSupport conquery) throws Exception { final String testJson = In.resource("/tests/query/SIMPLE_TREECONCEPT_QUERY/SIMPLE_TREECONCEPT_Query.test.json").withUTF8().readAll(); final QueryTest test = (QueryTest) JsonIntegrationTest.readJson(dataset.getId(), testJson); final QueryProcessor processor = new QueryProcessor(storage.getDatasetRegistry(), storage, conquery.getConfig()); - final User user = new User("testUser", "testUserLabel", storage); + final User user = new User("testUser", "testUserLabel", storage::updateUser); // Manually import data, so we can do our own work. { diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/GroupHandlingTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/GroupHandlingTest.java index 8c156caa29..e0cdf68b05 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/GroupHandlingTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/GroupHandlingTest.java @@ -21,10 +21,10 @@ public class GroupHandlingTest extends IntegrationTest.Simple implements Program public void execute(StandaloneSupport conquery) throws Exception { MetaStorage storage = conquery.getMetaStorage(); - Group group1 = new Group("company", "company", storage); - User user1 = new User("user", "user", storage); - User user1copy = new User("user", "user", storage); - User user2 = new User("user2", "user2", storage); + Group group1 = new Group("company", "company", storage::updateGroup); + User user1 = new User("user", "user", storage::updateUser); + User user1copy = new User("user", "user", storage::updateUser); + User user2 = new User("user2", "user2", storage::updateUser); try { storage.addGroup(group1); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/PermissionGroupHandlingTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/PermissionGroupHandlingTest.java index 8edf25ea9e..99f7190f1c 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/PermissionGroupHandlingTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/PermissionGroupHandlingTest.java @@ -30,9 +30,9 @@ public void execute(StandaloneSupport conquery) throws Exception { ManagedExecutionId query1 = new ManagedExecutionId(dataset1.getId(), UUID.randomUUID()); - Role role1 = new Role("role", "role", storage); + Role role1 = new Role("role", "role", storage::updateRole); TestUser user1 = new TestUser(storage); - Group group1 = new Group("company", "company", storage); + Group group1 = new Group("company", "company", storage::updateGroup); try { diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/PermissionRoleHandlingTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/PermissionRoleHandlingTest.java index 105abea3f7..448cf6ae7b 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/PermissionRoleHandlingTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/PermissionRoleHandlingTest.java @@ -20,7 +20,7 @@ public class PermissionRoleHandlingTest extends IntegrationTest.Simple implement @Override public void execute(StandaloneSupport conquery) throws Exception { MetaStorage storage = conquery.getMetaStorage(); - Role mandator1 = new Role("company", "company", storage); + Role mandator1 = new Role("company", "company", storage::updateRole); TestUser user1 = new TestUser(storage); Dataset dataset = conquery.getDataset(); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/RestartTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/RestartTest.java index 878f1ba86b..62fa7a5d10 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/RestartTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/RestartTest.java @@ -78,12 +78,12 @@ public void execute(String name, TestConquery testConquery) throws Exception { MetaStorage storage = conquery.getMetaStorage(); - Role role = new Role("role", "ROLE", storage); - Role roleToDelete = new Role("roleDelete", "ROLE_DELETE", storage); - User user = new User("user@test.email", "USER", storage); - User userToDelete = new User("userDelete@test.email", "USER_DELETE", storage); - Group group = new Group("group", "GROUP", storage); - Group groupToDelete = new Group("groupDelete", "GROUP_DELETE", storage); + Role role = new Role("role", "ROLE", storage::updateRole); + Role roleToDelete = new Role("roleDelete", "ROLE_DELETE", storage::updateRole); + User user = new User("user@test.email", "USER", storage::updateUser); + User userToDelete = new User("userDelete@test.email", "USER_DELETE", storage::updateUser); + Group group = new Group("group", "GROUP", storage::updateGroup); + Group groupToDelete = new Group("groupDelete", "GROUP_DELETE", storage::updateGroup); {// Auth testing (deletion and permission grant) // build constellation diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/ReusedQueryTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/ReusedQueryTest.java index a8efca5647..c915275fd5 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/ReusedQueryTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/ReusedQueryTest.java @@ -219,7 +219,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { reused.setSecondaryId(query.getSecondaryId()); - User shareHolder = new User("shareholder", "ShareHolder", conquery.getMetaStorage()); + User shareHolder = new User("shareholder", "ShareHolder", conquery.getMetaStorage()::updateUser); conquery.getMetaProcessor().addUser(shareHolder); shareHolder.addPermissions(Set.of( diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/RoleHandlingOnGroupTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/RoleHandlingOnGroupTest.java index 205e39d5de..0cd6892665 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/RoleHandlingOnGroupTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/RoleHandlingOnGroupTest.java @@ -27,8 +27,8 @@ public void execute(StandaloneSupport conquery) throws Exception { dataset1.setLabel("dataset1"); MetaStorage storage = conquery.getMetaStorage(); - Group group1 = new Group("company", "company", storage); - Role role = new Role("role1", "role1", storage); + Group group1 = new Group("company", "company", storage::updateGroup); + Role role = new Role("role1", "role1", storage::updateRole); TestUser user1 = new TestUser(storage); try { diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/RoleHandlingTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/RoleHandlingTest.java index ba797e04be..6c3f0ad01c 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/RoleHandlingTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/RoleHandlingTest.java @@ -24,10 +24,10 @@ public void execute(StandaloneSupport conquery) throws Exception { dataset1.setLabel("dataset1"); MetaStorage storage = conquery.getMetaStorage(); - Role mandator1 = new Role("company", "company", storage); - Role mandator1Copy = new Role("company", "company", storage); - Role mandator2 = new Role("company2", "company2", storage); - User user1 = new User("user", "user", storage); + Role mandator1 = new Role("company", "company", storage::updateRole); + Role mandator1Copy = new Role("company", "company", storage::updateRole); + Role mandator2 = new Role("company2", "company2", storage::updateRole); + User user1 = new User("user", "user", storage::updateUser); try { storage.addRole(mandator1); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/RoleUITest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/RoleUITest.java index 7145165d96..d344fb4d59 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/RoleUITest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/RoleUITest.java @@ -34,9 +34,9 @@ public class RoleUITest extends IntegrationTest.Simple implements ProgrammaticIn @Override public void execute(StandaloneSupport conquery) throws Exception { MetaStorage storage = conquery.getMetaStorage(); - Role mandator = new Role("testMandatorName", "testMandatorLabel", storage); + Role mandator = new Role("testMandatorName", "testMandatorLabel", storage::updateRole); RoleId mandatorId = mandator.getId(); - User user = new User("testUser@test.de", "testUserName", storage); + User user = new User("testUser@test.de", "testUserName", storage::updateUser); UserId userId = user.getId(); try { diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/SuperPermissionTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/SuperPermissionTest.java index d83c1bd9e5..a3aee8c05b 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/SuperPermissionTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/SuperPermissionTest.java @@ -22,7 +22,7 @@ public void execute(StandaloneSupport conquery) throws Exception { dataset1.setLabel("dataset1"); MetaStorage storage = conquery.getMetaStorage(); - Role role1 = new Role("company", "company", storage); + Role role1 = new Role("company", "company", storage::updateRole); TestUser user1 = new TestUser(storage); storage.addRole(role1); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/TestUser.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/TestUser.java index f07ccbca0b..b4b849b974 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/TestUser.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/TestUser.java @@ -10,7 +10,7 @@ public class TestUser extends User { public TestUser(MetaStorage storage) { - super("user", "user", storage); + super("user", "user", storage::updateUser); } public boolean isPermitted(Permission permission) { diff --git a/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/IdRefrenceTest.java b/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/IdRefrenceTest.java index f8c5a44d38..f47178460f 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/IdRefrenceTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/IdRefrenceTest.java @@ -33,7 +33,7 @@ public void testListReferences() throws IOException { registry.register(table); final CentralRegistry metaRegistry = new CentralRegistry(); - User user = new User("usermail", "userlabel", mock(MetaStorage.class)); + User user = new User("usermail", "userlabel", mock(MetaStorage.class)::updateUser); metaRegistry.register(user); String json = Jackson.MAPPER.writeValueAsString( diff --git a/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStoreDumpTest.java b/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStoreDumpTest.java index 54a700161b..f42e9f1668 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStoreDumpTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStoreDumpTest.java @@ -40,7 +40,6 @@ public class SerializingStoreDumpTest { public static final IStoreInfo USER_STORE_ID = StoreInfo.AUTH_USER.storeInfo(); - private final static MetaStorage STORAGE = new MetaStorage(Validators.newValidator(), new NonPersistentStoreFactory(), new DatasetRegistry(2)); private File tmpDir; private Environment env; private XodusStoreFactory config; @@ -49,7 +48,6 @@ public class SerializingStoreDumpTest { private final ManagedQuery managedQuery = new ManagedQuery(null, null, new Dataset("dataset")); private final ConceptQuery cQuery = new ConceptQuery( new CQReusedQuery(managedQuery.getId())); - private final User user = new User("username", "userlabel", STORAGE); @BeforeEach public void init() { @@ -79,6 +77,8 @@ public void testCorruptValueDump() throws JSONException, IOException { { // Open a store and insert a valid key-value pair (UserId & User) SerializingStore store = createSerializedStore(config, env, Validators.newValidator(), USER_STORE_ID); + + final User user = new User("username", "userlabel", u -> store.add(u.getId(),u)); store.add(user.getId(), user); } @@ -132,6 +132,8 @@ public void testCorruptKeyDump() throws JSONException, IOException { { // Open a store and insert a valid key-value pair (UserId & User) SerializingStore store = createSerializedStore(config, env, Validators.newValidator(), USER_STORE_ID); + + final User user = new User("username", "userlabel", u -> store.add(u.getId(),u)); store.add(new UserId("testU1"), user); } @@ -183,6 +185,7 @@ public void testCorruptionRemoval() throws JSONException, IOException { { // Open a store and insert a valid key-value pair (UserId & User) SerializingStore store = createSerializedStore(config, env, Validators.newValidator(), USER_STORE_ID); + final User user = new User("username", "userlabel", u -> store.add(u.getId(),u)); store.add(new UserId("testU1"), user); } diff --git a/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java b/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java index 1af1fca99c..b3b7f01328 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java +++ b/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java @@ -85,7 +85,7 @@ public void passwordCredential() throws IOException, JSONException { @Test public void role() throws IOException, JSONException { - Role mandator = new Role("company", "company", STORAGE); + Role mandator = new Role("company", "company", STORAGE::updateRole); SerializationTestUtil .forType(Role.class) @@ -99,13 +99,13 @@ public void role() throws IOException, JSONException { @Test public void user() throws IOException, JSONException { MetaStorage storage = new MetaStorage(null, new NonPersistentStoreFactory(), null); - User user = new User("user", "user", STORAGE); + User user = new User("user", "user", STORAGE::updateUser); user.addPermission(DatasetPermission.onInstance(Ability.READ, new DatasetId("test"))); user .addPermission( ExecutionPermission.onInstance(Ability.READ, new ManagedExecutionId(new DatasetId("dataset"), UUID.randomUUID())) ); - Role role = new Role("company", "company", STORAGE); + Role role = new Role("company", "company", STORAGE::updateRole); user.addRole(role); CentralRegistry registry = new CentralRegistry(); @@ -121,17 +121,17 @@ public void user() throws IOException, JSONException { @Test public void group() throws IOException, JSONException { MetaStorage storage = new MetaStorage(null, new NonPersistentStoreFactory(), null); - Group group = new Group("group", "group", STORAGE); + Group group = new Group("group", "group", STORAGE::updateGroup); group.addPermission(DatasetPermission.onInstance(Ability.READ, new DatasetId("test"))); group .addPermission( ExecutionPermission.onInstance(Ability.READ, new ManagedExecutionId(new DatasetId("dataset"), UUID.randomUUID())) ); - group.addRole(new Role("company", "company", STORAGE)); + group.addRole(new Role("company", "company", STORAGE::updateRole)); - Role role = new Role("company", "company", STORAGE); + Role role = new Role("company", "company", STORAGE::updateRole); group.addRole(role); - User user = new User("userName", "userLabel", STORAGE); + User user = new User("userName", "userLabel", STORAGE::updateUser); group.addMember(user); CentralRegistry registry = new CentralRegistry(); @@ -247,7 +247,7 @@ public void managedQuery() throws JSONException, IOException { final Dataset dataset = new Dataset("test-dataset"); - final User user = new User("test-user", "test-user", STORAGE); + final User user = new User("test-user", "test-user", STORAGE::updateUser); registry.register(dataset); registry.register(user); @@ -327,7 +327,7 @@ public void executionQueryJobError() throws JSONException, IOException { @Test public void meInformation() throws IOException, JSONException { - User user = new User("name", "labe", STORAGE); + User user = new User("name", "labe", STORAGE::updateUser); MeProcessor.FEMeInformation info = MeProcessor.FEMeInformation.builder() .userName(user.getLabel()) diff --git a/backend/src/test/java/com/bakdata/conquery/models/auth/CopyUserTest.java b/backend/src/test/java/com/bakdata/conquery/models/auth/CopyUserTest.java index ca96c6de74..c19873566b 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/auth/CopyUserTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/auth/CopyUserTest.java @@ -25,17 +25,17 @@ void testUserCopy(){ registry.setMetaStorage(storage); // Create test role - Role role = new Role("role", "role", storage); + Role role = new Role("role", "role", storage::updateRole); storage.addRole(role); role.addPermission(DatasetPermission.onInstance(Ability.READ, new DatasetId("dataset0"))); // Create test group - Group group = new Group("group", "group", storage); + Group group = new Group("group", "group", storage::updateGroup); storage.addGroup(group); group.addPermission(DatasetPermission.onInstance(Ability.READ, new DatasetId("dataset1"))); // Create original user with role and group mapping - User originUser = new User("user", "user", storage); + User originUser = new User("user", "user", storage::updateUser); storage.addUser(originUser); originUser.addRole(role); group.addMember(originUser); diff --git a/backend/src/test/java/com/bakdata/conquery/models/auth/IntrospectionDelegatingRealmTest.java b/backend/src/test/java/com/bakdata/conquery/models/auth/IntrospectionDelegatingRealmTest.java index f17a44bca6..d90d7bd12c 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/auth/IntrospectionDelegatingRealmTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/auth/IntrospectionDelegatingRealmTest.java @@ -60,7 +60,7 @@ public class IntrospectionDelegatingRealmTest { private static final BearerToken USER_3_TOKEN_WRAPPED = new BearerToken(USER_3_TOKEN); // Groups private static final String GROUPNAME_1 = "group1"; - private static final Group GROUP_1_EXISTING = new Group(GROUPNAME_1, GROUPNAME_1, STORAGE); + private static final Group GROUP_1_EXISTING = new Group(GROUPNAME_1, GROUPNAME_1, STORAGE::updateGroup); private static final String GROUPNAME_2 = "group2"; // Group is created during test private static OIDCMockServer OIDC_SERVER; @@ -157,12 +157,12 @@ public void tokenIntrospectionSimpleUserNew() { .usingRecursiveComparison() .ignoringFields(ConqueryAuthenticationInfo.Fields.credentials) .isEqualTo(new ConqueryAuthenticationInfo(new UserId(USER_1_NAME), USER1_TOKEN_WRAPPED, REALM, true)); - assertThat(STORAGE.getAllUsers()).containsOnly(new User(USER_1_NAME, USER_1_NAME, STORAGE)); + assertThat(STORAGE.getAllUsers()).containsOnly(new User(USER_1_NAME, USER_1_NAME, STORAGE::updateUser)); } @Test public void tokenIntrospectionSimpleUserExisting() { - User existingUser = new User(USER_1_NAME, USER_1_NAME, STORAGE); + User existingUser = new User(USER_1_NAME, USER_1_NAME, STORAGE::updateUser); STORAGE.addUser(existingUser); AuthenticationInfo info = REALM.doGetAuthenticationInfo(USER1_TOKEN_WRAPPED); @@ -182,7 +182,7 @@ public void tokenIntrospectionGroupedUser() { .usingRecursiveComparison() .ignoringFields(ConqueryAuthenticationInfo.Fields.credentials) .isEqualTo(new ConqueryAuthenticationInfo(new UserId(USER_2_NAME), USER_2_TOKEN_WRAPPED, REALM, true)); - assertThat(STORAGE.getAllUsers()).containsOnly(new User(USER_2_NAME, USER_2_LABEL, STORAGE)); + assertThat(STORAGE.getAllUsers()).containsOnly(new User(USER_2_NAME, USER_2_LABEL, STORAGE::updateUser)); assertThat(STORAGE.getAllGroups()).hasSize(2); // Pre-existing group and a second group that has been added in the process assertThat(STORAGE.getGroup(new GroupId(GROUPNAME_1)).getMembers()).contains(new UserId(USER_2_NAME)); assertThat(STORAGE.getGroup(new GroupId(GROUPNAME_2)).getMembers()).contains(new UserId(USER_2_NAME)); @@ -190,7 +190,7 @@ public void tokenIntrospectionGroupedUser() { @Test public void tokenIntrospectionGroupedUserRemoveGroupMapping() { - User user = new User(USER_3_NAME, USER_3_LABEL, STORAGE); + User user = new User(USER_3_NAME, USER_3_LABEL, STORAGE::updateUser); STORAGE.addUser(user); GROUP_1_EXISTING.addMember(user); @@ -202,7 +202,7 @@ public void tokenIntrospectionGroupedUserRemoveGroupMapping() { .usingRecursiveComparison() .ignoringFields(ConqueryAuthenticationInfo.Fields.credentials) .isEqualTo(new ConqueryAuthenticationInfo(new UserId(USER_3_NAME), USER_3_TOKEN_WRAPPED, REALM, true)); - assertThat(STORAGE.getAllUsers()).containsOnly(new User(USER_3_NAME, USER_3_LABEL, STORAGE)); + assertThat(STORAGE.getAllUsers()).containsOnly(new User(USER_3_NAME, USER_3_LABEL, STORAGE::updateUser)); assertThat(STORAGE.getAllGroups()).hasSize(1); // Pre-existing group assertThat(STORAGE.getGroup(new GroupId(GROUPNAME_1)).getMembers()).doesNotContain(new UserId(USER_3_NAME)); } diff --git a/backend/src/test/java/com/bakdata/conquery/models/auth/LocalAuthRealmTest.java b/backend/src/test/java/com/bakdata/conquery/models/auth/LocalAuthRealmTest.java index 670b0eff8e..fcbd15113d 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/auth/LocalAuthRealmTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/auth/LocalAuthRealmTest.java @@ -59,7 +59,7 @@ public void setupAll() throws Exception { @BeforeEach public void setupEach() { // Create User in Realm - user1 = new User("TestUser", "Test User", storage); + user1 = new User("TestUser", "Test User", storage::updateUser); PasswordCredential user1Password = new PasswordCredential("testPassword".toCharArray()); storage.addUser(user1); realm.addUser(user1, List.of(user1Password)); diff --git a/backend/src/test/java/com/bakdata/conquery/models/execution/DefaultLabelTest.java b/backend/src/test/java/com/bakdata/conquery/models/execution/DefaultLabelTest.java index 906b75bf2c..2902cbd9f7 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/execution/DefaultLabelTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/execution/DefaultLabelTest.java @@ -39,7 +39,7 @@ public class DefaultLabelTest { private static final DatasetRegistry DATASET_REGISTRY = Mockito.mock(DatasetRegistry.class); private static final Dataset DATASET = new Dataset("dataset"); - private static final User user = new User("user","user", STORAGE); + private static final User user = new User("user","user", STORAGE::updateUser); private static final TreeConcept CONCEPT = new TreeConcept() { { diff --git a/backend/src/test/java/com/bakdata/conquery/tasks/PermissionCleanupTaskTest.java b/backend/src/test/java/com/bakdata/conquery/tasks/PermissionCleanupTaskTest.java index debbc14e33..d828d468ce 100644 --- a/backend/src/test/java/com/bakdata/conquery/tasks/PermissionCleanupTaskTest.java +++ b/backend/src/test/java/com/bakdata/conquery/tasks/PermissionCleanupTaskTest.java @@ -56,7 +56,7 @@ void doNotDeletePermissionValidReference() { final ManagedQuery managedQuery = createManagedQuery(); // Saving the Execution - User user = new User("test", "test", storage); + User user = new User("test", "test", storage::updateUser); storage.updateUser(user); user.addPermission(ExecutionPermission.onInstance(AbilitySets.QUERY_CREATOR, managedQuery.getId())); @@ -73,7 +73,7 @@ void doDeletePermissionInvalidReference() { final ManagedQuery managedQuery = createManagedQuery(); // Removing the execution storage.removeExecution(managedQuery.getId()); - User user = new User("test", "test", storage); + User user = new User("test", "test", storage::updateUser); storage.updateUser(user); user.addPermission(ExecutionPermission.onInstance(AbilitySets.QUERY_CREATOR, managedQuery.getId())); @@ -91,7 +91,7 @@ void doDeletePartialPermissionWithInvalidReference() { final ManagedQuery managedQuery2 = createManagedQuery(); // Removing the second execution storage.removeExecution(managedQuery2.getId()); - User user = new User("test", "test", storage); + User user = new User("test", "test", storage::updateUser); storage.updateUser(user); user.addPermission( // Build a permission with multiple instances @@ -114,8 +114,8 @@ void doDeletePermissionsOfOwnedReference() { // Created owned execution final ManagedQuery managedQueryOwned = createManagedQuery(); // Setup user - User user = new User("test", "test", storage); - User user2 = new User("test2", "test2", storage); + User user = new User("test", "test", storage::updateUser); + User user2 = new User("test2", "test2", storage::updateUser); storage.updateUser(user); user.addPermission(ExecutionPermission.onInstance(AbilitySets.QUERY_CREATOR, managedQueryOwned.getId())); From c4780682db4f1946f68655320ecd8a34d85d8496 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 21 Sep 2021 14:14:55 +0200 Subject: [PATCH 03/35] fix serialization --- .../java/com/bakdata/conquery/io/storage/MetaStorage.java | 7 ++++++- .../com/bakdata/conquery/models/SerializationTests.java | 6 +++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java index 7eef215a98..8f6dc55766 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java @@ -15,6 +15,7 @@ import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.forms.configs.FormConfig; import com.bakdata.conquery.models.identifiable.CentralRegistry; +import com.bakdata.conquery.models.identifiable.Identifiable; import com.bakdata.conquery.models.identifiable.ids.specific.FormConfigId; import com.bakdata.conquery.models.identifiable.ids.specific.GroupId; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; @@ -203,5 +204,9 @@ public MutableInjectableValues inject(MutableInjectableValues values) { return values.add(MetaStorage.class, this); } - public interface StorageUpdater extends Consumer, Injectable{}; + /** + * An implementation of this interface should be provided by mutable stored entities, so that + * these entities can update their persistent state themselves on change in a thread safe manner. + */ + public interface StorageUpdater> extends Consumer, Injectable{}; } diff --git a/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java b/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java index b3b7f01328..1bd73b2be0 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java +++ b/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java @@ -89,7 +89,7 @@ public void role() throws IOException, JSONException { SerializationTestUtil .forType(Role.class) - .injectables(STORAGE) + .injectables(new Role.StorageUpdater(STORAGE::updateRole)) .test(mandator); } @@ -114,7 +114,7 @@ public void user() throws IOException, JSONException { SerializationTestUtil .forType(User.class) .registry(registry) - .injectables(STORAGE) + .injectables(new User.StorageUpdater(STORAGE::updateUser)) .test(user); } @@ -140,7 +140,7 @@ public void group() throws IOException, JSONException { SerializationTestUtil .forType(Group.class) - .injectables(STORAGE) + .injectables(new Group.StorageUpdater(STORAGE::updateGroup)) .registry(registry) .test(group); } From ec343393e93e1f7b8704e2380a78a7c82017d8d4 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 21 Sep 2021 12:19:00 +0000 Subject: [PATCH 04/35] Update AutoDoc --- docs/Config JSON.md | 18 +++++++++--------- docs/REST API JSONs.md | 8 ++++---- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/Config JSON.md b/docs/Config JSON.md index 28b363ed9a..124bf1001d 100644 --- a/docs/Config JSON.md +++ b/docs/Config JSON.md @@ -406,7 +406,7 @@ Supported Fields: | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusConfig.java#L51) | treeMaxPageSize | `int` or `null` | `null` | | |

-### Type XodusStoreFactory [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L84) +### Type XodusStoreFactory [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L76)
Details

@@ -417,12 +417,12 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L94) | directory | `Path` | `"file://./storage"` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L105) | nThreads | `@javax.validation.constraints.Min(1) int` | ␀ | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L108-L110) | removeUnreadableFromStore | `boolean` | `false` | | Flag for the {@link SerializingStore} whether to delete values from the underlying store, that cannot be mapped to an object anymore. | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L113-L115) | unreadableDataDumpDirectory | `Optional` | `null` | | When set, all values that could not be deserialized from the persistent store, are dump into individual files. | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L101) | useWeakDictionaryCaching | `boolean` | `true` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L96) | validateOnWrite | `boolean` | `false` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L102) | weakCacheDuration | `@NotNull Duration` | `"48 hours"` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L97) | xodus | [XodusConfig](#Type-XodusConfig) | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L86) | directory | `Path` | `"file://./storage"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L97) | nThreads | `@javax.validation.constraints.Min(1) int` | ␀ | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L100-L102) | removeUnreadableFromStore | `boolean` | `false` | | Flag for the {@link SerializingStore} whether to delete values from the underlying store, that cannot be mapped to an object anymore. | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L105-L107) | unreadableDataDumpDirectory | `File` | `null` | | When set, all values that could not be deserialized from the persistent store, are dump into individual files. | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L93) | useWeakDictionaryCaching | `boolean` | `true` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L88) | validateOnWrite | `boolean` | `false` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L94) | weakCacheDuration | `@NotNull Duration` | `"48 hours"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L89) | xodus | [XodusConfig](#Type-XodusConfig) | | | |

diff --git a/docs/REST API JSONs.md b/docs/REST API JSONs.md index 7ea938a80c..df693f03d3 100644 --- a/docs/REST API JSONs.md +++ b/docs/REST API JSONs.md @@ -952,7 +952,7 @@ No fields can be set for this type.

-### Type MetaDataPatch [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/MetaDataPatch.java#L30) +### Type MetaDataPatch [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/MetaDataPatch.java#L29)
Details

@@ -963,9 +963,9 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/MetaDataPatch.java#L38) | groups | list of ID of `Group` | `null` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/MetaDataPatch.java#L37) | label | `String` | `null` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/MetaDataPatch.java#L36) | tags | list of `String` | `null` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/MetaDataPatch.java#L37) | groups | list of ID of `Group` | `null` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/MetaDataPatch.java#L36) | label | `String` | `null` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/MetaDataPatch.java#L35) | tags | list of `String` | `null` | | |

### Type OverviewExecutionStatus [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/OverviewExecutionStatus.java#L5-L7) From 45f7dc39bfdbad1d8809f1c8b52f80d207ca4d0e Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 21 Sep 2021 14:21:14 +0200 Subject: [PATCH 05/35] pull up store updater --- .../bakdata/conquery/io/storage/MetaStorage.java | 14 ++++---------- .../conquery/models/config/StoreFactory.java | 6 +++--- .../conquery/models/config/XodusStoreFactory.java | 12 ++++++------ .../conquery/util/NonPersistentStoreFactory.java | 6 +++--- 4 files changed, 16 insertions(+), 22 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java index 8f6dc55766..786712eacc 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java @@ -7,7 +7,6 @@ import javax.validation.Validator; import com.bakdata.conquery.io.jackson.Injectable; -import com.bakdata.conquery.io.jackson.MutableInjectableValues; import com.bakdata.conquery.models.auth.entities.Group; import com.bakdata.conquery.models.auth.entities.Role; import com.bakdata.conquery.models.auth.entities.User; @@ -27,7 +26,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -public class MetaStorage implements ConqueryStorage, Injectable { +public class MetaStorage implements ConqueryStorage { private IdentifiableStore> executions; @@ -48,9 +47,9 @@ public MetaStorage(Validator validator, StoreFactory storageFactory, DatasetRegi this.validator = validator; - authUser = storageFactory.createUserStore(centralRegistry, "meta", this); - authRole = storageFactory.createRoleStore(centralRegistry, "meta", this); - authGroup = storageFactory.createGroupStore(centralRegistry, "meta", this); + authUser = storageFactory.createUserStore(centralRegistry, "meta", new User.StorageUpdater(this::updateUser)); + authRole = storageFactory.createRoleStore(centralRegistry, "meta", new Role.StorageUpdater(this::updateRole)); + authGroup = storageFactory.createGroupStore(centralRegistry, "meta", new Group.StorageUpdater(this::updateGroup)); // Executions depend on users executions = storageFactory.createExecutionsStore(centralRegistry, datasetRegistry, "meta"); formConfigs = storageFactory.createFormConfigStore(centralRegistry, datasetRegistry, "meta"); @@ -199,11 +198,6 @@ public void close() throws IOException { authGroup.close(); } - @Override - public MutableInjectableValues inject(MutableInjectableValues values) { - return values.add(MetaStorage.class, this); - } - /** * An implementation of this interface should be provided by mutable stored entities, so that * these entities can update their persistent state themselves on change in a thread safe manner. diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java index ebe79309b7..a6af0bb891 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java @@ -64,9 +64,9 @@ public interface StoreFactory { // MetaStorage IdentifiableStore> createExecutionsStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName); IdentifiableStore createFormConfigStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName); - IdentifiableStore createUserStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage); - IdentifiableStore createRoleStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage); - IdentifiableStore createGroupStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage); + IdentifiableStore createUserStore(CentralRegistry centralRegistry, String pathName, User.StorageUpdater storageUpdater); + IdentifiableStore createRoleStore(CentralRegistry centralRegistry, String pathName, Role.StorageUpdater storageUpdater); + IdentifiableStore createGroupStore(CentralRegistry centralRegistry, String pathName, Group.StorageUpdater storageUpdater); SingletonStore createPrimaryDictionaryStore(String pathName, SingletonNamespaceCollection namespaceCollection); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java index 53a9f4ab8a..c80f460689 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java @@ -315,19 +315,19 @@ public IdentifiableStore createFormConfigStore(CentralRegistry centr } @Override - public IdentifiableStore createUserStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage) { - return StoreInfo.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "users")), validator, AUTH_USER), centralRegistry, new User.StorageUpdater(storage::updateUser)); + public IdentifiableStore createUserStore(CentralRegistry centralRegistry, String pathName, User.StorageUpdater storageUpdater) { + return StoreInfo.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "users")), validator, AUTH_USER), centralRegistry, storageUpdater); } @Override - public IdentifiableStore createRoleStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage) { - return StoreInfo.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "roles")), validator, AUTH_ROLE), centralRegistry, new Role.StorageUpdater(storage::updateRole)); + public IdentifiableStore createRoleStore(CentralRegistry centralRegistry, String pathName, Role.StorageUpdater storageUpdater) { + return StoreInfo.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "roles")), validator, AUTH_ROLE), centralRegistry, storageUpdater); } @Override - public IdentifiableStore createGroupStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage) { - return StoreInfo.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "groups")), validator, AUTH_GROUP), centralRegistry, new Group.StorageUpdater(storage::updateGroup)); + public IdentifiableStore createGroupStore(CentralRegistry centralRegistry, String pathName, Group.StorageUpdater storageUpdater) { + return StoreInfo.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "groups")), validator, AUTH_GROUP), centralRegistry, storageUpdater); } @Override diff --git a/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java b/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java index d5aff76281..147fcde138 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java +++ b/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java @@ -117,17 +117,17 @@ public IdentifiableStore createFormConfigStore(CentralRegistry centr } @Override - public IdentifiableStore createUserStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage) { + public IdentifiableStore createUserStore(CentralRegistry centralRegistry, String pathName, User.StorageUpdater storageUpdater) { return AUTH_USER.identifiable(new NonPersistentStore(), centralRegistry); } @Override - public IdentifiableStore createRoleStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage) { + public IdentifiableStore createRoleStore(CentralRegistry centralRegistry, String pathName, Role.StorageUpdater storageUpdater) { return AUTH_ROLE.identifiable(new NonPersistentStore(), centralRegistry); } @Override - public IdentifiableStore createGroupStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage) { + public IdentifiableStore createGroupStore(CentralRegistry centralRegistry, String pathName, Group.StorageUpdater storageUpdater) { return AUTH_GROUP.identifiable(new NonPersistentStore(), centralRegistry); } From 18f27525da037d3ff0ba320200aac4a06192d06a Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 21 Sep 2021 18:26:01 +0200 Subject: [PATCH 06/35] review changes --- .../main/java/com/bakdata/conquery/apiv1/QueryProcessor.java | 2 +- .../forms/frontendconfiguration/FormConfigProcessor.java | 5 ++--- .../com/bakdata/conquery/api/form/config/FormConfigTest.java | 2 +- .../com/bakdata/conquery/integration/common/LoadingUtil.java | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java index 2e73e0f955..c2ed5802a9 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java @@ -288,7 +288,7 @@ private void translateToOtherDatasets(Dataset dataset, QueryDescription query, U Query translated = QueryTranslator.replaceDataset(datasetRegistry, translateable, targetDataset); targetNamespace.getExecutionManager() - .createQuery(datasetRegistry, translated, mq.getQueryId(), storage.getUser(user.getId()), targetDataset); + .createQuery(datasetRegistry, translated, mq.getQueryId(), user, targetDataset); } catch (Exception e) { log.trace("Could not translate Query[{}] to Dataset[{}]", mq.getId(), targetDataset.getId(), e); diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormConfigProcessor.java b/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormConfigProcessor.java index 7db632de33..c127f6f0d3 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormConfigProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormConfigProcessor.java @@ -127,7 +127,7 @@ public FormConfig addConfig(User user, Dataset targetDataset, FormConfigAPI conf * @return */ public FormConfig addConfigAndTranslations(User user, Dataset targetDataset, FormConfigAPI config) { - FormConfig internalConfig = FormConfigAPI.intern(config, storage.getUser(user.getId()), targetDataset); + FormConfig internalConfig = FormConfigAPI.intern(config, user, targetDataset); // Add the config immediately to the submitted dataset addConfigToDataset(internalConfig); @@ -160,8 +160,7 @@ public FormConfigFullRepresentation patchConfig(User user, FormConfig config, Fo /** * Deletes a configuration from the storage and all permissions, that have this configuration as target. */ - public void deleteConfig(User User, FormConfig config) { - User user = storage.getUser(User.getId()); + public void deleteConfig(User user, FormConfig config) { user.authorize( config, Ability.DELETE); storage.removeFormConfig(config.getId()); // Delete corresponding permissions (Maybe better to put it into a slow job) diff --git a/backend/src/test/java/com/bakdata/conquery/api/form/config/FormConfigTest.java b/backend/src/test/java/com/bakdata/conquery/api/form/config/FormConfigTest.java index 7df060731f..983807076d 100644 --- a/backend/src/test/java/com/bakdata/conquery/api/form/config/FormConfigTest.java +++ b/backend/src/test/java/com/bakdata/conquery/api/form/config/FormConfigTest.java @@ -261,7 +261,7 @@ public void visit(Consumer visitor) { @Test public void getConfigs() { // PREPARE - user.addPermission(DatasetPermission.onInstance(Ability.READ, datasetId)); + user.addPermission(dataset.createPermission(Ability.READ.asSet())); user.addPermission(FormPermission.onInstance(Ability.CREATE, form.getFormType())); ExportForm form2 = new ExportForm(); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java b/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java index cd6f2fb710..c5a90d60aa 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java @@ -71,7 +71,7 @@ public static void importPreviousQueries(StandaloneSupport support, RequiredData ManagedExecution managed = support.getNamespace().getExecutionManager() .createQuery(support.getNamespace().getNamespaces(),q, queryId, user, support.getNamespace().getDataset()); - user.addPermission(ExecutionPermission.onInstance(AbilitySets.QUERY_CREATOR, managed.getId())); + user.addPermission(managed.createPermission(AbilitySets.QUERY_CREATOR)); if (managed.getState() == ExecutionState.FAILED) { fail("Query failed"); From 4c76604ba510704b701e62ba7af81f74da2ce16f Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 22 Sep 2021 13:40:29 +0200 Subject: [PATCH 07/35] Revert "use an injectable storageUpdater instead of the storage it self" .java --- .../conquery/apiv1/auth/ProtoUser.java | 2 +- .../conquery/io/storage/MetaStorage.java | 20 ++++---- .../models/auth/AuthorizationController.java | 2 +- .../conquery/models/auth/entities/Group.java | 49 +++---------------- .../models/auth/entities/PermissionOwner.java | 23 ++++++--- .../conquery/models/auth/entities/Role.java | 49 +++---------------- .../conquery/models/auth/entities/User.java | 42 +++------------- .../oidc/IntrospectionDelegatingRealm.java | 4 +- .../conquery/models/config/StoreFactory.java | 6 +-- .../models/config/XodusStoreFactory.java | 12 ++--- .../api/StoredQueriesProcessorTest.java | 2 +- .../api/form/config/FormConfigTest.java | 6 +-- .../integration/DownloadLinkGeneration.java | 2 +- .../tests/ConceptPermissionTest.java | 2 +- .../integration/tests/GroupHandlingTest.java | 8 +-- .../tests/PermissionGroupHandlingTest.java | 4 +- .../tests/PermissionRoleHandlingTest.java | 2 +- .../integration/tests/RestartTest.java | 12 ++--- .../integration/tests/ReusedQueryTest.java | 2 +- .../tests/RoleHandlingOnGroupTest.java | 4 +- .../integration/tests/RoleHandlingTest.java | 8 +-- .../integration/tests/RoleUITest.java | 4 +- .../tests/SuperPermissionTest.java | 2 +- .../conquery/integration/tests/TestUser.java | 2 +- .../io/jackson/serializer/IdRefrenceTest.java | 2 +- .../stores/SerializingStoreDumpTest.java | 7 +-- .../conquery/models/SerializationTests.java | 24 ++++----- .../conquery/models/auth/CopyUserTest.java | 6 +-- .../IntrospectionDelegatingRealmTest.java | 12 ++--- .../models/auth/LocalAuthRealmTest.java | 2 +- .../models/execution/DefaultLabelTest.java | 2 +- .../tasks/PermissionCleanupTaskTest.java | 10 ++-- .../util/NonPersistentStoreFactory.java | 6 +-- 33 files changed, 124 insertions(+), 216 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/auth/ProtoUser.java b/backend/src/main/java/com/bakdata/conquery/apiv1/auth/ProtoUser.java index 0effb88e0a..b8d343f65c 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/auth/ProtoUser.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/auth/ProtoUser.java @@ -56,7 +56,7 @@ public Optional getUser(@NonNull MetaStorage storage, boolean override) { if (label == null) { label = name; } - user = new User(name, label, storage::updateUser); + user = new User(name, label, storage); storage.updateUser(user); for (String sPermission : permissions) { user.addPermission(new WildcardPermission(sPermission)); diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java index 786712eacc..d423a33367 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java @@ -2,11 +2,11 @@ import java.io.IOException; import java.util.Collection; -import java.util.function.Consumer; import javax.validation.Validator; import com.bakdata.conquery.io.jackson.Injectable; +import com.bakdata.conquery.io.jackson.MutableInjectableValues; import com.bakdata.conquery.models.auth.entities.Group; import com.bakdata.conquery.models.auth.entities.Role; import com.bakdata.conquery.models.auth.entities.User; @@ -14,7 +14,6 @@ import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.forms.configs.FormConfig; import com.bakdata.conquery.models.identifiable.CentralRegistry; -import com.bakdata.conquery.models.identifiable.Identifiable; import com.bakdata.conquery.models.identifiable.ids.specific.FormConfigId; import com.bakdata.conquery.models.identifiable.ids.specific.GroupId; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; @@ -26,7 +25,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -public class MetaStorage implements ConqueryStorage { +public class MetaStorage implements ConqueryStorage, Injectable { private IdentifiableStore> executions; @@ -47,9 +46,9 @@ public MetaStorage(Validator validator, StoreFactory storageFactory, DatasetRegi this.validator = validator; - authUser = storageFactory.createUserStore(centralRegistry, "meta", new User.StorageUpdater(this::updateUser)); - authRole = storageFactory.createRoleStore(centralRegistry, "meta", new Role.StorageUpdater(this::updateRole)); - authGroup = storageFactory.createGroupStore(centralRegistry, "meta", new Group.StorageUpdater(this::updateGroup)); + authUser = storageFactory.createUserStore(centralRegistry, "meta", this); + authRole = storageFactory.createRoleStore(centralRegistry, "meta", this); + authGroup = storageFactory.createGroupStore(centralRegistry, "meta", this); // Executions depend on users executions = storageFactory.createExecutionsStore(centralRegistry, datasetRegistry, "meta"); formConfigs = storageFactory.createFormConfigStore(centralRegistry, datasetRegistry, "meta"); @@ -198,9 +197,8 @@ public void close() throws IOException { authGroup.close(); } - /** - * An implementation of this interface should be provided by mutable stored entities, so that - * these entities can update their persistent state themselves on change in a thread safe manner. - */ - public interface StorageUpdater> extends Consumer, Injectable{}; + @Override + public MutableInjectableValues inject(MutableInjectableValues values) { + return values.add(MetaStorage.class, this); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationController.java b/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationController.java index 5959fc6a3d..8ba051408d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationController.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationController.java @@ -193,7 +193,7 @@ public static User flatCopyUser(@NonNull User originUser, String namePrefix, @No ); // Create copied user - User copy = new User(name, originUser.getLabel(), storage::updateUser); + User copy = new User(name, originUser.getLabel(), storage); storage.addUser(copy); copy.updatePermissions(copiedPermission); diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Group.java b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Group.java index 1c3f29f19f..b0d95b3e19 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Group.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Group.java @@ -3,19 +3,12 @@ import java.util.Collections; import java.util.HashSet; import java.util.Set; -import java.util.function.Consumer; -import com.bakdata.conquery.io.jackson.MutableInjectableValues; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.models.identifiable.ids.specific.GroupId; import com.bakdata.conquery.models.identifiable.ids.specific.RoleId; import com.bakdata.conquery.models.identifiable.ids.specific.UserId; -import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.OptBoolean; -import lombok.EqualsAndHashCode; -import lombok.NonNull; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; /** @@ -32,20 +25,13 @@ public class Group extends PermissionOwner implements RoleOwner { @JsonProperty private Set roles = Collections.synchronizedSet(new HashSet<>()); - - @JacksonInject(useInput = OptBoolean.FALSE) - @NonNull - @EqualsAndHashCode.Exclude - protected StorageUpdater storageUpdater; - - public Group(String name, String label, Consumer storageUpdater) { - super(name, label); - this.storageUpdater = new StorageUpdater(storageUpdater); + public Group(String name, String label, MetaStorage storage) { + super(name, label, storage); } @Override - protected void updateStorage() { - storageUpdater.accept(this); + protected void updateStorage(MetaStorage storage) { + storage.updateGroup(this); } @Override @@ -56,14 +42,14 @@ public GroupId createId() { public synchronized void addMember(User user) { if(members.add(user.getId())) { log.trace("Added user {} to group {}", user.getId(), getId()); - updateStorage(); + updateStorage(storage); } } public synchronized void removeMember(User user) { if (members.remove(user.getId())) { log.trace("Removed user {} from group {}", user.getId(), getId()); - updateStorage(); + updateStorage(storage); } } @@ -78,37 +64,18 @@ public Set getMembers() { public synchronized void addRole(Role role) { if (roles.add(role.getId())) { log.trace("Added role {} to group {}", role.getId(), getId()); - updateStorage(); + updateStorage(storage); } } public synchronized void removeRole(Role role) { if (roles.remove(role.getId())) { log.trace("Removed role {} from group {}", role.getId(), getId()); - updateStorage(); + updateStorage(storage); } } public Set getRoles() { return Collections.unmodifiableSet(roles); } - - - - @RequiredArgsConstructor - public static class StorageUpdater implements MetaStorage.StorageUpdater { - - private final Consumer storageUpdater; - - @Override - public void accept(Group group) { - storageUpdater.accept(group); - } - - @Override - public MutableInjectableValues inject(MutableInjectableValues values) { - values.add(Group.StorageUpdater.class, this); - return values; - } - } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/PermissionOwner.java b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/PermissionOwner.java index e072f503bb..052957a59b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/PermissionOwner.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/PermissionOwner.java @@ -12,6 +12,9 @@ import com.bakdata.conquery.models.auth.permissions.ConqueryPermission; import com.bakdata.conquery.models.identifiable.IdentifiableImpl; import com.bakdata.conquery.models.identifiable.ids.specific.PermissionOwnerId; +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.OptBoolean; import com.google.common.collect.ImmutableSet; import lombok.AccessLevel; import lombok.EqualsAndHashCode; @@ -57,10 +60,16 @@ public abstract class PermissionOwner permissions = new HashSet<>(); + @JacksonInject(useInput = OptBoolean.FALSE) + @NonNull + @EqualsAndHashCode.Exclude + protected MetaStorage storage; + - public PermissionOwner(String name, String label) { + public PermissionOwner(String name, String label, MetaStorage storage) { this.name = name; this.label = label; + this.storage = storage; } @@ -76,7 +85,7 @@ public synchronized void addPermissions(Set permissions) { .addAll(this.permissions) .addAll(permissions) .build(); - updateStorage(); + updateStorage(this.storage); } public synchronized void addPermission(ConqueryPermission permission) { @@ -85,7 +94,7 @@ public synchronized void addPermission(ConqueryPermission permission) { .addAll(this.permissions) .add(permission) .build(); - updateStorage(); + updateStorage(this.storage); } /** @@ -100,7 +109,7 @@ public boolean removePermissions(Set permissions) { Set newSet = new HashSet<>(this.permissions); ret = newSet.removeAll(permissions); this.permissions = newSet; - updateStorage(); + updateStorage(this.storage); } return ret; } @@ -111,7 +120,7 @@ public boolean removePermission(Permission permission) { Set newSet = new HashSet<>(this.permissions); ret = newSet.remove(permission); this.permissions = newSet; - updateStorage(); + updateStorage(this.storage); } return ret; } @@ -136,14 +145,14 @@ public void updatePermissions(Set permissionsNew) { Set newSet = new HashSet<>(permissionsNew.size()); newSet.addAll(permissionsNew); this.permissions = newSet; - updateStorage(); + updateStorage(this.storage); } } /** * Update this instance in the {@link MetaStorage}. */ - protected abstract void updateStorage(); + protected abstract void updateStorage(MetaStorage storage); @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Role.java b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Role.java index 7f4d68a368..1f73a79c24 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Role.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Role.java @@ -1,57 +1,24 @@ package com.bakdata.conquery.models.auth.entities; -import java.util.function.Consumer; - -import com.bakdata.conquery.io.jackson.MutableInjectableValues; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.models.identifiable.ids.specific.RoleId; -import com.fasterxml.jackson.annotation.JacksonInject; -import com.fasterxml.jackson.annotation.OptBoolean; -import lombok.EqualsAndHashCode; -import lombok.NonNull; -import lombok.RequiredArgsConstructor; public class Role extends PermissionOwner { + public Role(String name, String label, MetaStorage storage) { + super(name, label, storage); + } + @Override public RoleId createId() { return new RoleId(name); } - - @JacksonInject(useInput = OptBoolean.FALSE) - @NonNull - @EqualsAndHashCode.Exclude - protected StorageUpdater storageUpdater; - - public Role(String name, String label, Consumer storageUpdater) { - super(name, label); - this.storageUpdater = new StorageUpdater(storageUpdater); - } - + @Override - protected void updateStorage() { - storageUpdater.accept(this); - - } - - - - @RequiredArgsConstructor - public static class StorageUpdater implements MetaStorage.StorageUpdater { - - private final Consumer storageUpdater; - - @Override - public void accept(Role role) { - storageUpdater.accept(role); - } - - @Override - public MutableInjectableValues inject(MutableInjectableValues values) { - values.add(StorageUpdater.class, this); - return values; - } + protected void updateStorage(MetaStorage storage) { + storage.updateRole(this); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/User.java b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/User.java index 0c8d8aac54..7d5485923c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/User.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/User.java @@ -7,10 +7,8 @@ import java.util.HashSet; import java.util.List; import java.util.Set; -import java.util.function.Consumer; import java.util.stream.Collectors; -import com.bakdata.conquery.io.jackson.MutableInjectableValues; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.models.auth.permissions.Ability; import com.bakdata.conquery.models.auth.permissions.Authorized; @@ -18,16 +16,12 @@ import com.bakdata.conquery.models.execution.Owned; import com.bakdata.conquery.models.identifiable.ids.specific.RoleId; import com.bakdata.conquery.models.identifiable.ids.specific.UserId; -import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.OptBoolean; import it.unimi.dsi.fastutil.booleans.BooleanArrayList; import lombok.AccessLevel; -import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.NonNull; -import lombok.RequiredArgsConstructor; import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.apache.shiro.SecurityUtils; @@ -51,16 +45,9 @@ public class User extends PermissionOwner implements Principal, RoleOwne @Getter(AccessLevel.PROTECTED) private final transient ShiroUserAdapter shiroUserAdapter; - - @JacksonInject(useInput = OptBoolean.FALSE) - @NonNull - @EqualsAndHashCode.Exclude - protected User.StorageUpdater storageUpdater; - - public User(String name, String label, Consumer storageUpdater) { - super(name, label); + public User(String name, String label, MetaStorage storage) { + super(name, label, storage); this.shiroUserAdapter = new ShiroUserAdapter(); - this.storageUpdater = new StorageUpdater(storageUpdater); } @Override @@ -71,7 +58,7 @@ public UserId createId() { public synchronized void addRole(Role role) { if (roles.add(role.getId())) { log.trace("Added role {} to user {}", role.getId(), getId()); - updateStorage(); + updateStorage(storage); } } @@ -79,7 +66,7 @@ public synchronized void addRole(Role role) { public synchronized void removeRole(Role role) { if (roles.remove(role.getId())) { log.trace("Removed role {} from user {}", role.getId(), getId()); - updateStorage(); + updateStorage(storage); } } @@ -88,8 +75,8 @@ public Set getRoles() { } @Override - protected void updateStorage() { - storageUpdater.accept(this); + protected void updateStorage(MetaStorage storage) { + storage.updateUser(this); } public void authorize(@NonNull Authorized object, @NonNull Ability ability) { @@ -177,21 +164,4 @@ public Object getPrincipal() { } - - @RequiredArgsConstructor - public static class StorageUpdater implements MetaStorage.StorageUpdater { - - private final Consumer storageUpdater; - - @Override - public void accept(User user) { - storageUpdater.accept(user); - } - - @Override - public MutableInjectableValues inject(MutableInjectableValues values) { - values.add(StorageUpdater.class, this); - return values; - } - } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/IntrospectionDelegatingRealm.java b/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/IntrospectionDelegatingRealm.java index e8af2d6ad5..bbc2184346 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/IntrospectionDelegatingRealm.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/IntrospectionDelegatingRealm.java @@ -195,7 +195,7 @@ private synchronized User getOrCreateUser(TokenIntrospectionSuccessResponse succ } // try to construct a new User if none could be found in the storage String userLabel = successResponse.getStringParameter("name"); - user = new User(username, userLabel != null ? userLabel : username, storage::updateUser); + user = new User(username, userLabel != null ? userLabel : username, storage); storage.addUser(user); log.info("Created new user: {}", user); return user; @@ -213,7 +213,7 @@ private synchronized Group getOrCreateGroup(Pair groupNameId) { if (group != null) { return group; } - group = new Group(groupNameId.getValue().getGroup(), groupNameId.getKey(), storage::updateGroup); + group = new Group(groupNameId.getValue().getGroup(), groupNameId.getKey(), storage); storage.addGroup(group); log.info("Created new group: {}", group); return group; diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java index a6af0bb891..ebe79309b7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java @@ -64,9 +64,9 @@ public interface StoreFactory { // MetaStorage IdentifiableStore> createExecutionsStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName); IdentifiableStore createFormConfigStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName); - IdentifiableStore createUserStore(CentralRegistry centralRegistry, String pathName, User.StorageUpdater storageUpdater); - IdentifiableStore createRoleStore(CentralRegistry centralRegistry, String pathName, Role.StorageUpdater storageUpdater); - IdentifiableStore createGroupStore(CentralRegistry centralRegistry, String pathName, Group.StorageUpdater storageUpdater); + IdentifiableStore createUserStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage); + IdentifiableStore createRoleStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage); + IdentifiableStore createGroupStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage); SingletonStore createPrimaryDictionaryStore(String pathName, SingletonNamespaceCollection namespaceCollection); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java index c80f460689..753908b1b0 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java @@ -315,19 +315,19 @@ public IdentifiableStore createFormConfigStore(CentralRegistry centr } @Override - public IdentifiableStore createUserStore(CentralRegistry centralRegistry, String pathName, User.StorageUpdater storageUpdater) { - return StoreInfo.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "users")), validator, AUTH_USER), centralRegistry, storageUpdater); + public IdentifiableStore createUserStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage) { + return StoreInfo.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "users")), validator, AUTH_USER), centralRegistry, storage); } @Override - public IdentifiableStore createRoleStore(CentralRegistry centralRegistry, String pathName, Role.StorageUpdater storageUpdater) { - return StoreInfo.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "roles")), validator, AUTH_ROLE), centralRegistry, storageUpdater); + public IdentifiableStore createRoleStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage) { + return StoreInfo.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "roles")), validator, AUTH_ROLE), centralRegistry, storage); } @Override - public IdentifiableStore createGroupStore(CentralRegistry centralRegistry, String pathName, Group.StorageUpdater storageUpdater) { - return StoreInfo.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "groups")), validator, AUTH_GROUP), centralRegistry, storageUpdater); + public IdentifiableStore createGroupStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage) { + return StoreInfo.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "groups")), validator, AUTH_GROUP), centralRegistry, storage); } @Override diff --git a/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java b/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java index b21f1828a8..179860a07c 100644 --- a/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java +++ b/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java @@ -117,7 +117,7 @@ public void getQueriesFiltered() { } private static User mockUser(int id, List allowedQueryIds) { - final User user = new User("user" + id, null, STORAGE::updateUser); + final User user = new User("user" + id, null, STORAGE); STORAGE.addUser(user); diff --git a/backend/src/test/java/com/bakdata/conquery/api/form/config/FormConfigTest.java b/backend/src/test/java/com/bakdata/conquery/api/form/config/FormConfigTest.java index 983807076d..57bd8c7ef4 100644 --- a/backend/src/test/java/com/bakdata/conquery/api/form/config/FormConfigTest.java +++ b/backend/src/test/java/com/bakdata/conquery/api/form/config/FormConfigTest.java @@ -141,7 +141,7 @@ public void setupTest(){ mode.setFeatures(List.of(new CQConcept())); - user = new User("test","test", storage::updateUser); + user = new User("test","test", storage); storage.addUser(user); } @@ -327,9 +327,9 @@ public void getConfigs() { public void patchConfig() { // PREPARE user.addPermission(DatasetPermission.onInstance(Ability.READ, datasetId)); - Group group1 = new Group("test1","test1", storage::updateGroup); + Group group1 = new Group("test1","test1", storage); storage.addGroup(group1); - Group group2 = new Group("test2","test2", storage::updateGroup); + Group group2 = new Group("test2","test2", storage); storage.addGroup(group2); group1.addMember(user); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/DownloadLinkGeneration.java b/backend/src/test/java/com/bakdata/conquery/integration/DownloadLinkGeneration.java index 391e9aa0b7..ded19ed71b 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/DownloadLinkGeneration.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/DownloadLinkGeneration.java @@ -28,7 +28,7 @@ public class DownloadLinkGeneration extends IntegrationTest.Simple implements Pr public void execute(StandaloneSupport conquery) throws Exception { final MetaStorage storage = conquery.getMetaStorage(); - final User user = new User("testU", "testU", storage::updateUser); + final User user = new User("testU", "testU", storage); final String testJson = In.resource("/tests/query/SIMPLE_TREECONCEPT_QUERY/SIMPLE_TREECONCEPT_Query.test.json").withUTF8() .readAll(); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/ConceptPermissionTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/ConceptPermissionTest.java index 13c8e3968b..6698adca50 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/ConceptPermissionTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/ConceptPermissionTest.java @@ -33,7 +33,7 @@ public void execute(StandaloneSupport conquery) throws Exception { final String testJson = In.resource("/tests/query/SIMPLE_TREECONCEPT_QUERY/SIMPLE_TREECONCEPT_Query.test.json").withUTF8().readAll(); final QueryTest test = (QueryTest) JsonIntegrationTest.readJson(dataset.getId(), testJson); final QueryProcessor processor = new QueryProcessor(storage.getDatasetRegistry(), storage, conquery.getConfig()); - final User user = new User("testUser", "testUserLabel", storage::updateUser); + final User user = new User("testUser", "testUserLabel", storage); // Manually import data, so we can do our own work. { diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/GroupHandlingTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/GroupHandlingTest.java index e0cdf68b05..8c156caa29 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/GroupHandlingTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/GroupHandlingTest.java @@ -21,10 +21,10 @@ public class GroupHandlingTest extends IntegrationTest.Simple implements Program public void execute(StandaloneSupport conquery) throws Exception { MetaStorage storage = conquery.getMetaStorage(); - Group group1 = new Group("company", "company", storage::updateGroup); - User user1 = new User("user", "user", storage::updateUser); - User user1copy = new User("user", "user", storage::updateUser); - User user2 = new User("user2", "user2", storage::updateUser); + Group group1 = new Group("company", "company", storage); + User user1 = new User("user", "user", storage); + User user1copy = new User("user", "user", storage); + User user2 = new User("user2", "user2", storage); try { storage.addGroup(group1); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/PermissionGroupHandlingTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/PermissionGroupHandlingTest.java index 99f7190f1c..8edf25ea9e 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/PermissionGroupHandlingTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/PermissionGroupHandlingTest.java @@ -30,9 +30,9 @@ public void execute(StandaloneSupport conquery) throws Exception { ManagedExecutionId query1 = new ManagedExecutionId(dataset1.getId(), UUID.randomUUID()); - Role role1 = new Role("role", "role", storage::updateRole); + Role role1 = new Role("role", "role", storage); TestUser user1 = new TestUser(storage); - Group group1 = new Group("company", "company", storage::updateGroup); + Group group1 = new Group("company", "company", storage); try { diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/PermissionRoleHandlingTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/PermissionRoleHandlingTest.java index 448cf6ae7b..105abea3f7 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/PermissionRoleHandlingTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/PermissionRoleHandlingTest.java @@ -20,7 +20,7 @@ public class PermissionRoleHandlingTest extends IntegrationTest.Simple implement @Override public void execute(StandaloneSupport conquery) throws Exception { MetaStorage storage = conquery.getMetaStorage(); - Role mandator1 = new Role("company", "company", storage::updateRole); + Role mandator1 = new Role("company", "company", storage); TestUser user1 = new TestUser(storage); Dataset dataset = conquery.getDataset(); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/RestartTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/RestartTest.java index 62fa7a5d10..878f1ba86b 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/RestartTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/RestartTest.java @@ -78,12 +78,12 @@ public void execute(String name, TestConquery testConquery) throws Exception { MetaStorage storage = conquery.getMetaStorage(); - Role role = new Role("role", "ROLE", storage::updateRole); - Role roleToDelete = new Role("roleDelete", "ROLE_DELETE", storage::updateRole); - User user = new User("user@test.email", "USER", storage::updateUser); - User userToDelete = new User("userDelete@test.email", "USER_DELETE", storage::updateUser); - Group group = new Group("group", "GROUP", storage::updateGroup); - Group groupToDelete = new Group("groupDelete", "GROUP_DELETE", storage::updateGroup); + Role role = new Role("role", "ROLE", storage); + Role roleToDelete = new Role("roleDelete", "ROLE_DELETE", storage); + User user = new User("user@test.email", "USER", storage); + User userToDelete = new User("userDelete@test.email", "USER_DELETE", storage); + Group group = new Group("group", "GROUP", storage); + Group groupToDelete = new Group("groupDelete", "GROUP_DELETE", storage); {// Auth testing (deletion and permission grant) // build constellation diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/ReusedQueryTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/ReusedQueryTest.java index c915275fd5..a8efca5647 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/ReusedQueryTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/ReusedQueryTest.java @@ -219,7 +219,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { reused.setSecondaryId(query.getSecondaryId()); - User shareHolder = new User("shareholder", "ShareHolder", conquery.getMetaStorage()::updateUser); + User shareHolder = new User("shareholder", "ShareHolder", conquery.getMetaStorage()); conquery.getMetaProcessor().addUser(shareHolder); shareHolder.addPermissions(Set.of( diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/RoleHandlingOnGroupTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/RoleHandlingOnGroupTest.java index 0cd6892665..205e39d5de 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/RoleHandlingOnGroupTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/RoleHandlingOnGroupTest.java @@ -27,8 +27,8 @@ public void execute(StandaloneSupport conquery) throws Exception { dataset1.setLabel("dataset1"); MetaStorage storage = conquery.getMetaStorage(); - Group group1 = new Group("company", "company", storage::updateGroup); - Role role = new Role("role1", "role1", storage::updateRole); + Group group1 = new Group("company", "company", storage); + Role role = new Role("role1", "role1", storage); TestUser user1 = new TestUser(storage); try { diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/RoleHandlingTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/RoleHandlingTest.java index 6c3f0ad01c..ba797e04be 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/RoleHandlingTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/RoleHandlingTest.java @@ -24,10 +24,10 @@ public void execute(StandaloneSupport conquery) throws Exception { dataset1.setLabel("dataset1"); MetaStorage storage = conquery.getMetaStorage(); - Role mandator1 = new Role("company", "company", storage::updateRole); - Role mandator1Copy = new Role("company", "company", storage::updateRole); - Role mandator2 = new Role("company2", "company2", storage::updateRole); - User user1 = new User("user", "user", storage::updateUser); + Role mandator1 = new Role("company", "company", storage); + Role mandator1Copy = new Role("company", "company", storage); + Role mandator2 = new Role("company2", "company2", storage); + User user1 = new User("user", "user", storage); try { storage.addRole(mandator1); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/RoleUITest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/RoleUITest.java index d344fb4d59..7145165d96 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/RoleUITest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/RoleUITest.java @@ -34,9 +34,9 @@ public class RoleUITest extends IntegrationTest.Simple implements ProgrammaticIn @Override public void execute(StandaloneSupport conquery) throws Exception { MetaStorage storage = conquery.getMetaStorage(); - Role mandator = new Role("testMandatorName", "testMandatorLabel", storage::updateRole); + Role mandator = new Role("testMandatorName", "testMandatorLabel", storage); RoleId mandatorId = mandator.getId(); - User user = new User("testUser@test.de", "testUserName", storage::updateUser); + User user = new User("testUser@test.de", "testUserName", storage); UserId userId = user.getId(); try { diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/SuperPermissionTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/SuperPermissionTest.java index a3aee8c05b..d83c1bd9e5 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/SuperPermissionTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/SuperPermissionTest.java @@ -22,7 +22,7 @@ public void execute(StandaloneSupport conquery) throws Exception { dataset1.setLabel("dataset1"); MetaStorage storage = conquery.getMetaStorage(); - Role role1 = new Role("company", "company", storage::updateRole); + Role role1 = new Role("company", "company", storage); TestUser user1 = new TestUser(storage); storage.addRole(role1); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/TestUser.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/TestUser.java index b4b849b974..f07ccbca0b 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/TestUser.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/TestUser.java @@ -10,7 +10,7 @@ public class TestUser extends User { public TestUser(MetaStorage storage) { - super("user", "user", storage::updateUser); + super("user", "user", storage); } public boolean isPermitted(Permission permission) { diff --git a/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/IdRefrenceTest.java b/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/IdRefrenceTest.java index f47178460f..f8c5a44d38 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/IdRefrenceTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/IdRefrenceTest.java @@ -33,7 +33,7 @@ public void testListReferences() throws IOException { registry.register(table); final CentralRegistry metaRegistry = new CentralRegistry(); - User user = new User("usermail", "userlabel", mock(MetaStorage.class)::updateUser); + User user = new User("usermail", "userlabel", mock(MetaStorage.class)); metaRegistry.register(user); String json = Jackson.MAPPER.writeValueAsString( diff --git a/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStoreDumpTest.java b/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStoreDumpTest.java index f42e9f1668..54a700161b 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStoreDumpTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStoreDumpTest.java @@ -40,6 +40,7 @@ public class SerializingStoreDumpTest { public static final IStoreInfo USER_STORE_ID = StoreInfo.AUTH_USER.storeInfo(); + private final static MetaStorage STORAGE = new MetaStorage(Validators.newValidator(), new NonPersistentStoreFactory(), new DatasetRegistry(2)); private File tmpDir; private Environment env; private XodusStoreFactory config; @@ -48,6 +49,7 @@ public class SerializingStoreDumpTest { private final ManagedQuery managedQuery = new ManagedQuery(null, null, new Dataset("dataset")); private final ConceptQuery cQuery = new ConceptQuery( new CQReusedQuery(managedQuery.getId())); + private final User user = new User("username", "userlabel", STORAGE); @BeforeEach public void init() { @@ -77,8 +79,6 @@ public void testCorruptValueDump() throws JSONException, IOException { { // Open a store and insert a valid key-value pair (UserId & User) SerializingStore store = createSerializedStore(config, env, Validators.newValidator(), USER_STORE_ID); - - final User user = new User("username", "userlabel", u -> store.add(u.getId(),u)); store.add(user.getId(), user); } @@ -132,8 +132,6 @@ public void testCorruptKeyDump() throws JSONException, IOException { { // Open a store and insert a valid key-value pair (UserId & User) SerializingStore store = createSerializedStore(config, env, Validators.newValidator(), USER_STORE_ID); - - final User user = new User("username", "userlabel", u -> store.add(u.getId(),u)); store.add(new UserId("testU1"), user); } @@ -185,7 +183,6 @@ public void testCorruptionRemoval() throws JSONException, IOException { { // Open a store and insert a valid key-value pair (UserId & User) SerializingStore store = createSerializedStore(config, env, Validators.newValidator(), USER_STORE_ID); - final User user = new User("username", "userlabel", u -> store.add(u.getId(),u)); store.add(new UserId("testU1"), user); } diff --git a/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java b/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java index 1bd73b2be0..1af1fca99c 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java +++ b/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java @@ -85,11 +85,11 @@ public void passwordCredential() throws IOException, JSONException { @Test public void role() throws IOException, JSONException { - Role mandator = new Role("company", "company", STORAGE::updateRole); + Role mandator = new Role("company", "company", STORAGE); SerializationTestUtil .forType(Role.class) - .injectables(new Role.StorageUpdater(STORAGE::updateRole)) + .injectables(STORAGE) .test(mandator); } @@ -99,13 +99,13 @@ public void role() throws IOException, JSONException { @Test public void user() throws IOException, JSONException { MetaStorage storage = new MetaStorage(null, new NonPersistentStoreFactory(), null); - User user = new User("user", "user", STORAGE::updateUser); + User user = new User("user", "user", STORAGE); user.addPermission(DatasetPermission.onInstance(Ability.READ, new DatasetId("test"))); user .addPermission( ExecutionPermission.onInstance(Ability.READ, new ManagedExecutionId(new DatasetId("dataset"), UUID.randomUUID())) ); - Role role = new Role("company", "company", STORAGE::updateRole); + Role role = new Role("company", "company", STORAGE); user.addRole(role); CentralRegistry registry = new CentralRegistry(); @@ -114,24 +114,24 @@ public void user() throws IOException, JSONException { SerializationTestUtil .forType(User.class) .registry(registry) - .injectables(new User.StorageUpdater(STORAGE::updateUser)) + .injectables(STORAGE) .test(user); } @Test public void group() throws IOException, JSONException { MetaStorage storage = new MetaStorage(null, new NonPersistentStoreFactory(), null); - Group group = new Group("group", "group", STORAGE::updateGroup); + Group group = new Group("group", "group", STORAGE); group.addPermission(DatasetPermission.onInstance(Ability.READ, new DatasetId("test"))); group .addPermission( ExecutionPermission.onInstance(Ability.READ, new ManagedExecutionId(new DatasetId("dataset"), UUID.randomUUID())) ); - group.addRole(new Role("company", "company", STORAGE::updateRole)); + group.addRole(new Role("company", "company", STORAGE)); - Role role = new Role("company", "company", STORAGE::updateRole); + Role role = new Role("company", "company", STORAGE); group.addRole(role); - User user = new User("userName", "userLabel", STORAGE::updateUser); + User user = new User("userName", "userLabel", STORAGE); group.addMember(user); CentralRegistry registry = new CentralRegistry(); @@ -140,7 +140,7 @@ public void group() throws IOException, JSONException { SerializationTestUtil .forType(Group.class) - .injectables(new Group.StorageUpdater(STORAGE::updateGroup)) + .injectables(STORAGE) .registry(registry) .test(group); } @@ -247,7 +247,7 @@ public void managedQuery() throws JSONException, IOException { final Dataset dataset = new Dataset("test-dataset"); - final User user = new User("test-user", "test-user", STORAGE::updateUser); + final User user = new User("test-user", "test-user", STORAGE); registry.register(dataset); registry.register(user); @@ -327,7 +327,7 @@ public void executionQueryJobError() throws JSONException, IOException { @Test public void meInformation() throws IOException, JSONException { - User user = new User("name", "labe", STORAGE::updateUser); + User user = new User("name", "labe", STORAGE); MeProcessor.FEMeInformation info = MeProcessor.FEMeInformation.builder() .userName(user.getLabel()) diff --git a/backend/src/test/java/com/bakdata/conquery/models/auth/CopyUserTest.java b/backend/src/test/java/com/bakdata/conquery/models/auth/CopyUserTest.java index c19873566b..ca96c6de74 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/auth/CopyUserTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/auth/CopyUserTest.java @@ -25,17 +25,17 @@ void testUserCopy(){ registry.setMetaStorage(storage); // Create test role - Role role = new Role("role", "role", storage::updateRole); + Role role = new Role("role", "role", storage); storage.addRole(role); role.addPermission(DatasetPermission.onInstance(Ability.READ, new DatasetId("dataset0"))); // Create test group - Group group = new Group("group", "group", storage::updateGroup); + Group group = new Group("group", "group", storage); storage.addGroup(group); group.addPermission(DatasetPermission.onInstance(Ability.READ, new DatasetId("dataset1"))); // Create original user with role and group mapping - User originUser = new User("user", "user", storage::updateUser); + User originUser = new User("user", "user", storage); storage.addUser(originUser); originUser.addRole(role); group.addMember(originUser); diff --git a/backend/src/test/java/com/bakdata/conquery/models/auth/IntrospectionDelegatingRealmTest.java b/backend/src/test/java/com/bakdata/conquery/models/auth/IntrospectionDelegatingRealmTest.java index d90d7bd12c..f17a44bca6 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/auth/IntrospectionDelegatingRealmTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/auth/IntrospectionDelegatingRealmTest.java @@ -60,7 +60,7 @@ public class IntrospectionDelegatingRealmTest { private static final BearerToken USER_3_TOKEN_WRAPPED = new BearerToken(USER_3_TOKEN); // Groups private static final String GROUPNAME_1 = "group1"; - private static final Group GROUP_1_EXISTING = new Group(GROUPNAME_1, GROUPNAME_1, STORAGE::updateGroup); + private static final Group GROUP_1_EXISTING = new Group(GROUPNAME_1, GROUPNAME_1, STORAGE); private static final String GROUPNAME_2 = "group2"; // Group is created during test private static OIDCMockServer OIDC_SERVER; @@ -157,12 +157,12 @@ public void tokenIntrospectionSimpleUserNew() { .usingRecursiveComparison() .ignoringFields(ConqueryAuthenticationInfo.Fields.credentials) .isEqualTo(new ConqueryAuthenticationInfo(new UserId(USER_1_NAME), USER1_TOKEN_WRAPPED, REALM, true)); - assertThat(STORAGE.getAllUsers()).containsOnly(new User(USER_1_NAME, USER_1_NAME, STORAGE::updateUser)); + assertThat(STORAGE.getAllUsers()).containsOnly(new User(USER_1_NAME, USER_1_NAME, STORAGE)); } @Test public void tokenIntrospectionSimpleUserExisting() { - User existingUser = new User(USER_1_NAME, USER_1_NAME, STORAGE::updateUser); + User existingUser = new User(USER_1_NAME, USER_1_NAME, STORAGE); STORAGE.addUser(existingUser); AuthenticationInfo info = REALM.doGetAuthenticationInfo(USER1_TOKEN_WRAPPED); @@ -182,7 +182,7 @@ public void tokenIntrospectionGroupedUser() { .usingRecursiveComparison() .ignoringFields(ConqueryAuthenticationInfo.Fields.credentials) .isEqualTo(new ConqueryAuthenticationInfo(new UserId(USER_2_NAME), USER_2_TOKEN_WRAPPED, REALM, true)); - assertThat(STORAGE.getAllUsers()).containsOnly(new User(USER_2_NAME, USER_2_LABEL, STORAGE::updateUser)); + assertThat(STORAGE.getAllUsers()).containsOnly(new User(USER_2_NAME, USER_2_LABEL, STORAGE)); assertThat(STORAGE.getAllGroups()).hasSize(2); // Pre-existing group and a second group that has been added in the process assertThat(STORAGE.getGroup(new GroupId(GROUPNAME_1)).getMembers()).contains(new UserId(USER_2_NAME)); assertThat(STORAGE.getGroup(new GroupId(GROUPNAME_2)).getMembers()).contains(new UserId(USER_2_NAME)); @@ -190,7 +190,7 @@ public void tokenIntrospectionGroupedUser() { @Test public void tokenIntrospectionGroupedUserRemoveGroupMapping() { - User user = new User(USER_3_NAME, USER_3_LABEL, STORAGE::updateUser); + User user = new User(USER_3_NAME, USER_3_LABEL, STORAGE); STORAGE.addUser(user); GROUP_1_EXISTING.addMember(user); @@ -202,7 +202,7 @@ public void tokenIntrospectionGroupedUserRemoveGroupMapping() { .usingRecursiveComparison() .ignoringFields(ConqueryAuthenticationInfo.Fields.credentials) .isEqualTo(new ConqueryAuthenticationInfo(new UserId(USER_3_NAME), USER_3_TOKEN_WRAPPED, REALM, true)); - assertThat(STORAGE.getAllUsers()).containsOnly(new User(USER_3_NAME, USER_3_LABEL, STORAGE::updateUser)); + assertThat(STORAGE.getAllUsers()).containsOnly(new User(USER_3_NAME, USER_3_LABEL, STORAGE)); assertThat(STORAGE.getAllGroups()).hasSize(1); // Pre-existing group assertThat(STORAGE.getGroup(new GroupId(GROUPNAME_1)).getMembers()).doesNotContain(new UserId(USER_3_NAME)); } diff --git a/backend/src/test/java/com/bakdata/conquery/models/auth/LocalAuthRealmTest.java b/backend/src/test/java/com/bakdata/conquery/models/auth/LocalAuthRealmTest.java index fcbd15113d..670b0eff8e 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/auth/LocalAuthRealmTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/auth/LocalAuthRealmTest.java @@ -59,7 +59,7 @@ public void setupAll() throws Exception { @BeforeEach public void setupEach() { // Create User in Realm - user1 = new User("TestUser", "Test User", storage::updateUser); + user1 = new User("TestUser", "Test User", storage); PasswordCredential user1Password = new PasswordCredential("testPassword".toCharArray()); storage.addUser(user1); realm.addUser(user1, List.of(user1Password)); diff --git a/backend/src/test/java/com/bakdata/conquery/models/execution/DefaultLabelTest.java b/backend/src/test/java/com/bakdata/conquery/models/execution/DefaultLabelTest.java index 2902cbd9f7..906b75bf2c 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/execution/DefaultLabelTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/execution/DefaultLabelTest.java @@ -39,7 +39,7 @@ public class DefaultLabelTest { private static final DatasetRegistry DATASET_REGISTRY = Mockito.mock(DatasetRegistry.class); private static final Dataset DATASET = new Dataset("dataset"); - private static final User user = new User("user","user", STORAGE::updateUser); + private static final User user = new User("user","user", STORAGE); private static final TreeConcept CONCEPT = new TreeConcept() { { diff --git a/backend/src/test/java/com/bakdata/conquery/tasks/PermissionCleanupTaskTest.java b/backend/src/test/java/com/bakdata/conquery/tasks/PermissionCleanupTaskTest.java index d828d468ce..debbc14e33 100644 --- a/backend/src/test/java/com/bakdata/conquery/tasks/PermissionCleanupTaskTest.java +++ b/backend/src/test/java/com/bakdata/conquery/tasks/PermissionCleanupTaskTest.java @@ -56,7 +56,7 @@ void doNotDeletePermissionValidReference() { final ManagedQuery managedQuery = createManagedQuery(); // Saving the Execution - User user = new User("test", "test", storage::updateUser); + User user = new User("test", "test", storage); storage.updateUser(user); user.addPermission(ExecutionPermission.onInstance(AbilitySets.QUERY_CREATOR, managedQuery.getId())); @@ -73,7 +73,7 @@ void doDeletePermissionInvalidReference() { final ManagedQuery managedQuery = createManagedQuery(); // Removing the execution storage.removeExecution(managedQuery.getId()); - User user = new User("test", "test", storage::updateUser); + User user = new User("test", "test", storage); storage.updateUser(user); user.addPermission(ExecutionPermission.onInstance(AbilitySets.QUERY_CREATOR, managedQuery.getId())); @@ -91,7 +91,7 @@ void doDeletePartialPermissionWithInvalidReference() { final ManagedQuery managedQuery2 = createManagedQuery(); // Removing the second execution storage.removeExecution(managedQuery2.getId()); - User user = new User("test", "test", storage::updateUser); + User user = new User("test", "test", storage); storage.updateUser(user); user.addPermission( // Build a permission with multiple instances @@ -114,8 +114,8 @@ void doDeletePermissionsOfOwnedReference() { // Created owned execution final ManagedQuery managedQueryOwned = createManagedQuery(); // Setup user - User user = new User("test", "test", storage::updateUser); - User user2 = new User("test2", "test2", storage::updateUser); + User user = new User("test", "test", storage); + User user2 = new User("test2", "test2", storage); storage.updateUser(user); user.addPermission(ExecutionPermission.onInstance(AbilitySets.QUERY_CREATOR, managedQueryOwned.getId())); diff --git a/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java b/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java index 147fcde138..d5aff76281 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java +++ b/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java @@ -117,17 +117,17 @@ public IdentifiableStore createFormConfigStore(CentralRegistry centr } @Override - public IdentifiableStore createUserStore(CentralRegistry centralRegistry, String pathName, User.StorageUpdater storageUpdater) { + public IdentifiableStore createUserStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage) { return AUTH_USER.identifiable(new NonPersistentStore(), centralRegistry); } @Override - public IdentifiableStore createRoleStore(CentralRegistry centralRegistry, String pathName, Role.StorageUpdater storageUpdater) { + public IdentifiableStore createRoleStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage) { return AUTH_ROLE.identifiable(new NonPersistentStore(), centralRegistry); } @Override - public IdentifiableStore createGroupStore(CentralRegistry centralRegistry, String pathName, Group.StorageUpdater storageUpdater) { + public IdentifiableStore createGroupStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage) { return AUTH_GROUP.identifiable(new NonPersistentStore(), centralRegistry); } From 7c856221aa1537328ed5c19893acd6abed9d65f8 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Thu, 9 Sep 2021 16:18:46 +0200 Subject: [PATCH 08/35] capsule storage for group and role handling --- .../models/auth/AuthorizationController.java | 2 +- .../models/auth/AuthorizationHelper.java | 52 ++----------------- .../auth/ConqueryAuthorizationRealm.java | 2 +- .../conquery/models/auth/entities/Group.java | 11 ++++ .../models/auth/entities/PermissionOwner.java | 2 + .../conquery/models/auth/entities/Role.java | 8 +++ .../conquery/models/auth/entities/User.java | 22 ++++++++ .../conquery/models/auth/CopyUserTest.java | 2 +- 8 files changed, 51 insertions(+), 50 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationController.java b/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationController.java index 8ba051408d..cbcf402e0f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationController.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationController.java @@ -174,7 +174,7 @@ public static User flatCopyUser(@NonNull User originUser, String namePrefix, @No // Copy inherited permissions Set copiedPermission = new HashSet<>(); - copiedPermission.addAll(AuthorizationHelper.getEffectiveUserPermissions(originUser, storage)); + copiedPermission.addAll(originUser.getEffectivePermissions()); // Give read permission to all executions the original user owned copiedPermission.addAll( diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationHelper.java b/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationHelper.java index a0fa099ef7..39716545f6 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationHelper.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationHelper.java @@ -65,61 +65,19 @@ public static Optional getPrimaryGroup(@NonNull User user, @NonNull MetaS return Optional.of(groups.get(0)); } - - - /** - * Returns a list of the effective permissions. These are the permissions of the owner and - * the permission of the roles it inherits. - * @return Owned and inherited permissions. - */ - public static Set getEffectiveUserPermissions(User user, MetaStorage storage) { - Set tmpView = collectRolePermissions(storage, user, user.getPermissions()); - - for (Group group : storage.getAllGroups()) { - if (group.containsMember(user)) { - // Get effective permissions of the group - tmpView = Sets.union(tmpView, getEffectiveGroupPermissions(group, storage)); - } - } - return tmpView; - } - - /** - * Returns a list of the effective permissions. These are the permissions of the owner and - * the permission of the roles it inherits. - */ - public static Set getEffectiveGroupPermissions(Group group, MetaStorage storage) { - // Combine permissions held by the group with those inherited from roles - return collectRolePermissions(storage, group, group.getPermissions()); - } - - private static Set collectRolePermissions(MetaStorage storage, RoleOwner roleOwner, Set tmpView) { - for (RoleId roleId : roleOwner.getRoles()) { - Role role = storage.getRole(roleId); - if (role == null) { - log.warn("Could not resolve role id [{}]", roleId); - continue; - } - tmpView = Sets.union(tmpView, role.getPermissions()); - } - return tmpView; - } - - /** * Returns a list of the effective permissions. These are the permissions of the owner and * the permission of the roles it inherits. The query can be filtered by the Permission domain. * @return Owned and inherited permissions. */ public static Multimap getEffectiveUserPermissions(User user, List domainSpecifier, MetaStorage storage) { - Set permissions = getEffectiveUserPermissions(user, storage); + Set permissions = user.getEffectivePermissions(); Multimap mappedPerms = ArrayListMultimap.create(); - for(Permission perm : permissions) { - ConqueryPermission cPerm = (ConqueryPermission) perm; - Set domains = cPerm.getDomains(); - if(!Collections.disjoint(domainSpecifier, cPerm.getDomains())) { + for(ConqueryPermission perm : permissions) { + Set domains = perm.getDomains(); + if(!Collections.disjoint(domainSpecifier, perm.getDomains())) { for(String domain : domains) { - mappedPerms.put(domain, cPerm); + mappedPerms.put(domain, perm); } } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/ConqueryAuthorizationRealm.java b/backend/src/main/java/com/bakdata/conquery/models/auth/ConqueryAuthorizationRealm.java index f5bb06ca7d..c660693916 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/ConqueryAuthorizationRealm.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/ConqueryAuthorizationRealm.java @@ -45,7 +45,7 @@ protected AuthorizationInfo doGetAuthorizationInfo(PrincipalCollection principal UserId userId = (UserId) principals.getPrimaryPrincipal(); SimpleAuthorizationInfo info = new ConqueryAuthorizationInfo(); - info.addObjectPermissions(Collections.unmodifiableSet(AuthorizationHelper.getEffectiveUserPermissions(storage.getUser(userId), storage))); + info.addObjectPermissions(Collections.unmodifiableSet(storage.getUser(userId).getEffectivePermissions())); return info; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Group.java b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Group.java index b0d95b3e19..ba429d0ed7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Group.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Group.java @@ -5,10 +5,12 @@ import java.util.Set; import com.bakdata.conquery.io.storage.MetaStorage; +import com.bakdata.conquery.models.auth.permissions.ConqueryPermission; import com.bakdata.conquery.models.identifiable.ids.specific.GroupId; import com.bakdata.conquery.models.identifiable.ids.specific.RoleId; import com.bakdata.conquery.models.identifiable.ids.specific.UserId; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.Sets; import lombok.extern.slf4j.Slf4j; /** @@ -29,6 +31,15 @@ public Group(String name, String label, MetaStorage storage) { super(name, label, storage); } + @Override + public Set getEffectivePermissions() { + Set permissions = getPermissions(); + for (RoleId roleId : roles) { + permissions = Sets.union(permissions,storage.getRole(roleId).getEffectivePermissions()); + } + return permissions; + } + @Override protected void updateStorage(MetaStorage storage) { storage.updateGroup(this); diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/PermissionOwner.java b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/PermissionOwner.java index 052957a59b..9b251ff230 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/PermissionOwner.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/PermissionOwner.java @@ -140,6 +140,8 @@ public Set getPermissions() { } + public abstract Set getEffectivePermissions(); + public void updatePermissions(Set permissionsNew) { synchronized (this) { Set newSet = new HashSet<>(permissionsNew.size()); diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Role.java b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Role.java index 1f73a79c24..1386ab4b41 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Role.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Role.java @@ -1,6 +1,9 @@ package com.bakdata.conquery.models.auth.entities; +import java.util.Set; + import com.bakdata.conquery.io.storage.MetaStorage; +import com.bakdata.conquery.models.auth.permissions.ConqueryPermission; import com.bakdata.conquery.models.identifiable.ids.specific.RoleId; public class Role extends PermissionOwner { @@ -10,6 +13,11 @@ public Role(String name, String label, MetaStorage storage) { super(name, label, storage); } + @Override + public Set getEffectivePermissions() { + return getPermissions(); + } + @Override public RoleId createId() { return new RoleId(name); diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/User.java b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/User.java index 7d5485923c..15ad195987 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/User.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/User.java @@ -12,12 +12,14 @@ import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.models.auth.permissions.Ability; import com.bakdata.conquery.models.auth.permissions.Authorized; +import com.bakdata.conquery.models.auth.permissions.ConqueryPermission; import com.bakdata.conquery.models.auth.util.SinglePrincipalCollection; import com.bakdata.conquery.models.execution.Owned; import com.bakdata.conquery.models.identifiable.ids.specific.RoleId; import com.bakdata.conquery.models.identifiable.ids.specific.UserId; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.Sets; import it.unimi.dsi.fastutil.booleans.BooleanArrayList; import lombok.AccessLevel; import lombok.Getter; @@ -50,6 +52,26 @@ public User(String name, String label, MetaStorage storage) { this.shiroUserAdapter = new ShiroUserAdapter(); } + @Override + public Set getEffectivePermissions() { + Set permissions = getPermissions(); + for (RoleId roleId : roles) { + Role role = storage.getRole(roleId); + if (role == null) { + log.warn("Could not find role {} to gather permissions", roleId); + continue; + } + permissions = Sets.union(permissions, role.getEffectivePermissions()); + } + for (Group group : storage.getAllGroups()) { + if (!group.containsMember(this)) { + continue; + } + permissions = Sets.union(permissions, group.getEffectivePermissions()); + } + return permissions; + } + @Override public UserId createId() { return new UserId(name); diff --git a/backend/src/test/java/com/bakdata/conquery/models/auth/CopyUserTest.java b/backend/src/test/java/com/bakdata/conquery/models/auth/CopyUserTest.java index ca96c6de74..2cf97d5d25 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/auth/CopyUserTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/auth/CopyUserTest.java @@ -51,7 +51,7 @@ void testUserCopy(){ assertThat(copy.getRoles()).isEmpty(); // Check that the flat map worked - assertThat(copy.getPermissions()).containsExactlyInAnyOrderElementsOf(AuthorizationHelper.getEffectiveUserPermissions(originUser,storage)); + assertThat(copy.getPermissions()).containsExactlyInAnyOrderElementsOf(originUser.getEffectivePermissions()); } } From c05d178bbb87469ec37639a69b22ace74ebf30ef Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 22 Sep 2021 14:14:36 +0200 Subject: [PATCH 09/35] fix serialization --- .../bakdata/conquery/models/auth/entities/PermissionOwner.java | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/PermissionOwner.java b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/PermissionOwner.java index 9b251ff230..8a2600fda1 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/PermissionOwner.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/PermissionOwner.java @@ -140,6 +140,7 @@ public Set getPermissions() { } + @JsonIgnore public abstract Set getEffectivePermissions(); public void updatePermissions(Set permissionsNew) { From ba83a4704c00b6b15c6225c44ae73325fef7948b Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 22 Sep 2021 14:46:10 +0200 Subject: [PATCH 10/35] check for all stores in a namespaces storage --- .../models/config/XodusStoreFactory.java | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java index 753908b1b0..5f87112fcb 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java @@ -11,6 +11,9 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.validation.Valid; import javax.validation.Validator; @@ -83,6 +86,16 @@ @CPSType(id = "XODUS", base = StoreFactory.class) public class XodusStoreFactory implements StoreFactory { + public static final Supplier> NAMESPACED_STORES = () -> Stream.of(DATASET, SECONDARY_IDS, TABLES, DICTIONARIES, IMPORTS, CONCEPTS); + public static final List NAMESPACE_STORES = Stream.concat(NAMESPACED_STORES.get(), Stream.of(ID_MAPPING, STRUCTURE, WORKER_TO_BUCKETS, PRIMARY_DICTIONARY)) + .map(StoreInfo::storeInfo) + .map(IStoreInfo::getName) + .collect(Collectors.toList()); + public static final List WORKER_STORES = Stream.concat(NAMESPACED_STORES.get(), Stream.of(WORKER, BUCKETS, C_BLOCKS)) + .map(StoreInfo::storeInfo) + .map(IStoreInfo::getName) + .collect(Collectors.toList()); + private Path directory = Path.of("storage"); private boolean validateOnWrite = false; @@ -142,17 +155,17 @@ private void configureMapper(ConqueryConfig config) { @Override @SneakyThrows public Collection loadNamespaceStorages() { - return loadNamespacedStores("dataset_", (elements) -> new NamespaceStorage(validator, this, elements)); + return loadNamespacedStores("dataset_", (elements) -> new NamespaceStorage(validator, this, elements), NAMESPACE_STORES); } @Override @SneakyThrows public Collection loadWorkerStorages() { - return loadNamespacedStores("worker_", (elements) -> new WorkerStorage(validator, this, elements)); + return loadNamespacedStores("worker_", (elements) -> new WorkerStorage(validator, this, elements), WORKER_STORES); } - private Queue loadNamespacedStores(String prefix, Function creator) + private Queue loadNamespacedStores(String prefix, Function creator, List storesToTest) throws InterruptedException { File baseDir = getDirectory().toFile(); @@ -172,7 +185,7 @@ private Queue loadNamespacedStores(String prefi try { ConqueryMDC.setLocation(directory.toString()); - if (!environmentHasStores(directory)) { + if (!environmentHasStores(directory, storesToTest)) { log.warn("No valid WorkerStorage found."); return; } @@ -205,9 +218,9 @@ private Queue loadNamespacedStores(String prefi return storages; } - private boolean environmentHasStores(File pathName) { + private boolean environmentHasStores(File pathName, List storesToTest) { Environment env = findEnvironment(pathName); - boolean exists = env.computeInTransaction(t -> env.storeExists(StoreInfo.DATASET.storeInfo().getName(), t)); + boolean exists = env.computeInTransaction(t -> env.getAllStoreNames(t).containsAll(storesToTest)); env.computeInTransaction(env::getAllStoreNames); if (!exists) { closeEnvironment(env); From 71ffe95dd1940e2c675e0e93f7843eef26e8dfa2 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 22 Sep 2021 15:34:30 +0200 Subject: [PATCH 11/35] check namespaced storages for stores --- .../io/storage/xodus/stores/BigStore.java | 6 +- .../models/config/XodusStoreFactory.java | 63 ++++++++++++++----- 2 files changed, 52 insertions(+), 17 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/BigStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/BigStore.java index b37e9d0e19..2082dc5936 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/BigStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/BigStore.java @@ -43,6 +43,8 @@ @Getter public class BigStore implements Store, Closeable { + public static final String META = "_META"; + public static final String DATA = "_DATA"; private final SerializingStore metaStore; private final SerializingStore dataStore; private final ObjectWriter valueWriter; @@ -61,7 +63,7 @@ public BigStore(XodusStoreFactory config, Validator validator, Environment env, this.chunkByteSize = Ints.checkedCast(config.getXodus().getLogFileSize().toBytes() / 4L); metaStore = new SerializingStore<>( - new XodusStore(env, storeInfo.getName() + "_META", openStores, envCloseHook, envRemoveHook), + new XodusStore(env, storeInfo.getName() + META, openStores, envCloseHook, envRemoveHook), validator, mapper, storeInfo.getKeyType(), @@ -72,7 +74,7 @@ public BigStore(XodusStoreFactory config, Validator validator, Environment env, ); dataStore = new SerializingStore<>( - new XodusStore(env, storeInfo.getName() + "_DATA", openStores, envCloseHook, envRemoveHook), + new XodusStore(env, storeInfo.getName() + DATA, openStores, envCloseHook, envRemoveHook), validator, mapper, UUID.class, diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java index 5f87112fcb..437417aa27 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java @@ -63,6 +63,7 @@ import com.google.common.collect.Multimap; import com.google.common.collect.MultimapBuilder; import com.google.common.collect.Multimaps; +import com.google.common.collect.Sets; import io.dropwizard.util.Duration; import jetbrains.exodus.env.Environment; import jetbrains.exodus.env.Environments; @@ -86,15 +87,35 @@ @CPSType(id = "XODUS", base = StoreFactory.class) public class XodusStoreFactory implements StoreFactory { - public static final Supplier> NAMESPACED_STORES = () -> Stream.of(DATASET, SECONDARY_IDS, TABLES, DICTIONARIES, IMPORTS, CONCEPTS); - public static final List NAMESPACE_STORES = Stream.concat(NAMESPACED_STORES.get(), Stream.of(ID_MAPPING, STRUCTURE, WORKER_TO_BUCKETS, PRIMARY_DICTIONARY)) - .map(StoreInfo::storeInfo) - .map(IStoreInfo::getName) - .collect(Collectors.toList()); - public static final List WORKER_STORES = Stream.concat(NAMESPACED_STORES.get(), Stream.of(WORKER, BUCKETS, C_BLOCKS)) - .map(StoreInfo::storeInfo) - .map(IStoreInfo::getName) - .collect(Collectors.toList()); + public static final Supplier> NAMESPACED_STORES = () -> Stream.of( + DATASET.storeInfo().getName(), + SECONDARY_IDS.storeInfo().getName(), + TABLES.storeInfo().getName(), + DICTIONARIES.storeInfo().getName() + BigStore.META, + DICTIONARIES.storeInfo().getName() + BigStore.DATA, + IMPORTS.storeInfo().getName(), + CONCEPTS.storeInfo().getName() + ); + public static final Set NAMESPACE_STORES = Stream.concat( + NAMESPACED_STORES.get(), + Stream.of( + ID_MAPPING.storeInfo().getName() + BigStore.META, + ID_MAPPING.storeInfo().getName() + BigStore.DATA, + STRUCTURE.storeInfo().getName(), + WORKER_TO_BUCKETS.storeInfo().getName(), + PRIMARY_DICTIONARY.storeInfo().getName() + ) + ) + .collect(Collectors.toUnmodifiableSet()); + public static final Set WORKER_STORES = Stream.concat( + NAMESPACED_STORES.get(), + Stream.of( + WORKER.storeInfo().getName(), + BUCKETS.storeInfo().getName(), + C_BLOCKS.storeInfo().getName() + ) + ) + .collect(Collectors.toUnmodifiableSet()); private Path directory = Path.of("storage"); @@ -165,7 +186,7 @@ public Collection loadWorkerStorages() { } - private Queue loadNamespacedStores(String prefix, Function creator, List storesToTest) + private Queue loadNamespacedStores(String prefix, Function creator, Set storesToTest) throws InterruptedException { File baseDir = getDirectory().toFile(); @@ -190,11 +211,11 @@ private Queue loadNamespacedStores(String prefi return; } - T workerStorage = creator.apply(name); + T namespacedStorage = creator.apply(name); log.debug("BEGIN reading Storage"); - workerStorage.loadData(); + namespacedStorage.loadData(); - storages.add(workerStorage); + storages.add(namespacedStorage); } catch (Exception e) { @@ -218,9 +239,21 @@ private Queue loadNamespacedStores(String prefi return storages; } - private boolean environmentHasStores(File pathName, List storesToTest) { + private boolean environmentHasStores(File pathName, Set storesToTest) { Environment env = findEnvironment(pathName); - boolean exists = env.computeInTransaction(t -> env.getAllStoreNames(t).containsAll(storesToTest)); + boolean exists = env.computeInTransaction(t -> { + final List allStoreNames = env.getAllStoreNames(t); + final boolean complete = allStoreNames.containsAll(storesToTest); + if(complete) { + log.trace("Storage contained all stores: {}", storesToTest); + return true; + } + + final HashSet missing = Sets.newHashSet(storesToTest); + missing.removeAll(allStoreNames); + log.warn("Storage did not contain all required stores. It is missing: {}. It had {}", missing, allStoreNames); + return false; + }); env.computeInTransaction(env::getAllStoreNames); if (!exists) { closeEnvironment(env); From 0ad7a65f59a8d80bc59cbb7dc0e1e6c17e1b514e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 22 Sep 2021 14:21:56 +0000 Subject: [PATCH 12/35] Update AutoDoc --- docs/Config JSON.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/Config JSON.md b/docs/Config JSON.md index e1deaf8117..167f384a25 100644 --- a/docs/Config JSON.md +++ b/docs/Config JSON.md @@ -406,7 +406,7 @@ Supported Fields: | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusConfig.java#L51) | treeMaxPageSize | `int` or `null` | `null` | | |

-### Type XodusStoreFactory [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L84) +### Type XodusStoreFactory [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L93)
Details

@@ -417,12 +417,12 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L94) | directory | `Path` | `"file://./storage"` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L105) | nThreads | `@javax.validation.constraints.Min(1) int` | ␀ | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L108-L110) | removeUnreadableFromStore | `boolean` | `false` | | Flag for the {@link SerializingStore} whether to delete values from the underlying store, that cannot be mapped to an object anymore. | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L113-L115) | unreadableDataDumpDirectory | `File` | `null` | | When set, all values that could not be deserialized from the persistent store, are dump into individual files. | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L101) | useWeakDictionaryCaching | `boolean` | `true` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L96) | validateOnWrite | `boolean` | `false` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L102) | weakCacheDuration | `@NotNull Duration` | `"48 hours"` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L97) | xodus | [XodusConfig](#Type-XodusConfig) | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L133) | directory | `Path` | `"file://./storage"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L144) | nThreads | `@javax.validation.constraints.Min(1) int` | ␀ | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L147-L149) | removeUnreadableFromStore | `boolean` | `false` | | Flag for the {@link SerializingStore} whether to delete values from the underlying store, that cannot be mapped to an object anymore. | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L152-L154) | unreadableDataDumpDirectory | `File` | `null` | | When set, all values that could not be deserialized from the persistent store, are dump into individual files. | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L140) | useWeakDictionaryCaching | `boolean` | `true` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L135) | validateOnWrite | `boolean` | `false` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L141) | weakCacheDuration | `@NotNull Duration` | `"48 hours"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L136) | xodus | [XodusConfig](#Type-XodusConfig) | | | |

From 64c0e4698b0097b06e9ac8e403d70a9d1eab8cc0 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 22 Sep 2021 16:49:56 +0200 Subject: [PATCH 13/35] review changes --- .../bakdata/conquery/apiv1/QueryProcessor.java | 2 +- .../bakdata/conquery/apiv1/auth/ProtoUser.java | 17 +++++++++-------- .../models/auth/AuthorizationController.java | 12 +++++------- .../UserAuthenticationManagementProcessor.java | 14 +++++++------- .../models/auth/develop/DevAuthConfig.java | 14 ++++++++++---- .../conquery/models/auth/entities/Group.java | 10 +++++----- .../models/auth/entities/PermissionOwner.java | 12 ++++++------ .../conquery/models/auth/entities/Role.java | 2 +- .../conquery/models/auth/entities/User.java | 6 +++--- .../models/config/XodusStoreFactory.java | 10 +++++----- .../UserAuthenticationManagementResource.java | 2 +- .../conquery/util/support/TestConquery.java | 4 +--- .../util/support/TestLoggingFactory.java | 1 - 13 files changed, 54 insertions(+), 52 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java index c2ed5802a9..c0fcd59e52 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java @@ -112,7 +112,7 @@ public ManagedExecution postQuery(Dataset dataset, QueryDescription query, Us { final Optional executionId = visitors.getInstance(QueryUtils.OnlyReusingChecker.class).getOnlyReused(); - final Optional> execution = executionId.map(id -> tryReuse(query, id, datasetRegistry, config, executionManager, storage.getUser(user.getId()))); + final Optional> execution = executionId.map(id -> tryReuse(query, id, datasetRegistry, config, executionManager, user)); if (execution.isPresent()) { return execution.get(); diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/auth/ProtoUser.java b/backend/src/main/java/com/bakdata/conquery/apiv1/auth/ProtoUser.java index b8d343f65c..b491bd2c16 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/auth/ProtoUser.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/auth/ProtoUser.java @@ -48,22 +48,23 @@ public class ProtoUser { @Valid private List credentials = Collections.emptyList(); - public Optional getUser(@NonNull MetaStorage storage, boolean override) { - User user = storage.getUser(new UserId(name)); - if (!override) { - return Optional.ofNullable(user); - } + public User createOrOverwriteUser(@NonNull MetaStorage storage) { if (label == null) { label = name; } - user = new User(name, label, storage); + User user = new User(name, label, storage); storage.updateUser(user); for (String sPermission : permissions) { user.addPermission(new WildcardPermission(sPermission)); } - return Optional.of(user); + return user; + } + + @org.jetbrains.annotations.NotNull + public UserId getId() { + return new UserId(name); } - + public static boolean registerForAuthentication(UserManageable userManager, User user, List credentials, boolean override) { if(override) { return userManager.updateUser(user, credentials); diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationController.java b/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationController.java index cbcf402e0f..a702bef3ba 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationController.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationController.java @@ -137,14 +137,12 @@ public void registerStaticSecurityManager() { */ private static void initializeAuthConstellation(@NonNull AuthorizationConfig config, @NonNull List realms, @NonNull MetaStorage storage) { for (ProtoUser pUser : config.getInitialUsers()) { - final Optional user = pUser.getUser(storage, true); - user.ifPresent(u -> { - for (Realm realm : realms) { - if (realm instanceof UserManageable) { - ProtoUser.registerForAuthentication((UserManageable) realm, u, pUser.getCredentials(), true); - } + final User user = pUser.createOrOverwriteUser(storage); + for (Realm realm : realms) { + if (realm instanceof UserManageable) { + ProtoUser.registerForAuthentication((UserManageable) realm, user, pUser.getCredentials(), true); } - }); + } } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/basic/UserAuthenticationManagementProcessor.java b/backend/src/main/java/com/bakdata/conquery/models/auth/basic/UserAuthenticationManagementProcessor.java index d326fc3d33..156d74a233 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/basic/UserAuthenticationManagementProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/basic/UserAuthenticationManagementProcessor.java @@ -20,15 +20,14 @@ public class UserAuthenticationManagementProcessor { private final LocalAuthenticationRealm realm; private final MetaStorage storage; - public boolean addUser(ProtoUser pUser) { + public boolean tryRegister(ProtoUser pUser) { // Throws an exception if it would override the existing user - final Optional optUser = pUser.getUser(storage,false); - if (optUser.isEmpty()){ + final UserId id = pUser.getId(); + User user = storage.getUser(id); + if (user == null) { log.warn("Unable to add new user {}. Probably already existed.", pUser); return false; } - final User user = optUser.get(); - final UserId id = user.getId(); log.trace("Added the user {} to the authorization storage", id); if(ProtoUser.registerForAuthentication(realm, user, pUser.getCredentials(),false)) { log.trace("Added the user {} to the realm {}", id, realm.getName()); @@ -39,8 +38,9 @@ public boolean addUser(ProtoUser pUser) { } public boolean updateUser(ProtoUser pUser) { - final Optional user = pUser.getUser(storage, true); - return user.map(u -> ProtoUser.registerForAuthentication(realm, u,pUser.getCredentials(),false)).orElse(false); + final User user = pUser.createOrOverwriteUser(storage); + ProtoUser.registerForAuthentication(realm, user,pUser.getCredentials(),false); + return true; } public void remove(User user) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/develop/DevAuthConfig.java b/backend/src/main/java/com/bakdata/conquery/models/auth/develop/DevAuthConfig.java index 26899fedda..058bc15801 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/develop/DevAuthConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/develop/DevAuthConfig.java @@ -1,12 +1,14 @@ package com.bakdata.conquery.models.auth.develop; +import java.util.List; + +import com.bakdata.conquery.apiv1.auth.ProtoUser; import com.bakdata.conquery.commands.ManagerNode; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.config.auth.AuthenticationRealmFactory; import com.bakdata.conquery.models.auth.ConqueryAuthenticationRealm; import com.bakdata.conquery.models.auth.entities.User; - -import java.util.Objects; +import com.google.common.base.Preconditions; /** * Default configuration for the auth system. Sets up all other default components. @@ -17,8 +19,12 @@ public class DevAuthConfig implements AuthenticationRealmFactory { @Override public ConqueryAuthenticationRealm createRealm(ManagerNode managerNode) { - User defaultUser = managerNode.getConfig() - .getAuthorizationRealms().getInitialUsers().get(0).getUser(managerNode.getStorage(), true).orElseThrow(() -> new IllegalStateException("There must be at least one initial user configured.")); + final List initialUsers = managerNode.getConfig() + .getAuthorizationRealms().getInitialUsers(); + + Preconditions.checkState(initialUsers.size()>0, "There must be at least one initial user configured."); + + User defaultUser = initialUsers.get(0).createOrOverwriteUser(managerNode.getStorage()); managerNode.getAuthController().getAuthenticationFilter().registerTokenExtractor(new UserIdTokenExtractor(defaultUser)); diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Group.java b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Group.java index ba429d0ed7..ecbbcd4b73 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Group.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Group.java @@ -41,7 +41,7 @@ public Set getEffectivePermissions() { } @Override - protected void updateStorage(MetaStorage storage) { + protected void updateStorage() { storage.updateGroup(this); } @@ -53,14 +53,14 @@ public GroupId createId() { public synchronized void addMember(User user) { if(members.add(user.getId())) { log.trace("Added user {} to group {}", user.getId(), getId()); - updateStorage(storage); + updateStorage(); } } public synchronized void removeMember(User user) { if (members.remove(user.getId())) { log.trace("Removed user {} from group {}", user.getId(), getId()); - updateStorage(storage); + updateStorage(); } } @@ -75,14 +75,14 @@ public Set getMembers() { public synchronized void addRole(Role role) { if (roles.add(role.getId())) { log.trace("Added role {} to group {}", role.getId(), getId()); - updateStorage(storage); + updateStorage(); } } public synchronized void removeRole(Role role) { if (roles.remove(role.getId())) { log.trace("Removed role {} from group {}", role.getId(), getId()); - updateStorage(storage); + updateStorage(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/PermissionOwner.java b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/PermissionOwner.java index 8a2600fda1..46118c5934 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/PermissionOwner.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/PermissionOwner.java @@ -85,7 +85,7 @@ public synchronized void addPermissions(Set permissions) { .addAll(this.permissions) .addAll(permissions) .build(); - updateStorage(this.storage); + updateStorage(); } public synchronized void addPermission(ConqueryPermission permission) { @@ -94,7 +94,7 @@ public synchronized void addPermission(ConqueryPermission permission) { .addAll(this.permissions) .add(permission) .build(); - updateStorage(this.storage); + updateStorage(); } /** @@ -109,7 +109,7 @@ public boolean removePermissions(Set permissions) { Set newSet = new HashSet<>(this.permissions); ret = newSet.removeAll(permissions); this.permissions = newSet; - updateStorage(this.storage); + updateStorage(); } return ret; } @@ -120,7 +120,7 @@ public boolean removePermission(Permission permission) { Set newSet = new HashSet<>(this.permissions); ret = newSet.remove(permission); this.permissions = newSet; - updateStorage(this.storage); + updateStorage(); } return ret; } @@ -148,14 +148,14 @@ public void updatePermissions(Set permissionsNew) { Set newSet = new HashSet<>(permissionsNew.size()); newSet.addAll(permissionsNew); this.permissions = newSet; - updateStorage(this.storage); + updateStorage(); } } /** * Update this instance in the {@link MetaStorage}. */ - protected abstract void updateStorage(MetaStorage storage); + protected abstract void updateStorage(); @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Role.java b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Role.java index 1386ab4b41..3783c93369 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Role.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/Role.java @@ -24,7 +24,7 @@ public RoleId createId() { } @Override - protected void updateStorage(MetaStorage storage) { + protected void updateStorage() { storage.updateRole(this); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/User.java b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/User.java index 15ad195987..5d1c84dc22 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/entities/User.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/entities/User.java @@ -80,7 +80,7 @@ public UserId createId() { public synchronized void addRole(Role role) { if (roles.add(role.getId())) { log.trace("Added role {} to user {}", role.getId(), getId()); - updateStorage(storage); + updateStorage(); } } @@ -88,7 +88,7 @@ public synchronized void addRole(Role role) { public synchronized void removeRole(Role role) { if (roles.remove(role.getId())) { log.trace("Removed role {} from user {}", role.getId(), getId()); - updateStorage(storage); + updateStorage(); } } @@ -97,7 +97,7 @@ public Set getRoles() { } @Override - protected void updateStorage(MetaStorage storage) { + protected void updateStorage() { storage.updateUser(this); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java index 82283d8fba..acf0d59532 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java @@ -262,13 +262,13 @@ private boolean environmentHasStores(File pathName, Set storesToTest) { log.trace("Storage contained all stores: {}", storesToTest); return true; } - - final HashSet missing = Sets.newHashSet(storesToTest); - missing.removeAll(allStoreNames); - log.warn("Storage did not contain all required stores. It is missing: {}. It had {}", missing, allStoreNames); + if(log.isWarnEnabled()) { + final HashSet missing = Sets.newHashSet(storesToTest); + allStoreNames.forEach(missing::remove); + log.warn("Storage did not contain all required stores. It is missing: {}. It had {}", missing, allStoreNames); + } return false; }); - env.computeInTransaction(env::getAllStoreNames); if (!exists) { closeEnvironment(env); } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/UserAuthenticationManagementResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/UserAuthenticationManagementResource.java index be66e3d03d..81fd293903 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/UserAuthenticationManagementResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/UserAuthenticationManagementResource.java @@ -28,7 +28,7 @@ public class UserAuthenticationManagementResource extends HAuthorized{ @Consumes(MediaType.APPLICATION_JSON) public Response addUser(ProtoUser pUser) { - if(processor.addUser(pUser)) { + if(processor.tryRegister(pUser)) { return Response.status(Status.CREATED).build(); } return Response.serverError().status(Status.CONFLICT).build(); diff --git a/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java b/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java index b85a4983b9..33bdc3dd69 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java +++ b/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java @@ -20,8 +20,6 @@ import com.bakdata.conquery.commands.ShardNode; import com.bakdata.conquery.commands.StandaloneCommand; import com.bakdata.conquery.integration.IntegrationTests; -import com.bakdata.conquery.integration.json.JsonIntegrationTest; -import com.bakdata.conquery.io.jackson.Jackson; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.config.ConqueryConfig; @@ -289,7 +287,7 @@ private boolean isBusy() { public void beforeEach() { final MetaStorage storage = standaloneCommand.getManager().getStorage(); - testUser = standaloneCommand.getManager().getConfig().getAuthorizationRealms().getInitialUsers().get(0).getUser(storage, true).orElseThrow(); + testUser = standaloneCommand.getManager().getConfig().getAuthorizationRealms().getInitialUsers().get(0).createOrOverwriteUser(storage); storage.updateUser(testUser); } } diff --git a/backend/src/test/java/com/bakdata/conquery/util/support/TestLoggingFactory.java b/backend/src/test/java/com/bakdata/conquery/util/support/TestLoggingFactory.java index 983494c0d7..7e13bda4da 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/support/TestLoggingFactory.java +++ b/backend/src/test/java/com/bakdata/conquery/util/support/TestLoggingFactory.java @@ -176,7 +176,6 @@ private Logger configureLoggers() { root.setLevel(Level.WARN); loggerContext.getLogger("com.bakdata").setLevel(Level.DEBUG); - loggerContext.getLogger("com.bakdata.conquery.io.storage.xodus.stores").setLevel(Level.DEBUG); return root; } From 7f9e9427005f679f7f2be5ce08847e96daa04225 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 22 Sep 2021 14:52:29 +0000 Subject: [PATCH 14/35] Update AutoDoc --- docs/Config JSON.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Config JSON.md b/docs/Config JSON.md index 167f384a25..e340e80922 100644 --- a/docs/Config JSON.md +++ b/docs/Config JSON.md @@ -47,7 +47,7 @@ An `AuthenticationConfig` is used to define how specific realms for authenticati Different types of AuthenticationRealmFactory can be used by setting `type` to one of the following values: -### DEVELOPMENT [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/auth/develop/DevAuthConfig.java#L11-L14) +### DEVELOPMENT [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/auth/develop/DevAuthConfig.java#L13-L16) Default configuration for the auth system. Sets up all other default components. This configuration causes that every request is handled as invoked by the super user.
Details

From abfeff4eec42d61418eac4aaa5fbada3c6e32527 Mon Sep 17 00:00:00 2001 From: MT <12283268+thoniTUB@users.noreply.github.com> Date: Thu, 23 Sep 2021 09:25:59 +0200 Subject: [PATCH 15/35] Update backend/src/main/java/com/bakdata/conquery/apiv1/auth/ProtoUser.java Co-authored-by: awildturtok <1553491+awildturtok@users.noreply.github.com> --- .../main/java/com/bakdata/conquery/apiv1/auth/ProtoUser.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/auth/ProtoUser.java b/backend/src/main/java/com/bakdata/conquery/apiv1/auth/ProtoUser.java index b491bd2c16..ab77bfaa3c 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/auth/ProtoUser.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/auth/ProtoUser.java @@ -20,7 +20,8 @@ import lombok.NonNull; /** - * Container class for holding information about initial users. + * Factory class to create configured initial users. + */ @Getter @Builder From 66274e87e3dafac8e9f3a19a1b837745ec8473d8 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Thu, 23 Sep 2021 09:30:03 +0200 Subject: [PATCH 16/35] move auth registration method to AuthorizationHelper.java --- .../java/com/bakdata/conquery/apiv1/QueryProcessor.java | 4 ++-- .../java/com/bakdata/conquery/apiv1/auth/ProtoUser.java | 8 -------- .../conquery/models/auth/AuthorizationController.java | 3 ++- .../conquery/models/auth/AuthorizationHelper.java | 9 +++++++++ .../basic/UserAuthenticationManagementProcessor.java | 6 ++++-- .../bakdata/conquery/api/form/config/FormConfigTest.java | 2 +- 6 files changed, 18 insertions(+), 14 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java index c0fcd59e52..5e43bc644a 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java @@ -121,7 +121,7 @@ public ManagedExecution postQuery(Dataset dataset, QueryDescription query, Us // Run the query on behalf of the user - ManagedExecution mq = executionManager.runQuery(datasetRegistry, query, storage.getUser(user.getId()), dataset, config); + ManagedExecution mq = executionManager.runQuery(datasetRegistry, query, user, dataset, config); if (query instanceof Query) { translateToOtherDatasets(dataset, query, user, mq); @@ -396,7 +396,7 @@ public ExternalUploadResult uploadEntities(User user, Dataset dataset, ExternalU // We only create the Query, really no need to execute it as it's only useful for composition. final ManagedQuery execution = ((ManagedQuery) datasetRegistry.get(dataset.getId()).getExecutionManager() - .createExecution(datasetRegistry, query, storage.getUser(user.getId()), dataset)); + .createExecution(datasetRegistry, query, user, dataset)); execution.setLastResultCount((long) statistic.getResolved().size()); diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/auth/ProtoUser.java b/backend/src/main/java/com/bakdata/conquery/apiv1/auth/ProtoUser.java index ab77bfaa3c..734d3bd0c9 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/auth/ProtoUser.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/auth/ProtoUser.java @@ -61,15 +61,7 @@ public User createOrOverwriteUser(@NonNull MetaStorage storage) { return user; } - @org.jetbrains.annotations.NotNull public UserId getId() { return new UserId(name); } - - public static boolean registerForAuthentication(UserManageable userManager, User user, List credentials, boolean override) { - if(override) { - return userManager.updateUser(user, credentials); - } - return userManager.addUser(user, credentials); - } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationController.java b/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationController.java index a702bef3ba..0c0b17836e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationController.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationController.java @@ -3,6 +3,7 @@ import java.util.*; import java.util.stream.Collectors; +import com.bakdata.conquery.apiv1.auth.CredentialType; import com.bakdata.conquery.apiv1.auth.ProtoUser; import com.bakdata.conquery.commands.ManagerNode; import com.bakdata.conquery.io.storage.MetaStorage; @@ -140,7 +141,7 @@ private static void initializeAuthConstellation(@NonNull AuthorizationConfig con final User user = pUser.createOrOverwriteUser(storage); for (Realm realm : realms) { if (realm instanceof UserManageable) { - ProtoUser.registerForAuthentication((UserManageable) realm, user, pUser.getCredentials(), true); + AuthorizationHelper.registerForAuthentication((UserManageable) realm, user, pUser.getCredentials(), true); } } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationHelper.java b/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationHelper.java index 39716545f6..736f9c5d7d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationHelper.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationHelper.java @@ -10,6 +10,7 @@ import java.util.Set; import java.util.stream.Collectors; +import com.bakdata.conquery.apiv1.auth.CredentialType; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.models.auth.entities.Group; import com.bakdata.conquery.models.auth.entities.Role; @@ -150,4 +151,12 @@ public static Map> buildDatasetAbilityMap(User user, Dat } return datasetAbilities; } + + + public static boolean registerForAuthentication(UserManageable userManager, User user, List credentials, boolean override) { + if(override) { + return userManager.updateUser(user, credentials); + } + return userManager.addUser(user, credentials); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/basic/UserAuthenticationManagementProcessor.java b/backend/src/main/java/com/bakdata/conquery/models/auth/basic/UserAuthenticationManagementProcessor.java index 156d74a233..88e2699219 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/basic/UserAuthenticationManagementProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/basic/UserAuthenticationManagementProcessor.java @@ -2,6 +2,8 @@ import com.bakdata.conquery.apiv1.auth.ProtoUser; import com.bakdata.conquery.io.storage.MetaStorage; +import com.bakdata.conquery.models.auth.AuthorizationController; +import com.bakdata.conquery.models.auth.AuthorizationHelper; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.identifiable.ids.specific.UserId; import com.bakdata.conquery.resources.admin.rest.UserAuthenticationManagementResource; @@ -29,7 +31,7 @@ public boolean tryRegister(ProtoUser pUser) { return false; } log.trace("Added the user {} to the authorization storage", id); - if(ProtoUser.registerForAuthentication(realm, user, pUser.getCredentials(),false)) { + if(AuthorizationHelper.registerForAuthentication(realm, user, pUser.getCredentials(), false)) { log.trace("Added the user {} to the realm {}", id, realm.getName()); return true; } @@ -39,7 +41,7 @@ public boolean tryRegister(ProtoUser pUser) { public boolean updateUser(ProtoUser pUser) { final User user = pUser.createOrOverwriteUser(storage); - ProtoUser.registerForAuthentication(realm, user,pUser.getCredentials(),false); + AuthorizationHelper.registerForAuthentication(realm, user,pUser.getCredentials(),false); return true; } diff --git a/backend/src/test/java/com/bakdata/conquery/api/form/config/FormConfigTest.java b/backend/src/test/java/com/bakdata/conquery/api/form/config/FormConfigTest.java index 57bd8c7ef4..b64cc3ab87 100644 --- a/backend/src/test/java/com/bakdata/conquery/api/form/config/FormConfigTest.java +++ b/backend/src/test/java/com/bakdata/conquery/api/form/config/FormConfigTest.java @@ -183,7 +183,7 @@ public void deleteConfig() { // CHECK assertThat(storage.getAllFormConfigs()).doesNotContain(formConfig); - assertThat(storage.getUser(user.getId()).getPermissions()).doesNotContain(FormConfigPermission.onInstance(AbilitySets.FORM_CONFIG_CREATOR, formConfig.getId())); + assertThat(user.getPermissions()).doesNotContain(FormConfigPermission.onInstance(AbilitySets.FORM_CONFIG_CREATOR, formConfig.getId())); } @Test From f4945e5268c6c0ab73f3ef3fbdf5550382c13afb Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Thu, 23 Sep 2021 09:49:57 +0200 Subject: [PATCH 17/35] review changes --- .../conquery/io/storage/StoreMappings.java | 16 +----- .../xodus/stores/SerializingStore.java | 6 +-- .../models/auth/develop/DevAuthConfig.java | 11 ++-- .../models/config/XodusStoreFactory.java | 51 +++++++++---------- .../util/NonPersistentStoreFactory.java | 4 +- 5 files changed, 34 insertions(+), 54 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/StoreMappings.java b/backend/src/main/java/com/bakdata/conquery/io/storage/StoreMappings.java index 5ce211c4af..d607ae4b28 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/StoreMappings.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/StoreMappings.java @@ -81,22 +81,8 @@ public , CLASS_V extends Class> Sto /** * Store for identifiable values, with injectors. Store is also cached. */ - public static > DirectIdentifiableStore identifiable(Store, T> baseStore, CentralRegistry centralRegistry, Injectable... injectables) { - - for (Injectable injectable : injectables) { - baseStore.inject(injectable); - } - - baseStore.inject(centralRegistry); - - return new DirectIdentifiableStore<>(centralRegistry, baseStore); - } - - /** - * Store for identifiable values, without injectors. Store is also cached. - */ public static > DirectIdentifiableStore identifiable(Store, T> baseStore, CentralRegistry centralRegistry) { - return identifiable(baseStore, centralRegistry, new SingletonNamespaceCollection(centralRegistry)); + return new DirectIdentifiableStore<>(centralRegistry, baseStore); } /** diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java index d0a10f55ec..73c91cb95d 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java @@ -103,8 +103,7 @@ public , CLASS_V extends Class> SerializingSto CLASS_V valueType, boolean validateOnWrite, boolean removeUnreadableFromStore, - File unreadableDataDumpDirectory, - Injectable ... injectables) { + File unreadableDataDumpDirectory) { this.store = store; this.validator = validator; this.validateOnWrite = validateOnWrite; @@ -116,9 +115,6 @@ public , CLASS_V extends Class> SerializingSto valueWriter = objectMapper.writerFor(this.valueType); valueReader = objectMapper.readerFor(this.valueType); - for (Injectable injectable : injectables) { - valueReader = injectable.injectInto(valueReader); - } keyWriter = objectMapper.writerFor(keyType); diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/develop/DevAuthConfig.java b/backend/src/main/java/com/bakdata/conquery/models/auth/develop/DevAuthConfig.java index 058bc15801..fbcbd153a6 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/develop/DevAuthConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/develop/DevAuthConfig.java @@ -19,12 +19,11 @@ public class DevAuthConfig implements AuthenticationRealmFactory { @Override public ConqueryAuthenticationRealm createRealm(ManagerNode managerNode) { - final List initialUsers = managerNode.getConfig() - .getAuthorizationRealms().getInitialUsers(); - - Preconditions.checkState(initialUsers.size()>0, "There must be at least one initial user configured."); - - User defaultUser = initialUsers.get(0).createOrOverwriteUser(managerNode.getStorage()); + User defaultUser = managerNode.getConfig() + .getAuthorizationRealms() + .getInitialUsers() + .get(0) + .createOrOverwriteUser(managerNode.getStorage()); managerNode.getAuthController().getAuthenticationFilter().registerTokenExtractor(new UserIdTokenExtractor(defaultUser)); diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java index acf0d59532..0238f871f7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java @@ -100,35 +100,31 @@ @CPSType(id = "XODUS", base = StoreFactory.class) public class XodusStoreFactory implements StoreFactory { - public static final Supplier> NAMESPACED_STORES = () -> Stream.of( + public static final Set NAMESPACED_STORES = Set.of( DATASET.storeInfo().getName(), SECONDARY_IDS.storeInfo().getName(), TABLES.storeInfo().getName(), DICTIONARIES.storeInfo().getName() + BigStore.META, DICTIONARIES.storeInfo().getName() + BigStore.DATA, IMPORTS.storeInfo().getName(), - CONCEPTS.storeInfo().getName() - ); - public static final Set NAMESPACE_STORES = Stream.concat( - NAMESPACED_STORES.get(), - Stream.of( + CONCEPTS.storeInfo().getName()); + + public static final Set NAMESPACE_STORES = Sets.union( + NAMESPACED_STORES, + Set.of( ID_MAPPING.storeInfo().getName() + BigStore.META, ID_MAPPING.storeInfo().getName() + BigStore.DATA, STRUCTURE.storeInfo().getName(), WORKER_TO_BUCKETS.storeInfo().getName(), PRIMARY_DICTIONARY.storeInfo().getName() - ) - ) - .collect(Collectors.toUnmodifiableSet()); - public static final Set WORKER_STORES = Stream.concat( - NAMESPACED_STORES.get(), - Stream.of( + )); + public static final Set WORKER_STORES = Sets.union( + NAMESPACED_STORES, + Set.of( WORKER.storeInfo().getName(), BUCKETS.storeInfo().getName(), C_BLOCKS.storeInfo().getName() - ) - ) - .collect(Collectors.toUnmodifiableSet()); + )); private Path directory = Path.of("storage"); @@ -159,7 +155,7 @@ public class XodusStoreFactory implements StoreFactory { private transient Validator validator; @JsonIgnore - private transient ObjectMapper objectMapper = Jackson.BINARY_MAPPER.copy(); + private transient final ObjectMapper objectMapper = Jackson.BINARY_MAPPER.copy(); @JsonIgnore private final BiMap activeEnvironments = HashBiMap.create(); @@ -172,7 +168,7 @@ public class XodusStoreFactory implements StoreFactory { @Override public void init(ManagerNode managerNode) { validator = managerNode.getValidator(); - configureMapper(managerNode.getConfig()); + configureMapper(managerNode.getConfig(), managerNode.getDatasetRegistry(), managerNode.getStorage()); } @Override @@ -181,10 +177,14 @@ public void init(ShardNode shardNode) { configureMapper(shardNode.getConfig()); } - private void configureMapper(ConqueryConfig config) { + private void configureMapper(ConqueryConfig config, Injectable ... injectables) { config.configureObjectMapper(objectMapper); objectMapper.setConfig(objectMapper.getDeserializationConfig().withView(InternalOnly.class)); objectMapper.setConfig(objectMapper.getSerializationConfig().withView(InternalOnly.class)); + + for (Injectable injectable : injectables) { + injectable.injectInto(objectMapper); + } } @Override @@ -366,28 +366,28 @@ public SingletonStore createStructureStore(String pathName, Sin @Override public IdentifiableStore> createExecutionsStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName) { - return StoreMappings.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "executions")), validator, EXECUTIONS), centralRegistry, datasetRegistry); + return StoreMappings.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "executions")), validator, EXECUTIONS), centralRegistry); } @Override public IdentifiableStore createFormConfigStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName) { - return StoreMappings.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "formConfigs")), validator, FORM_CONFIG), centralRegistry, datasetRegistry); + return StoreMappings.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "formConfigs")), validator, FORM_CONFIG), centralRegistry); } @Override public IdentifiableStore createUserStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage) { - return StoreMappings.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "users")), validator, AUTH_USER), centralRegistry, storage); + return StoreMappings.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "users")), validator, AUTH_USER), centralRegistry); } @Override public IdentifiableStore createRoleStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage) { - return StoreMappings.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "roles")), validator, AUTH_ROLE), centralRegistry, storage); + return StoreMappings.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "roles")), validator, AUTH_ROLE), centralRegistry); } @Override public IdentifiableStore createGroupStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage) { - return StoreMappings.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "groups")), validator, AUTH_GROUP), centralRegistry, storage); + return StoreMappings.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "groups")), validator, AUTH_GROUP), centralRegistry); } @Override @@ -473,7 +473,7 @@ public static void removeEnvironmentHook(Environment env) { } } - public Store < KEY, VALUE > createStore(Environment environment, Validator validator, StoreMappings storeId, Injectable ...injectables){ + public Store < KEY, VALUE > createStore(Environment environment, Validator validator, StoreMappings storeId){ final StoreInfo storeInfo = storeId.storeInfo(); synchronized (openStoresInEnv) { return new CachedStore<>( @@ -486,8 +486,7 @@ public Store < KEY, VALUE > createStore(Environment environment, V storeInfo.getValueType(), this.isValidateOnWrite(), this.isRemoveUnreadableFromStore(), - this.getUnreadableDataDumpDirectory(), - injectables + this.getUnreadableDataDumpDirectory() )); } } diff --git a/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java b/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java index 310a8b6902..0411729930 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java +++ b/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java @@ -108,12 +108,12 @@ public SingletonStore createStructureStore(String pathName, Sin @Override public IdentifiableStore> createExecutionsStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName) { - return EXECUTIONS.identifiable(new NonPersistentStore(), centralRegistry, datasetRegistry); + return EXECUTIONS.identifiable(new NonPersistentStore(), centralRegistry); } @Override public IdentifiableStore createFormConfigStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName) { - return FORM_CONFIG.identifiable(new NonPersistentStore(), centralRegistry, datasetRegistry); + return FORM_CONFIG.identifiable(new NonPersistentStore(), centralRegistry); } @Override From 872d47f7aa710f411ab99dbf98a9e8a8ff5afa7b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 23 Sep 2021 07:52:02 +0000 Subject: [PATCH 18/35] Update AutoDoc --- docs/Config JSON.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/Config JSON.md b/docs/Config JSON.md index e340e80922..9e63e416cb 100644 --- a/docs/Config JSON.md +++ b/docs/Config JSON.md @@ -417,12 +417,12 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L133) | directory | `Path` | `"file://./storage"` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L144) | nThreads | `@javax.validation.constraints.Min(1) int` | ␀ | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L147-L149) | removeUnreadableFromStore | `boolean` | `false` | | Flag for the {@link SerializingStore} whether to delete values from the underlying store, that cannot be mapped to an object anymore. | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L152-L154) | unreadableDataDumpDirectory | `File` | `null` | | When set, all values that could not be deserialized from the persistent store, are dump into individual files. | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L140) | useWeakDictionaryCaching | `boolean` | `true` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L135) | validateOnWrite | `boolean` | `false` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L141) | weakCacheDuration | `@NotNull Duration` | `"48 hours"` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L136) | xodus | [XodusConfig](#Type-XodusConfig) | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L129) | directory | `Path` | `"file://./storage"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L140) | nThreads | `@javax.validation.constraints.Min(1) int` | ␀ | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L143-L145) | removeUnreadableFromStore | `boolean` | `false` | | Flag for the {@link SerializingStore} whether to delete values from the underlying store, that cannot be mapped to an object anymore. | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L148-L150) | unreadableDataDumpDirectory | `File` | `null` | | When set, all values that could not be deserialized from the persistent store, are dump into individual files. | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L136) | useWeakDictionaryCaching | `boolean` | `true` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L131) | validateOnWrite | `boolean` | `false` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L137) | weakCacheDuration | `@NotNull Duration` | `"48 hours"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L132) | xodus | [XodusConfig](#Type-XodusConfig) | | | |

From 368a13374a9adbb6ba42aab4dda30c8aaa80f9f3 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Thu, 23 Sep 2021 09:55:07 +0200 Subject: [PATCH 19/35] adds JsonIgnore to ProtoUser.java --- .../main/java/com/bakdata/conquery/apiv1/auth/ProtoUser.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/auth/ProtoUser.java b/backend/src/main/java/com/bakdata/conquery/apiv1/auth/ProtoUser.java index 734d3bd0c9..584370c805 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/auth/ProtoUser.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/auth/ProtoUser.java @@ -15,6 +15,7 @@ import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.auth.permissions.WildcardPermission; import com.bakdata.conquery.models.identifiable.ids.specific.UserId; +import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Builder; import lombok.Getter; import lombok.NonNull; @@ -61,6 +62,7 @@ public User createOrOverwriteUser(@NonNull MetaStorage storage) { return user; } + @JsonIgnore public UserId getId() { return new UserId(name); } From 2f9c5bbeb0f687d2967b3a37ec2be7be36b00f19 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Thu, 23 Sep 2021 11:23:16 +0200 Subject: [PATCH 20/35] split phase for instantiation and loading of MetaStorage.java to enable global injection into json deserialization --- .../com/bakdata/conquery/apiv1/ApiV1.java | 2 +- .../conquery/commands/ManagerNode.java | 15 ++-- .../conquery/io/jackson/Injectable.java | 15 +++- .../io/jackson/PathParamInjector.java | 2 +- .../conquery/io/mina/BinaryJacksonCoder.java | 2 +- .../conquery/io/storage/MetaStorage.java | 22 +++--- .../io/storage/xodus/stores/BigStore.java | 2 +- .../xodus/stores/SerializingStore.java | 2 +- .../conquery/models/config/StoreFactory.java | 4 +- .../models/config/XodusStoreFactory.java | 24 +++---- .../FormConfigProcessor.java | 6 +- .../models/preproc/PreprocessedReader.java | 2 +- .../models/query/QueryTranslator.java | 2 +- .../models/worker/DatasetRegistry.java | 10 ++- .../conquery/models/worker/Worker.java | 2 +- .../conquery/models/worker/Workers.java | 4 +- .../com/bakdata/conquery/util/io/Cloner.java | 2 +- .../api/form/config/FormConfigTest.java | 7 +- .../integration/common/LoadingUtil.java | 4 +- .../integration/json/ConqueryTestSpec.java | 8 +-- .../integration/json/JsonIntegrationTest.java | 2 +- .../tests/ConceptPermissionTest.java | 2 +- .../integration/tests/RestartTest.java | 15 ++-- .../tests/SecondaryIdEndpointTest.java | 4 +- .../ConceptUpdateAndDeletionTest.java | 2 +- .../tests/deletion/DatasetDeletionTest.java | 8 +-- .../tests/deletion/ImportDeletionTest.java | 2 +- .../tests/deletion/TableDeletionTest.java | 2 +- .../io/jackson/serializer/IdRefrenceTest.java | 2 +- .../serializer/SerializationTestUtil.java | 4 +- .../io/storage/xodus/stores/BigStoreTest.java | 2 +- .../stores/SerializingStoreDumpTest.java | 2 +- .../conquery/models/SerializationTests.java | 21 +++--- .../conquery/models/auth/CopyUserTest.java | 3 +- .../concepts/tree/GroovyIndexedTest.java | 4 +- .../models/execution/DefaultLabelTest.java | 3 +- .../tasks/PermissionCleanupTaskTest.java | 54 ++++++++------- .../conquery/tasks/QueryCleanupTaskTest.java | 69 ++++++++++--------- .../util/NonPersistentStoreFactory.java | 7 +- .../util/support/StandaloneSupport.java | 5 ++ 40 files changed, 193 insertions(+), 157 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/ApiV1.java b/backend/src/main/java/com/bakdata/conquery/apiv1/ApiV1.java index 64a2676931..03bfd5de79 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/ApiV1.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/ApiV1.java @@ -47,7 +47,7 @@ protected void configure() { bind(new ConceptsProcessor(manager.getDatasetRegistry())).to(ConceptsProcessor.class); bind(new MeProcessor(manager.getStorage(), datasets)).to(MeProcessor.class); bind(new QueryProcessor(datasets, manager.getStorage(), manager.getConfig())).to(QueryProcessor.class); - bind(new FormConfigProcessor(manager.getValidator(), manager.getStorage())).to(FormConfigProcessor.class); + bind(new FormConfigProcessor(manager.getValidator(), manager.getStorage(), datasets)).to(FormConfigProcessor.class); } }); diff --git a/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java b/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java index 42195b199a..2db0d5b3be 100644 --- a/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java +++ b/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java @@ -13,7 +13,6 @@ import com.bakdata.conquery.io.cps.CPSTypeIdResolver; import com.bakdata.conquery.io.jackson.InternalOnly; import com.bakdata.conquery.io.jackson.Jackson; -import com.bakdata.conquery.io.jackson.MutableInjectableValues; import com.bakdata.conquery.io.jersey.RESTServer; import com.bakdata.conquery.io.mina.BinaryJacksonCoder; import com.bakdata.conquery.io.mina.CQProtocolCodecFilter; @@ -33,7 +32,6 @@ import com.bakdata.conquery.models.messages.network.MessageToManagerNode; import com.bakdata.conquery.models.messages.network.NetworkMessageContext; import com.bakdata.conquery.models.worker.DatasetRegistry; -import com.bakdata.conquery.models.worker.IdResolveContext; import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.models.worker.Worker; import com.bakdata.conquery.resources.ResourcesProvider; @@ -104,11 +102,13 @@ public ManagerNode(@NonNull String name) { public void run(ConqueryConfig config, Environment environment) throws InterruptedException { client = new JerseyClientBuilder(environment).using(config.getJerseyClient()) .build(getName()); + + // Instantiate DatasetRegistry and MetaStorage so they are ready for injection into the object mapper (API + Storage) datasetRegistry = new DatasetRegistry(config.getCluster().getEntityBucketSize()); + storage = new MetaStorage(validator); - //inject datasets into the objectmapper - ((MutableInjectableValues) environment.getObjectMapper().getInjectableValues()) - .add(IdResolveContext.class, datasetRegistry); + datasetRegistry.injectInto(environment.getObjectMapper()); + storage.injectInto(environment.getObjectMapper()); jobManager = new JobManager("ManagerNode", config.isFailOnError()); @@ -183,8 +183,9 @@ public void run(ConqueryConfig config, Environment environment) throws Interrupt } private void loadMetaStorage() { - log.info("Started meta storage"); - storage = new MetaStorage(validator, config.getStorage(), datasetRegistry); + log.info("Opening MetaStorage"); + storage.openStores(config.getStorage()); + log.info("Loading MetaStorage"); storage.loadData(); log.info("MetaStorage loaded {}", storage); diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/Injectable.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/Injectable.java index 48e139cc1c..717eb08efe 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jackson/Injectable.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jackson/Injectable.java @@ -8,7 +8,7 @@ */ public interface Injectable { - default ObjectReader injectInto(ObjectReader reader) { + default ObjectReader injectIntoNew(ObjectReader reader) { // If is already MutableInjectable, add my values to other, else begin from scratch. if (reader.getInjectableValues() instanceof MutableInjectableValues) { return reader.with(inject(((MutableInjectableValues) reader.getInjectableValues()).copy())); @@ -17,16 +17,27 @@ default ObjectReader injectInto(ObjectReader reader) { return reader.with(inject(new MutableInjectableValues())); } - default ObjectMapper injectInto(ObjectMapper mapper) { + default ObjectMapper injectIntoNew(ObjectMapper mapper) { // If is already MutableInjectable, add my values to other, else begin from scratch. if (mapper.getInjectableValues() instanceof MutableInjectableValues) { return mapper.copy() .setInjectableValues(inject(((MutableInjectableValues) mapper.getInjectableValues()).copy())); } + // TODO unsuse return mapper.copy() .setInjectableValues(inject(new MutableInjectableValues())); } + default void injectInto(ObjectMapper mapper) { + // If is already MutableInjectable, add my values to other, else begin from scratch. + + if (mapper.getInjectableValues() instanceof MutableInjectableValues) { + mapper.setInjectableValues(inject(((MutableInjectableValues) mapper.getInjectableValues()))); + return; + } + throw new IllegalStateException("Cannot add additional injectables if the mapper does not provide MutableInjectableValues"); + } + MutableInjectableValues inject(MutableInjectableValues values); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/PathParamInjector.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/PathParamInjector.java index c77b464f0b..867b5e4d14 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jackson/PathParamInjector.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jackson/PathParamInjector.java @@ -30,7 +30,7 @@ public static class Modifier extends ObjectReaderModifier implements Injectable @Override public ObjectReader modify(EndpointConfigBase endpoint, MultivaluedMap httpHeaders, JavaType resultType, ObjectReader reader, JsonParser p) throws IOException { - return this.injectInto(reader); + return this.injectIntoNew(reader); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/io/mina/BinaryJacksonCoder.java b/backend/src/main/java/com/bakdata/conquery/io/mina/BinaryJacksonCoder.java index 5d32bbc367..36572590f6 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/mina/BinaryJacksonCoder.java +++ b/backend/src/main/java/com/bakdata/conquery/io/mina/BinaryJacksonCoder.java @@ -31,7 +31,7 @@ public BinaryJacksonCoder(IdResolveContext datasets, Validator validator, Object .writerFor(NetworkMessage.class) .withView(InternalOnly.class); this.reader = datasets - .injectInto(objectMapper.readerFor(NetworkMessage.class)) + .injectIntoNew(objectMapper.readerFor(NetworkMessage.class)) .without(Feature.AUTO_CLOSE_SOURCE) .withView(InternalOnly.class); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java index d423a33367..fbb6b0a782 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java @@ -20,6 +20,7 @@ import com.bakdata.conquery.models.identifiable.ids.specific.RoleId; import com.bakdata.conquery.models.identifiable.ids.specific.UserId; import com.bakdata.conquery.models.worker.DatasetRegistry; +import com.google.common.base.Preconditions; import lombok.Getter; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @@ -34,28 +35,33 @@ public class MetaStorage implements ConqueryStorage, Injectable { private IdentifiableStore authRole; private IdentifiableStore authGroup; - @Getter - private DatasetRegistry datasetRegistry; @Getter protected final CentralRegistry centralRegistry = new CentralRegistry(); @Getter protected final Validator validator; - public MetaStorage(Validator validator, StoreFactory storageFactory, DatasetRegistry datasetRegistry) { - this.datasetRegistry = datasetRegistry; + public MetaStorage(Validator validator) { this.validator = validator; + } - + public void openStores(StoreFactory storageFactory) { authUser = storageFactory.createUserStore(centralRegistry, "meta", this); authRole = storageFactory.createRoleStore(centralRegistry, "meta", this); authGroup = storageFactory.createGroupStore(centralRegistry, "meta", this); // Executions depend on users - executions = storageFactory.createExecutionsStore(centralRegistry, datasetRegistry, "meta"); - formConfigs = storageFactory.createFormConfigStore(centralRegistry, datasetRegistry, "meta"); - } + executions = storageFactory.createExecutionsStore(centralRegistry, "meta"); + formConfigs = storageFactory.createFormConfigStore(centralRegistry, "meta"); + + } @Override public void loadData() { + Preconditions.checkNotNull(authUser, "User storage was not created"); + Preconditions.checkNotNull(authRole, "Role storage was not created"); + Preconditions.checkNotNull(authGroup, "Group storage was not created"); + Preconditions.checkNotNull(executions, "Execution storage was not created"); + Preconditions.checkNotNull(formConfigs, "FormConfig storage was not created"); + authUser.loadData(); authRole.loadData(); authGroup.loadData(); diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/BigStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/BigStore.java index ed1d0ba8b4..6a8560440d 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/BigStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/BigStore.java @@ -227,7 +227,7 @@ public Stream loadData(SerializingStore dataStore) { @Override public void inject(Injectable injectable) { - valueReader = injectable.injectInto(valueReader); + valueReader = injectable.injectIntoNew(valueReader); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java index 73c91cb95d..0dc56d6f66 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java @@ -390,7 +390,7 @@ public Collection getAll() { @Override public void inject(Injectable injectable) { - valueReader = injectable.injectInto(valueReader); + valueReader = injectable.injectIntoNew(valueReader); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java index ebe79309b7..cc5a9acb37 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java @@ -62,8 +62,8 @@ public interface StoreFactory { SingletonStore createStructureStore(String pathName, SingletonNamespaceCollection centralRegistry); // MetaStorage - IdentifiableStore> createExecutionsStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName); - IdentifiableStore createFormConfigStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName); + IdentifiableStore> createExecutionsStore(CentralRegistry centralRegistry, String pathName); + IdentifiableStore createFormConfigStore(CentralRegistry centralRegistry, String pathName); IdentifiableStore createUserStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage); IdentifiableStore createRoleStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage); IdentifiableStore createGroupStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage); diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java index 0238f871f7..b189958659 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java @@ -16,9 +16,6 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.function.Function; -import java.util.function.Supplier; -import java.util.stream.Collectors; -import java.util.stream.Stream; import javax.annotation.Nullable; import javax.validation.Valid; @@ -29,9 +26,7 @@ import com.bakdata.conquery.commands.ManagerNode; import com.bakdata.conquery.commands.ShardNode; import com.bakdata.conquery.io.cps.CPSType; -import com.bakdata.conquery.io.jackson.Injectable; import com.bakdata.conquery.io.jackson.InternalOnly; -import com.bakdata.conquery.io.jackson.Jackson; import com.bakdata.conquery.io.storage.IdentifiableStore; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.io.storage.NamespaceStorage; @@ -63,7 +58,6 @@ import com.bakdata.conquery.models.identifiable.CentralRegistry; import com.bakdata.conquery.models.identifiable.ids.IId; import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; -import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.SingletonNamespaceCollection; import com.bakdata.conquery.models.worker.WorkerInformation; import com.bakdata.conquery.models.worker.WorkerToBucketsMap; @@ -155,7 +149,7 @@ public class XodusStoreFactory implements StoreFactory { private transient Validator validator; @JsonIgnore - private transient final ObjectMapper objectMapper = Jackson.BINARY_MAPPER.copy(); + private transient ObjectMapper objectMapper; @JsonIgnore private final BiMap activeEnvironments = HashBiMap.create(); @@ -168,23 +162,21 @@ public class XodusStoreFactory implements StoreFactory { @Override public void init(ManagerNode managerNode) { validator = managerNode.getValidator(); - configureMapper(managerNode.getConfig(), managerNode.getDatasetRegistry(), managerNode.getStorage()); + objectMapper = managerNode.getEnvironment().getObjectMapper(); + configureMapper(managerNode.getConfig()); } @Override public void init(ShardNode shardNode) { validator = shardNode.getValidator(); + objectMapper = shardNode.getEnvironment().getObjectMapper(); configureMapper(shardNode.getConfig()); } - private void configureMapper(ConqueryConfig config, Injectable ... injectables) { + private void configureMapper(ConqueryConfig config) { config.configureObjectMapper(objectMapper); objectMapper.setConfig(objectMapper.getDeserializationConfig().withView(InternalOnly.class)); objectMapper.setConfig(objectMapper.getSerializationConfig().withView(InternalOnly.class)); - - for (Injectable injectable : injectables) { - injectable.injectInto(objectMapper); - } } @Override @@ -307,7 +299,7 @@ public IdentifiableStore createDictionaryStore(CentralRegistry centr DICTIONARIES.storeInfo(), this::closeStore, this::removeStore, - namespaceCollection.injectInto(objectMapper) + namespaceCollection.injectIntoNew(objectMapper) ); } @@ -365,12 +357,12 @@ public SingletonStore createStructureStore(String pathName, Sin } @Override - public IdentifiableStore> createExecutionsStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName) { + public IdentifiableStore> createExecutionsStore(CentralRegistry centralRegistry, String pathName) { return StoreMappings.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "executions")), validator, EXECUTIONS), centralRegistry); } @Override - public IdentifiableStore createFormConfigStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName) { + public IdentifiableStore createFormConfigStore(CentralRegistry centralRegistry, String pathName) { return StoreMappings.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "formConfigs")), validator, FORM_CONFIG), centralRegistry); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormConfigProcessor.java b/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormConfigProcessor.java index c127f6f0d3..038ddb6de6 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormConfigProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormConfigProcessor.java @@ -27,6 +27,7 @@ import com.bakdata.conquery.models.identifiable.ids.NamespacedId; import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; import com.bakdata.conquery.models.identifiable.ids.specific.FormConfigId; +import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.resources.api.FormConfigResource; import com.fasterxml.jackson.databind.ObjectMapper; @@ -48,6 +49,7 @@ public class FormConfigProcessor { private final Validator validator; private final MetaStorage storage; + private final DatasetRegistry datasetRegistry; @Getter(onMethod = @__({@TestOnly})) private final static ObjectMapper MAPPER = Jackson.MAPPER.copy().disable(SerializationFeature.WRITE_EMPTY_JSON_ARRAYS, SerializationFeature.WRITE_NULL_MAP_VALUES);; @@ -106,11 +108,11 @@ public FormConfigFullRepresentation getConfig(User user, FormConfig form) { public FormConfig addConfig(User user, Dataset targetDataset, FormConfigAPI config) { //TODO clear this up - final Namespace namespace = storage.getDatasetRegistry().get(targetDataset.getId()); + final Namespace namespace = datasetRegistry.get(targetDataset.getId()); user.authorize(namespace.getDataset(), Ability.READ); - List translateToDatasets = storage.getDatasetRegistry().getAllDatasets() + List translateToDatasets = datasetRegistry.getAllDatasets() .stream() .filter(dId -> user.isPermitted(dId, Ability.READ)) .map(Identifiable::getId) diff --git a/backend/src/main/java/com/bakdata/conquery/models/preproc/PreprocessedReader.java b/backend/src/main/java/com/bakdata/conquery/models/preproc/PreprocessedReader.java index 8662e856d1..d76b74aa55 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/preproc/PreprocessedReader.java +++ b/backend/src/main/java/com/bakdata/conquery/models/preproc/PreprocessedReader.java @@ -47,7 +47,7 @@ public PreprocessedReader(InputStream inputStream) throws IOException { final InjectingCentralRegistry injectingCentralRegistry = new InjectingCentralRegistry(replacements); final SingletonNamespaceCollection namespaceCollection = new SingletonNamespaceCollection(injectingCentralRegistry); - parser = namespaceCollection.injectInto(Jackson.BINARY_MAPPER.copy()) + parser = namespaceCollection.injectIntoNew(Jackson.BINARY_MAPPER.copy()) .enable(JsonGenerator.Feature.AUTO_CLOSE_TARGET) .getFactory() .createParser(inputStream); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/QueryTranslator.java b/backend/src/main/java/com/bakdata/conquery/models/query/QueryTranslator.java index 6b6ae27935..f9852f619a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/QueryTranslator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/QueryTranslator.java @@ -51,7 +51,7 @@ public T replaceDataset(DatasetRegistry namespaces, T elem } return (T)namespaces - .injectInto(Jackson.MAPPER.copy()) + .injectIntoNew(Jackson.MAPPER.copy()) .readValue(value, element.getClass()); } catch(Exception e) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/DatasetRegistry.java b/backend/src/main/java/com/bakdata/conquery/models/worker/DatasetRegistry.java index 9d421b6514..71e6ae1b9f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/DatasetRegistry.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/DatasetRegistry.java @@ -2,18 +2,17 @@ import java.io.Closeable; import java.net.SocketAddress; -import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.NoSuchElementException; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.function.Supplier; import java.util.stream.Collectors; import javax.validation.constraints.NotNull; import com.bakdata.conquery.commands.ManagerNode; +import com.bakdata.conquery.io.jackson.MutableInjectableValues; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.io.storage.NamespaceStorage; import com.bakdata.conquery.models.datasets.Dataset; @@ -125,4 +124,11 @@ public void close() { } } } + + @Override + public MutableInjectableValues inject(MutableInjectableValues values) { + // Make this class also availiable under DatasetRegistry + return super.inject(values) + .add(DatasetRegistry.class, this); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/Worker.java b/backend/src/main/java/com/bakdata/conquery/models/worker/Worker.java index 52163ae71c..fca98d45dc 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/Worker.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/Worker.java @@ -125,7 +125,7 @@ public MessageToManagerNode transform(NamespaceMessage message) { public ObjectMapper inject(ObjectMapper binaryMapper) { return new SingletonNamespaceCollection(storage.getCentralRegistry()) - .injectInto(binaryMapper); + .injectIntoNew(binaryMapper); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/Workers.java b/backend/src/main/java/com/bakdata/conquery/models/worker/Workers.java index fb4bab0dfb..ff486f3fa5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/Workers.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/Workers.java @@ -49,8 +49,8 @@ public Workers(ThreadPoolDefinition queryThreadPoolDefinition, ObjectMapper mapp jobsThreadPool = queryThreadPoolDefinition.createService("Workers"); - this.mapper = injectInto(mapper); - this.binaryMapper = injectInto(binaryMapper); + this.mapper = injectIntoNew(mapper); + this.binaryMapper = injectIntoNew(binaryMapper); this.entityBucketSize = entityBucketSize; jobsThreadPool.prestartAllCoreThreads(); diff --git a/backend/src/main/java/com/bakdata/conquery/util/io/Cloner.java b/backend/src/main/java/com/bakdata/conquery/util/io/Cloner.java index f972e6a516..30829dc1c7 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/io/Cloner.java +++ b/backend/src/main/java/com/bakdata/conquery/util/io/Cloner.java @@ -35,7 +35,7 @@ public static ConqueryConfig clone(ConqueryConfig config, Map, T> i public static T clone(T element, Injectable injectable, Class valueType) { try { return injectable - .injectInto(Jackson.BINARY_MAPPER) + .injectIntoNew(Jackson.BINARY_MAPPER) .readValue( Jackson.BINARY_MAPPER.writeValueAsBytes(element), valueType diff --git a/backend/src/test/java/com/bakdata/conquery/api/form/config/FormConfigTest.java b/backend/src/test/java/com/bakdata/conquery/api/form/config/FormConfigTest.java index b64cc3ab87..260e30ea42 100644 --- a/backend/src/test/java/com/bakdata/conquery/api/form/config/FormConfigTest.java +++ b/backend/src/test/java/com/bakdata/conquery/api/form/config/FormConfigTest.java @@ -114,15 +114,16 @@ else if (id.equals(datasetId1)) { return namespaceMock; }).when(namespacesMock).get(any(DatasetId.class)); when(namespacesMock.getAllDatasets()).thenReturn(List.of(dataset,dataset1)); - when(namespacesMock.injectInto(any(ObjectMapper.class))).thenCallRealMethod(); + when(namespacesMock.injectIntoNew(any(ObjectMapper.class))).thenCallRealMethod(); when(namespacesMock.inject(any(MutableInjectableValues.class))).thenCallRealMethod(); - storage = new MetaStorage(null, new NonPersistentStoreFactory(), namespacesMock); + storage = new MetaStorage(null); + storage.openStores(new NonPersistentStoreFactory()); ((MutableInjectableValues)FormConfigProcessor.getMAPPER().getInjectableValues()) .add(IdResolveContext.class, namespacesMock); - processor = new FormConfigProcessor(validator, storage); + processor = new FormConfigProcessor(validator, storage, namespacesMock); controller = new AuthorizationController(storage, new DevelopmentAuthorizationConfig()); controller.start(); } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java b/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java index c5a90d60aa..d1cb1e6e95 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java @@ -79,8 +79,8 @@ public static void importPreviousQueries(StandaloneSupport support, RequiredData } for (JsonNode queryNode : content.getPreviousQueries()) { - ObjectMapper mapper = new SingletonNamespaceCollection(support.getNamespaceStorage().getCentralRegistry()).injectInto(Jackson.MAPPER); - mapper = support.getDataset().injectInto(mapper); + ObjectMapper mapper = new SingletonNamespaceCollection(support.getNamespaceStorage().getCentralRegistry()).injectIntoNew(Jackson.MAPPER); + mapper = support.getDataset().injectIntoNew(mapper); Query query = mapper.readerFor(Query.class).readValue(queryNode); UUID queryId = new UUID(0L, id++); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/ConqueryTestSpec.java b/backend/src/test/java/com/bakdata/conquery/integration/json/ConqueryTestSpec.java index 44a186366b..8efd48d9ed 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/ConqueryTestSpec.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/ConqueryTestSpec.java @@ -62,9 +62,9 @@ public static T parseSubTree(StandaloneSupport support, JsonNode node, JavaT } public static T parseSubTree(StandaloneSupport support, JsonNode node, JavaType expectedType, Consumer modifierBeforeValidation) throws IOException, JSONException { - ObjectMapper mapper = support.getDataset().injectInto( + ObjectMapper mapper = support.getDataset().injectIntoNew( new SingletonNamespaceCollection(support.getNamespace().getStorage().getCentralRegistry(), support.getMetaStorage().getCentralRegistry()) - .injectInto( + .injectIntoNew( Jackson.MAPPER.copy().addHandler(new DatasetPlaceHolderFiller(support)) ) ); @@ -80,8 +80,8 @@ public static T parseSubTree(StandaloneSupport support, JsonNode node, Java } public static List parseSubTreeList(StandaloneSupport support, ArrayNode node, Class expectedType, Consumer modifierBeforeValidation) throws IOException, JSONException { - ObjectMapper mapper = support.getDataset().injectInto( - new SingletonNamespaceCollection(support.getNamespace().getStorage().getCentralRegistry()).injectInto( + ObjectMapper mapper = support.getDataset().injectIntoNew( + new SingletonNamespaceCollection(support.getNamespace().getStorage().getCentralRegistry()).injectIntoNew( Jackson.MAPPER.copy().addHandler(new DatasetPlaceHolderFiller(support)) ) ); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/JsonIntegrationTest.java b/backend/src/test/java/com/bakdata/conquery/integration/json/JsonIntegrationTest.java index bbc601eb95..e542ce5270 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/JsonIntegrationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/JsonIntegrationTest.java @@ -58,7 +58,7 @@ public static ConqueryTestSpec readJson(DatasetId dataset, String json) throws I } public static ConqueryTestSpec readJson(Dataset dataset, String json) throws IOException { - return readJson(dataset.getId(), json, dataset.injectInto(TEST_SPEC_READER)); + return readJson(dataset.getId(), json, dataset.injectIntoNew(TEST_SPEC_READER)); } private static ConqueryTestSpec readJson(DatasetId dataset, String json, ObjectReader jsonReader) throws IOException { diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/ConceptPermissionTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/ConceptPermissionTest.java index 6698adca50..346891325d 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/ConceptPermissionTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/ConceptPermissionTest.java @@ -32,7 +32,7 @@ public void execute(StandaloneSupport conquery) throws Exception { final Dataset dataset = conquery.getDataset(); final String testJson = In.resource("/tests/query/SIMPLE_TREECONCEPT_QUERY/SIMPLE_TREECONCEPT_Query.test.json").withUTF8().readAll(); final QueryTest test = (QueryTest) JsonIntegrationTest.readJson(dataset.getId(), testJson); - final QueryProcessor processor = new QueryProcessor(storage.getDatasetRegistry(), storage, conquery.getConfig()); + final QueryProcessor processor = new QueryProcessor(conquery.getDatasetRegistry(), storage, conquery.getConfig()); final User user = new User("testUser", "testUserLabel", storage); // Manually import data, so we can do our own work. diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/RestartTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/RestartTest.java index 878f1ba86b..3b7044441f 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/RestartTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/RestartTest.java @@ -18,6 +18,7 @@ import com.bakdata.conquery.models.identifiable.IdMapSerialisationTest; import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; +import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.resources.admin.rest.AdminDatasetProcessor; import com.bakdata.conquery.resources.admin.rest.AdminProcessor; import com.bakdata.conquery.util.support.StandaloneSupport; @@ -141,6 +142,8 @@ public void execute(String name, TestConquery testConquery) throws Exception { log.info("Restart complete"); + + DatasetRegistry datasetRegistry = support.getDatasetsProcessor().getDatasetRegistry(); test.executeTest(support); @@ -154,16 +157,16 @@ public void execute(String name, TestConquery testConquery) throws Exception { assertThat(storage.getRole(roleToDelete.getId())).as("deleted role should stay deleted").isNull(); assertThat(storage.getGroup(groupToDelete.getId())).as("deleted group should stay deleted").isNull(); - assertThat(userStored.isPermitted(storage.getDatasetRegistry().get(TEST_DATASET_1.getId()).getDataset(), Ability.READ)).isTrue(); - assertThat(userStored.isPermitted(storage.getDatasetRegistry() + assertThat(userStored.isPermitted(datasetRegistry.get(TEST_DATASET_1.getId()).getDataset(), Ability.READ)).isTrue(); + assertThat(userStored.isPermitted(datasetRegistry .get(TEST_DATASET_2.getId()) .getDataset(), Ability.READ)).isFalse(); // Was never permitted - assertThat(userStored.isPermitted(storage.getDatasetRegistry().get(TEST_DATASET_3.getId()).getDataset(), Ability.READ)).isTrue(); - assertThat(userStored.isPermitted(storage.getDatasetRegistry() + assertThat(userStored.isPermitted(datasetRegistry.get(TEST_DATASET_3.getId()).getDataset(), Ability.READ)).isTrue(); + assertThat(userStored.isPermitted(datasetRegistry .get(TEST_DATASET_4.getId()) .getDataset(), Ability.READ)).isFalse(); // Was permitted by deleted role - assertThat(userStored.isPermitted(storage.getDatasetRegistry().get(TEST_DATASET_5.getId()).getDataset(), Ability.READ)).isTrue(); - assertThat(userStored.isPermitted(storage.getDatasetRegistry() + assertThat(userStored.isPermitted(datasetRegistry.get(TEST_DATASET_5.getId()).getDataset(), Ability.READ)).isTrue(); + assertThat(userStored.isPermitted(datasetRegistry .get(TEST_DATASET_6.getId()) .getDataset(), Ability.READ)).isFalse(); // Was permitted by deleted group diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/SecondaryIdEndpointTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/SecondaryIdEndpointTest.java index f828cfc243..24bca6fe6a 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/SecondaryIdEndpointTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/SecondaryIdEndpointTest.java @@ -133,8 +133,8 @@ private static Set fetchSecondaryIdDescriptions(StandaloneSupport .get(ObjectNode.class); // The injection is necessary to deserialize the dataset. - ObjectMapper mapper = conquery.getNamespace().getNamespaces().injectInto(Jackson.MAPPER); - mapper = conquery.getDataset().injectInto(mapper); + ObjectMapper mapper = conquery.getNamespace().getNamespaces().injectIntoNew(Jackson.MAPPER); + mapper = conquery.getDataset().injectIntoNew(mapper); return objectNode.get("secondaryIds") .traverse(mapper.getFactory().getCodec()) diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ConceptUpdateAndDeletionTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ConceptUpdateAndDeletionTest.java index 336ae94a20..a3a60453dd 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ConceptUpdateAndDeletionTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ConceptUpdateAndDeletionTest.java @@ -46,7 +46,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { final String testJson2 = In.resource("/tests/query/UPDATE_CONCEPT_TESTS/SIMPLE_TREECONCEPT_2_Query.json").withUTF8().readAll(); final Dataset dataset = conquery.getDataset(); - final Namespace namespace = storage.getDatasetRegistry().get(dataset.getId()); + final Namespace namespace = conquery.getNamespace(); final ConceptId conceptId = ConceptId.Parser.INSTANCE.parse(dataset.getName(), "test_tree"); final Concept concept; diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/DatasetDeletionTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/DatasetDeletionTest.java index 9ec75af71a..9f3e86177e 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/DatasetDeletionTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/DatasetDeletionTest.java @@ -39,7 +39,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { StandaloneSupport conquery = testConquery.getSupport(name); final MetaStorage storage = conquery.getMetaStorage(); final Dataset dataset = conquery.getDataset(); - Namespace namespace = storage.getDatasetRegistry().get(dataset.getId()); + Namespace namespace = conquery.getNamespace(); final String testJson = In.resource("/tests/query/DELETE_IMPORT_TESTS/SIMPLE_TREECONCEPT_Query.test.json").withUTF8().readAll(); final QueryTest test = (QueryTest) JsonIntegrationTest.readJson(dataset, testJson); @@ -170,8 +170,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { final StandaloneSupport conquery2 = new StandaloneSupport( testConquery, - storage.getDatasetRegistry() - .get(dataset.getId()), + namespace, newDataset, conquery.getTmpDir(), conquery.getConfig(), @@ -180,9 +179,6 @@ public void execute(String name, TestConquery testConquery) throws Exception { conquery.getTestUser() ); - - namespace = storage.getDatasetRegistry().get(dataset.getId()); - // only import the deleted import/table LoadingUtil.importTables(conquery2,test.getContent().getTables()); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ImportDeletionTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ImportDeletionTest.java index 4a2493e39e..ffca267783 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ImportDeletionTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ImportDeletionTest.java @@ -60,7 +60,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { final String testJson = In.resource("/tests/query/DELETE_IMPORT_TESTS/SIMPLE_TREECONCEPT_Query.test.json").withUTF8().readAll(); final Dataset dataset = conquery.getDataset(); - final Namespace namespace = storage.getDatasetRegistry().get(dataset.getId()); + final Namespace namespace = conquery.getNamespace(); final ImportId importId = ImportId.Parser.INSTANCE.parse(dataset.getName(), "test_table2", "test_table2"); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/TableDeletionTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/TableDeletionTest.java index 6ef09ec845..e32949c249 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/TableDeletionTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/TableDeletionTest.java @@ -48,7 +48,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { final String testJson = In.resource("/tests/query/DELETE_IMPORT_TESTS/SIMPLE_TREECONCEPT_Query.test.json").withUTF8().readAll(); final Dataset dataset = conquery.getDataset(); - final Namespace namespace = storage.getDatasetRegistry().get(dataset.getId()); + final Namespace namespace = conquery.getNamespace(); final TableId tableId = TableId.Parser.INSTANCE.parse(dataset.getName(), "test_table2"); diff --git a/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/IdRefrenceTest.java b/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/IdRefrenceTest.java index f8c5a44d38..f02d46d7f3 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/IdRefrenceTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/IdRefrenceTest.java @@ -48,7 +48,7 @@ public void testListReferences() throws IOException { .contains("\"dataset.table\""); ListHolder holder = new SingletonNamespaceCollection(registry, metaRegistry) - .injectInto(Jackson.MAPPER.readerFor(ListHolder.class)) + .injectIntoNew(Jackson.MAPPER.readerFor(ListHolder.class)) .readValue(json); assertThat(holder.getUsers().get(0)).isSameAs(user); diff --git a/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/SerializationTestUtil.java b/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/SerializationTestUtil.java index 5555abaeb7..efced7107a 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/SerializationTestUtil.java +++ b/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/SerializationTestUtil.java @@ -71,10 +71,10 @@ public void test(T value) throws JSONException, IOException { private void test(T value, T expected, ObjectMapper mapper) throws IOException { if (registry != null) { - mapper = new SingletonNamespaceCollection(registry, registry).injectInto(mapper); + mapper = new SingletonNamespaceCollection(registry, registry).injectIntoNew(mapper); } for (Injectable injectable : injectables) { - mapper = injectable.injectInto(mapper); + mapper = injectable.injectIntoNew(mapper); } ObjectWriter writer = mapper.writerFor(type).withView(InternalOnly.class); ObjectReader reader = mapper.readerFor(type).withView(InternalOnly.class); diff --git a/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/BigStoreTest.java b/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/BigStoreTest.java index fe93fe407f..58b0a1fdce 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/BigStoreTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/BigStoreTest.java @@ -39,7 +39,7 @@ public class BigStoreTest { private static final CentralRegistry CENTRAL_REGISTRY = new CentralRegistry(); private static SingletonNamespaceCollection NAMESPACE_COLLECTION = new SingletonNamespaceCollection(CENTRAL_REGISTRY); - private static ObjectMapper MAPPER = NAMESPACE_COLLECTION.injectInto(Jackson.BINARY_MAPPER); + private static ObjectMapper MAPPER = NAMESPACE_COLLECTION.injectIntoNew(Jackson.BINARY_MAPPER); @BeforeAll public static void setupRegistry(){ diff --git a/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStoreDumpTest.java b/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStoreDumpTest.java index e30a06de21..052c39fc5d 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStoreDumpTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStoreDumpTest.java @@ -37,7 +37,7 @@ public class SerializingStoreDumpTest { public static final StoreInfo USER_STORE_ID = StoreMappings.AUTH_USER.storeInfo(); - private final static MetaStorage STORAGE = new MetaStorage(Validators.newValidator(), new NonPersistentStoreFactory(), new DatasetRegistry(2)); + private final static MetaStorage STORAGE = new MetaStorage(Validators.newValidator()); private File tmpDir; private Environment env; private XodusStoreFactory config; diff --git a/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java b/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java index 1af1fca99c..28c902b97e 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java +++ b/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java @@ -57,12 +57,19 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.dropwizard.jersey.validation.Validators; import lombok.extern.slf4j.Slf4j; +import org.junit.Before; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @Slf4j public class SerializationTests { - private final static MetaStorage STORAGE = new MetaStorage(Validators.newValidator(), new NonPersistentStoreFactory(), new DatasetRegistry(2)); + private final static MetaStorage STORAGE = new MetaStorage(null); + + @BeforeAll + public static void beforAll() { + STORAGE.openStores(new NonPersistentStoreFactory()); + } @Test public void dataset() throws IOException, JSONException { @@ -98,13 +105,9 @@ public void role() throws IOException, JSONException { */ @Test public void user() throws IOException, JSONException { - MetaStorage storage = new MetaStorage(null, new NonPersistentStoreFactory(), null); User user = new User("user", "user", STORAGE); user.addPermission(DatasetPermission.onInstance(Ability.READ, new DatasetId("test"))); - user - .addPermission( - ExecutionPermission.onInstance(Ability.READ, new ManagedExecutionId(new DatasetId("dataset"), UUID.randomUUID())) - ); + user.addPermission(ExecutionPermission.onInstance(Ability.READ, new ManagedExecutionId(new DatasetId("dataset"), UUID.randomUUID()))); Role role = new Role("company", "company", STORAGE); user.addRole(role); @@ -120,13 +123,9 @@ public void user() throws IOException, JSONException { @Test public void group() throws IOException, JSONException { - MetaStorage storage = new MetaStorage(null, new NonPersistentStoreFactory(), null); Group group = new Group("group", "group", STORAGE); group.addPermission(DatasetPermission.onInstance(Ability.READ, new DatasetId("test"))); - group - .addPermission( - ExecutionPermission.onInstance(Ability.READ, new ManagedExecutionId(new DatasetId("dataset"), UUID.randomUUID())) - ); + group.addPermission(ExecutionPermission.onInstance(Ability.READ, new ManagedExecutionId(new DatasetId("dataset"), UUID.randomUUID()))); group.addRole(new Role("company", "company", STORAGE)); Role role = new Role("company", "company", STORAGE); diff --git a/backend/src/test/java/com/bakdata/conquery/models/auth/CopyUserTest.java b/backend/src/test/java/com/bakdata/conquery/models/auth/CopyUserTest.java index 2cf97d5d25..7fad584d0b 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/auth/CopyUserTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/auth/CopyUserTest.java @@ -20,7 +20,8 @@ void testUserCopy(){ final DatasetRegistry registry = new DatasetRegistry(0); - MetaStorage storage = new MetaStorage(null, new NonPersistentStoreFactory(), registry); + MetaStorage storage = new MetaStorage(null); + storage.openStores(new NonPersistentStoreFactory()); registry.setMetaStorage(storage); diff --git a/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/tree/GroovyIndexedTest.java b/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/tree/GroovyIndexedTest.java index 1c3e02f877..582fd959bb 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/tree/GroovyIndexedTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/tree/GroovyIndexedTest.java @@ -83,14 +83,14 @@ public static void init() throws IOException, JSONException, ConfigurationExcept // load tree twice to to avoid references - indexedConcept = new SingletonNamespaceCollection(registry).injectInto(dataset.injectInto(Jackson.MAPPER.readerFor(Concept.class))).readValue(node); + indexedConcept = new SingletonNamespaceCollection(registry).injectIntoNew(dataset.injectIntoNew(Jackson.MAPPER.readerFor(Concept.class))).readValue(node); indexedConcept.setDataset(dataset); indexedConcept.initElements(Validators.newValidator()); TreeChildPrefixIndex.putIndexInto(indexedConcept); - oldConcept = new SingletonNamespaceCollection(registry).injectInto(dataset.injectInto(Jackson.MAPPER.readerFor(Concept.class))).readValue(node); + oldConcept = new SingletonNamespaceCollection(registry).injectIntoNew(dataset.injectIntoNew(Jackson.MAPPER.readerFor(Concept.class))).readValue(node); oldConcept.setDataset(dataset); oldConcept.initElements(Validators.newValidator()); diff --git a/backend/src/test/java/com/bakdata/conquery/models/execution/DefaultLabelTest.java b/backend/src/test/java/com/bakdata/conquery/models/execution/DefaultLabelTest.java index 906b75bf2c..6c5b9846d5 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/execution/DefaultLabelTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/execution/DefaultLabelTest.java @@ -35,7 +35,7 @@ public class DefaultLabelTest { - private final static MetaStorage STORAGE = new MetaStorage(Validators.newValidator(), new NonPersistentStoreFactory(), new DatasetRegistry(2)); + private final static MetaStorage STORAGE = new MetaStorage(null); private static final DatasetRegistry DATASET_REGISTRY = Mockito.mock(DatasetRegistry.class); private static final Dataset DATASET = new Dataset("dataset"); @@ -52,6 +52,7 @@ public class DefaultLabelTest { @BeforeAll public static void beforeAll() { + STORAGE.openStores(new NonPersistentStoreFactory()); I18n.init(); diff --git a/backend/src/test/java/com/bakdata/conquery/tasks/PermissionCleanupTaskTest.java b/backend/src/test/java/com/bakdata/conquery/tasks/PermissionCleanupTaskTest.java index debbc14e33..116fb3c6f8 100644 --- a/backend/src/test/java/com/bakdata/conquery/tasks/PermissionCleanupTaskTest.java +++ b/backend/src/test/java/com/bakdata/conquery/tasks/PermissionCleanupTaskTest.java @@ -23,16 +23,22 @@ import com.bakdata.conquery.apiv1.query.concept.specific.CQAnd; import com.bakdata.conquery.util.NonPersistentStoreFactory; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; class PermissionCleanupTaskTest { - private MetaStorage storage = new MetaStorage(null, new NonPersistentStoreFactory(), null); + private static final MetaStorage STORAGE = new MetaStorage(null); + + @BeforeAll + public static void beforeAll() { + STORAGE.openStores(new NonPersistentStoreFactory()); + } @AfterEach public void teardownAfterEach() { - storage.clear(); + STORAGE.clear(); } private ManagedQuery createManagedQuery() { @@ -45,22 +51,22 @@ private ManagedQuery createManagedQuery() { managedQuery.setCreationTime(LocalDateTime.now().minusDays(1)); - storage.addExecution(managedQuery); + STORAGE.addExecution(managedQuery); return managedQuery; } @Test void doNotDeletePermissionValidReference() { - assertThat(storage.getAllExecutions()).isEmpty(); + assertThat(STORAGE.getAllExecutions()).isEmpty(); final ManagedQuery managedQuery = createManagedQuery(); // Saving the Execution - User user = new User("test", "test", storage); - storage.updateUser(user); + User user = new User("test", "test", STORAGE); + STORAGE.updateUser(user); user.addPermission(ExecutionPermission.onInstance(AbilitySets.QUERY_CREATOR, managedQuery.getId())); - deleteQueryPermissionsWithMissingRef(storage, storage.getAllUsers()); + deleteQueryPermissionsWithMissingRef(STORAGE, STORAGE.getAllUsers()); assertThat(user.getPermissions()).containsOnly(ExecutionPermission.onInstance(AbilitySets.QUERY_CREATOR, managedQuery.getId())); @@ -68,16 +74,16 @@ void doNotDeletePermissionValidReference() { @Test void doDeletePermissionInvalidReference() { - assertThat(storage.getAllExecutions()).isEmpty(); + assertThat(STORAGE.getAllExecutions()).isEmpty(); final ManagedQuery managedQuery = createManagedQuery(); // Removing the execution - storage.removeExecution(managedQuery.getId()); - User user = new User("test", "test", storage); - storage.updateUser(user); + STORAGE.removeExecution(managedQuery.getId()); + User user = new User("test", "test", STORAGE); + STORAGE.updateUser(user); user.addPermission(ExecutionPermission.onInstance(AbilitySets.QUERY_CREATOR, managedQuery.getId())); - deleteQueryPermissionsWithMissingRef(storage, storage.getAllUsers()); + deleteQueryPermissionsWithMissingRef(STORAGE, STORAGE.getAllUsers()); assertThat(user.getPermissions()).isEmpty(); @@ -85,14 +91,14 @@ void doDeletePermissionInvalidReference() { @Test void doDeletePartialPermissionWithInvalidReference() { - assertThat(storage.getAllExecutions()).isEmpty(); + assertThat(STORAGE.getAllExecutions()).isEmpty(); final ManagedQuery managedQuery1 = createManagedQuery(); final ManagedQuery managedQuery2 = createManagedQuery(); // Removing the second execution - storage.removeExecution(managedQuery2.getId()); - User user = new User("test", "test", storage); - storage.updateUser(user); + STORAGE.removeExecution(managedQuery2.getId()); + User user = new User("test", "test", STORAGE); + STORAGE.updateUser(user); user.addPermission( // Build a permission with multiple instances new WildcardPermission(List.of( @@ -100,7 +106,7 @@ void doDeletePartialPermissionWithInvalidReference() { Set.of(Ability.READ.toString().toLowerCase()), Set.of(managedQuery1.getId().toString(), managedQuery2.getId().toString())), Instant.now())); - deleteQueryPermissionsWithMissingRef(storage, storage.getAllUsers()); + deleteQueryPermissionsWithMissingRef(STORAGE, STORAGE.getAllUsers()); assertThat(user.getPermissions()).containsOnly(ExecutionPermission.onInstance(Ability.READ, managedQuery1.getId())); @@ -109,19 +115,19 @@ void doDeletePartialPermissionWithInvalidReference() { @Test void doDeletePermissionsOfOwnedReference() { - assertThat(storage.getAllExecutions()).isEmpty(); + assertThat(STORAGE.getAllExecutions()).isEmpty(); // Created owned execution final ManagedQuery managedQueryOwned = createManagedQuery(); // Setup user - User user = new User("test", "test", storage); - User user2 = new User("test2", "test2", storage); + User user = new User("test", "test", STORAGE); + User user2 = new User("test2", "test2", STORAGE); - storage.updateUser(user); + STORAGE.updateUser(user); user.addPermission(ExecutionPermission.onInstance(AbilitySets.QUERY_CREATOR, managedQueryOwned.getId())); managedQueryOwned.setOwner(user); - storage.updateExecution(managedQueryOwned); + STORAGE.updateExecution(managedQueryOwned); // Created not owned execution final ManagedQuery managedQueryNotOwned = createManagedQuery(); @@ -130,9 +136,9 @@ void doDeletePermissionsOfOwnedReference() { // Set owner managedQueryNotOwned.setOwner(user2); - storage.updateExecution(managedQueryNotOwned); + STORAGE.updateExecution(managedQueryNotOwned); - deletePermissionsOfOwnedInstances(storage, ExecutionPermission.DOMAIN.toLowerCase(), ManagedExecutionId.Parser.INSTANCE, storage::getExecution); + deletePermissionsOfOwnedInstances(STORAGE, ExecutionPermission.DOMAIN.toLowerCase(), ManagedExecutionId.Parser.INSTANCE, STORAGE::getExecution); assertThat(user.getPermissions()).containsOnly(ExecutionPermission.onInstance(Ability.READ, managedQueryNotOwned.getId())); diff --git a/backend/src/test/java/com/bakdata/conquery/tasks/QueryCleanupTaskTest.java b/backend/src/test/java/com/bakdata/conquery/tasks/QueryCleanupTaskTest.java index 10ee39bad6..7267f8750d 100644 --- a/backend/src/test/java/com/bakdata/conquery/tasks/QueryCleanupTaskTest.java +++ b/backend/src/test/java/com/bakdata/conquery/tasks/QueryCleanupTaskTest.java @@ -17,6 +17,7 @@ import com.bakdata.conquery.apiv1.query.concept.specific.CQReusedQuery; import com.bakdata.conquery.util.NonPersistentStoreFactory; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.TestInstance.Lifecycle; @@ -38,77 +39,83 @@ private ManagedQuery createManagedQuery() { managedQuery.setCreationTime(LocalDateTime.now().minus(queryExpiration).minusDays(1)); - storage.addExecution(managedQuery); + STORAGE.addExecution(managedQuery); return managedQuery; } - private MetaStorage storage = new MetaStorage(null, new NonPersistentStoreFactory(), null); + private static final MetaStorage STORAGE = new MetaStorage(null); + + @BeforeAll + public static void beforeAll() { + STORAGE.openStores(new NonPersistentStoreFactory()); + } + @AfterEach public void teardownAfterEach() { - storage.clear(); + STORAGE.clear(); } @Test void emptyIsEmpty() { - assertThat(storage.getAllExecutions()).isEmpty(); + assertThat(STORAGE.getAllExecutions()).isEmpty(); } @Test void singleUnnamed() throws Exception { - assertThat(storage.getAllExecutions()).isEmpty(); + assertThat(STORAGE.getAllExecutions()).isEmpty(); createManagedQuery(); - new QueryCleanupTask(storage, queryExpiration).execute( Map.of(QueryCleanupTask.EXPIRATION_PARAM, List.of("PT719H")), null); + new QueryCleanupTask(STORAGE, queryExpiration).execute(Map.of(QueryCleanupTask.EXPIRATION_PARAM, List.of("PT719H")), null); - assertThat(storage.getAllExecutions()).isEmpty(); + assertThat(STORAGE.getAllExecutions()).isEmpty(); } @Test void singleNamed() throws Exception { - assertThat(storage.getAllExecutions()).isEmpty(); + assertThat(STORAGE.getAllExecutions()).isEmpty(); final ManagedQuery managedQuery = createManagedQuery(); managedQuery.setLabel("test"); - new QueryCleanupTask(storage, queryExpiration).execute( Map.of(), null); + new QueryCleanupTask(STORAGE, queryExpiration).execute(Map.of(), null); - assertThat(storage.getAllExecutions()).containsExactlyInAnyOrder(managedQuery); + assertThat(STORAGE.getAllExecutions()).containsExactlyInAnyOrder(managedQuery); } @Test void singleNamedButUUID() throws Exception { - assertThat(storage.getAllExecutions()).isEmpty(); + assertThat(STORAGE.getAllExecutions()).isEmpty(); final ManagedQuery managedQuery = createManagedQuery(); managedQuery.setLabel(UUID.randomUUID().toString()); - new QueryCleanupTask(storage, queryExpiration).execute( Map.of(), null); + new QueryCleanupTask(STORAGE, queryExpiration).execute(Map.of(), null); - assertThat(storage.getAllExecutions()).isEmpty(); + assertThat(STORAGE.getAllExecutions()).isEmpty(); } @Test void reusedNoNames() throws Exception { - assertThat(storage.getAllExecutions()).isEmpty(); + assertThat(STORAGE.getAllExecutions()).isEmpty(); final ManagedQuery managedQuery = createManagedQuery(); final ManagedQuery managedQueryReused = createManagedQuery(); managedQuery.setQuery(new ConceptQuery(new CQReusedQuery(managedQueryReused.getId()))); - new QueryCleanupTask(storage, queryExpiration).execute( Map.of(), null); + new QueryCleanupTask(STORAGE, queryExpiration).execute(Map.of(), null); - assertThat(storage.getAllExecutions()).isEmpty(); + assertThat(STORAGE.getAllExecutions()).isEmpty(); } @Test void reusedBothNames() throws Exception { - assertThat(storage.getAllExecutions()).isEmpty(); + assertThat(STORAGE.getAllExecutions()).isEmpty(); final ManagedQuery managedQuery = createManagedQuery(); managedQuery.setLabel("test1"); @@ -118,15 +125,15 @@ void reusedBothNames() throws Exception { managedQuery.setQuery(new ConceptQuery(new CQReusedQuery(managedQueryReused.getId()))); - new QueryCleanupTask(storage, queryExpiration).execute(Map.of(), null); + new QueryCleanupTask(STORAGE, queryExpiration).execute(Map.of(), null); - assertThat(storage.getAllExecutions()) + assertThat(STORAGE.getAllExecutions()) .containsExactlyInAnyOrder(managedQuery, managedQueryReused); } @Test void reusedNames() throws Exception { - assertThat(storage.getAllExecutions()).isEmpty(); + assertThat(STORAGE.getAllExecutions()).isEmpty(); final ManagedQuery managedQuery = createManagedQuery(); @@ -135,14 +142,14 @@ void reusedNames() throws Exception { managedQuery.setQuery(new ConceptQuery(new CQReusedQuery(managedQueryReused.getId()))); - new QueryCleanupTask(storage, queryExpiration).execute( Map.of(), null); + new QueryCleanupTask(STORAGE, queryExpiration).execute(Map.of(), null); - assertThat(storage.getAllExecutions()).containsExactlyInAnyOrder(managedQueryReused); + assertThat(STORAGE.getAllExecutions()).containsExactlyInAnyOrder(managedQueryReused); } @Test void reusedOtherName() throws Exception { - assertThat(storage.getAllExecutions()).isEmpty(); + assertThat(STORAGE.getAllExecutions()).isEmpty(); final ManagedQuery managedQuery = createManagedQuery(); managedQuery.setLabel("test2"); @@ -151,14 +158,14 @@ void reusedOtherName() throws Exception { managedQuery.setQuery(new ConceptQuery(new CQReusedQuery(managedQueryReused.getId()))); - new QueryCleanupTask(storage, queryExpiration).execute( Map.of(), null); + new QueryCleanupTask(STORAGE, queryExpiration).execute(Map.of(), null); - assertThat(storage.getAllExecutions()).containsExactlyInAnyOrder(managedQueryReused, managedQuery); + assertThat(STORAGE.getAllExecutions()).containsExactlyInAnyOrder(managedQueryReused, managedQuery); } @Test void reusedTagged() throws Exception { - assertThat(storage.getAllExecutions()).isEmpty(); + assertThat(STORAGE.getAllExecutions()).isEmpty(); final ManagedQuery managedQuery = createManagedQuery(); @@ -167,14 +174,14 @@ void reusedTagged() throws Exception { managedQuery.setQuery(new ConceptQuery(new CQReusedQuery(managedQueryReused.getId()))); - new QueryCleanupTask(storage, queryExpiration).execute( Map.of(), null); + new QueryCleanupTask(STORAGE, queryExpiration).execute(Map.of(), null); - assertThat(storage.getAllExecutions()).containsExactlyInAnyOrder(managedQueryReused); + assertThat(STORAGE.getAllExecutions()).containsExactlyInAnyOrder(managedQueryReused); } @Test void reusedYoung() throws Exception { - assertThat(storage.getAllExecutions()).isEmpty(); + assertThat(STORAGE.getAllExecutions()).isEmpty(); final ManagedQuery managedQuery = createManagedQuery(); @@ -183,9 +190,9 @@ void reusedYoung() throws Exception { managedQuery.setQuery(new ConceptQuery(new CQReusedQuery(managedQueryReused.getId()))); - new QueryCleanupTask(storage, queryExpiration).execute( Map.of(), null); + new QueryCleanupTask(STORAGE, queryExpiration).execute(Map.of(), null); - assertThat(storage.getAllExecutions()).containsExactlyInAnyOrder(managedQueryReused); + assertThat(STORAGE.getAllExecutions()).containsExactlyInAnyOrder(managedQueryReused); } } \ No newline at end of file diff --git a/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java b/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java index 0411729930..951e34780c 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java +++ b/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java @@ -28,7 +28,6 @@ import com.bakdata.conquery.models.forms.configs.FormConfig; import com.bakdata.conquery.models.identifiable.CentralRegistry; import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; -import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.SingletonNamespaceCollection; import com.bakdata.conquery.models.worker.WorkerInformation; import com.bakdata.conquery.models.worker.WorkerToBucketsMap; @@ -107,12 +106,12 @@ public SingletonStore createStructureStore(String pathName, Sin } @Override - public IdentifiableStore> createExecutionsStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName) { + public IdentifiableStore> createExecutionsStore(CentralRegistry centralRegistry, String pathName) { return EXECUTIONS.identifiable(new NonPersistentStore(), centralRegistry); } @Override - public IdentifiableStore createFormConfigStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName) { + public IdentifiableStore createFormConfigStore(CentralRegistry centralRegistry, String pathName) { return FORM_CONFIG.identifiable(new NonPersistentStore(), centralRegistry); } @@ -140,6 +139,6 @@ public SingletonStore createPrimaryDictionaryStore(String pathName, * @implNote intended for Unit-tests */ public MetaStorage createMetaStorage() { - return new MetaStorage(null, this, null); + return new MetaStorage(null); } } diff --git a/backend/src/test/java/com/bakdata/conquery/util/support/StandaloneSupport.java b/backend/src/test/java/com/bakdata/conquery/util/support/StandaloneSupport.java index 6f750f78cb..bd3dc3a04f 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/support/StandaloneSupport.java +++ b/backend/src/test/java/com/bakdata/conquery/util/support/StandaloneSupport.java @@ -20,6 +20,7 @@ import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.datasets.Dataset; +import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.resources.admin.rest.AdminDatasetProcessor; import com.bakdata.conquery.resources.admin.rest.AdminProcessor; @@ -91,6 +92,10 @@ public NamespaceStorage getNamespaceStorage() { return testConquery.getStandaloneCommand().getManager().getDatasetRegistry().get(dataset.getId()).getStorage(); } + public DatasetRegistry getDatasetRegistry() { + return testConquery.getStandaloneCommand().getManager().getDatasetRegistry(); + } + public List getShardNodes() { return testConquery.getStandaloneCommand().getShardNodes(); } From 849164f98a3ff55ba2f8ab685d4f6bc99c460bea Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Thu, 23 Sep 2021 13:37:20 +0200 Subject: [PATCH 21/35] remove validator from MetaStorage --- .../bakdata/conquery/commands/ManagerNode.java | 2 +- .../com/bakdata/conquery/commands/ShardNode.java | 10 +++++++--- .../conquery/io/storage/ConqueryStorage.java | 1 - .../bakdata/conquery/io/storage/MetaStorage.java | 6 ------ .../ModificationShieldedWorkerStorage.java | 5 ----- .../conquery/io/storage/NamespacedStorage.java | 5 +---- .../conquery/models/config/XodusStoreFactory.java | 9 +++++---- .../models/datasets/concepts/Concept.java | 2 +- .../datasets/concepts/tree/TreeConcept.java | 13 ++++++++++++- .../models/identifiable/CentralRegistry.java | 5 ++++- .../namespaces/specific/UpdateConcept.java | 15 ++++++++++++++- .../bakdata/conquery/models/worker/Workers.java | 5 +---- .../conquery/api/StoredQueriesProcessorTest.java | 1 + .../conquery/api/form/config/FormConfigTest.java | 2 +- .../conquery/integration/IntegrationTests.java | 1 + .../integration/json/filter/FilterTest.java | 3 ++- .../xodus/stores/SerializingStoreDumpTest.java | 2 +- .../conquery/models/SerializationTests.java | 5 +---- .../conquery/models/auth/CopyUserTest.java | 2 +- .../conquery/models/auth/LocalAuthRealmTest.java | 3 +-- .../datasets/concepts/tree/GroovyIndexedTest.java | 4 ++-- .../models/execution/DefaultLabelTest.java | 3 +-- .../models/query/DefaultColumnNameTest.java | 4 +++- .../conquery/tasks/PermissionCleanupTaskTest.java | 2 +- .../conquery/tasks/QueryCleanupTaskTest.java | 2 +- .../conquery/util/NonPersistentStoreFactory.java | 4 +++- 26 files changed, 66 insertions(+), 50 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java b/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java index 2db0d5b3be..cd2e21a83b 100644 --- a/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java +++ b/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java @@ -105,7 +105,7 @@ public void run(ConqueryConfig config, Environment environment) throws Interrupt // Instantiate DatasetRegistry and MetaStorage so they are ready for injection into the object mapper (API + Storage) datasetRegistry = new DatasetRegistry(config.getCluster().getEntityBucketSize()); - storage = new MetaStorage(validator); + storage = new MetaStorage(); datasetRegistry.injectInto(environment.getObjectMapper()); storage.injectInto(environment.getObjectMapper()); diff --git a/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java b/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java index 9fd0b3a491..33bce6decc 100644 --- a/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java +++ b/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java @@ -8,6 +8,7 @@ import javax.validation.Validator; import com.bakdata.conquery.io.jackson.Jackson; +import com.bakdata.conquery.io.jackson.MutableInjectableValues; import com.bakdata.conquery.io.mina.BinaryJacksonCoder; import com.bakdata.conquery.io.mina.CQProtocolCodecFilter; import com.bakdata.conquery.io.mina.ChunkReader; @@ -98,11 +99,14 @@ protected void run(Environment environment, Namespace namespace, ConqueryConfig scheduler.scheduleAtFixedRate(this::reportJobManagerStatus, 30, 1, TimeUnit.SECONDS); + + final ObjectMapper binaryMapper = config.configureObjectMapper(Jackson.BINARY_MAPPER); + ((MutableInjectableValues) binaryMapper.getInjectableValues()).add(Validator.class, environment.getValidator()); + workers = new Workers( getConfig().getQueries().getExecutionPool(), - config.configureObjectMapper(Jackson.MAPPER), - config.configureObjectMapper(Jackson.BINARY_MAPPER) - , getConfig().getCluster().getEntityBucketSize() + binaryMapper, + getConfig().getCluster().getEntityBucketSize() ); final Collection workerStorages = config.getStorage().loadWorkerStorages(); diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/ConqueryStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/ConqueryStorage.java index 1994729280..ce403fc940 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/ConqueryStorage.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/ConqueryStorage.java @@ -9,7 +9,6 @@ public interface ConqueryStorage extends Closeable { - Validator getValidator(); CentralRegistry getCentralRegistry(); void loadData(); diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java index fbb6b0a782..d44e065b4d 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java @@ -37,12 +37,6 @@ public class MetaStorage implements ConqueryStorage, Injectable { @Getter protected final CentralRegistry centralRegistry = new CentralRegistry(); - @Getter - protected final Validator validator; - - public MetaStorage(Validator validator) { - this.validator = validator; - } public void openStores(StoreFactory storageFactory) { authUser = storageFactory.createUserStore(centralRegistry, "meta", this); diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/ModificationShieldedWorkerStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/ModificationShieldedWorkerStorage.java index 5a237a9e86..a509f86a2c 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/ModificationShieldedWorkerStorage.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/ModificationShieldedWorkerStorage.java @@ -27,11 +27,6 @@ public class ModificationShieldedWorkerStorage { private final WorkerStorage delegate; - public Validator getValidator() { - return delegate.getValidator(); - } - - public CentralRegistry getCentralRegistry() { return delegate.getCentralRegistry(); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java index 807a728128..8ee8f543d8 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java @@ -7,7 +7,6 @@ import javax.validation.Validator; -import com.bakdata.conquery.ConqueryConstants; import com.bakdata.conquery.io.storage.xodus.stores.SingletonStore; import com.bakdata.conquery.models.datasets.concepts.Concept; import com.bakdata.conquery.models.datasets.concepts.Connector; @@ -19,8 +18,6 @@ import com.bakdata.conquery.models.datasets.SecondaryIdDescription; import com.bakdata.conquery.models.datasets.Table; import com.bakdata.conquery.models.dictionary.Dictionary; -import com.bakdata.conquery.models.dictionary.EncodedDictionary; -import com.bakdata.conquery.models.events.stores.specific.string.StringTypeEncoded; import com.bakdata.conquery.models.identifiable.CentralRegistry; import com.bakdata.conquery.models.identifiable.ids.specific.ConceptId; import com.bakdata.conquery.models.identifiable.ids.specific.DictionaryId; @@ -150,7 +147,7 @@ private void decorateConceptStore(IdentifiableStore> store) { concept.setDataset(dataset.get()); - concept.initElements(getValidator()); + concept.initElements(); concept.getSelects().forEach(centralRegistry::register); for (Connector connector : concept.getConnectors()) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java index b189958659..bd8bc81f1a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java @@ -27,6 +27,7 @@ import com.bakdata.conquery.commands.ShardNode; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.jackson.InternalOnly; +import com.bakdata.conquery.io.jackson.Jackson; import com.bakdata.conquery.io.storage.IdentifiableStore; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.io.storage.NamespaceStorage; @@ -149,7 +150,7 @@ public class XodusStoreFactory implements StoreFactory { private transient Validator validator; @JsonIgnore - private transient ObjectMapper objectMapper; + private transient ObjectMapper objectMapper = Jackson.BINARY_MAPPER.copy(); @JsonIgnore private final BiMap activeEnvironments = HashBiMap.create(); @@ -182,13 +183,13 @@ private void configureMapper(ConqueryConfig config) { @Override @SneakyThrows public Collection loadNamespaceStorages() { - return loadNamespacedStores("dataset_", (elements) -> new NamespaceStorage(validator, this, elements), NAMESPACE_STORES); + return loadNamespacedStores("dataset_", (storesToTest) -> new NamespaceStorage(validator, this, storesToTest), NAMESPACE_STORES); } @Override @SneakyThrows public Collection loadWorkerStorages() { - return loadNamespacedStores("worker_", (elements) -> new WorkerStorage(validator, this, elements), WORKER_STORES); + return loadNamespacedStores("worker_", (storesToTest) -> new WorkerStorage(validator, this, storesToTest), WORKER_STORES); } @@ -473,7 +474,7 @@ public Store < KEY, VALUE > createStore(Environment environment, V new XodusStore(environment, storeInfo.getName(), this::closeStore, this::removeStore), validator, - objectMapper, + objectMapper.copy(), storeInfo.getKeyType(), storeInfo.getValueType(), this.isValidateOnWrite(), diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Concept.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Concept.java index d4eed3f9c6..bbb6e86412 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Concept.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Concept.java @@ -64,7 +64,7 @@ public List
createTableStore(CentralRegistry centralRegistry, String pathName) { - return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, TABLES), centralRegistry); + return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, TABLES, centralRegistry.injectIntoNew(objectMapper)), centralRegistry); } @Override public IdentifiableStore createDictionaryStore(CentralRegistry centralRegistry, String pathName) { final Environment environment = findEnvironment(pathName); - final SingletonNamespaceCollection namespaceCollection = new SingletonNamespaceCollection(centralRegistry); - final BigStore, Dictionary> bigStore; synchronized (openStoresInEnv) { @@ -300,7 +298,7 @@ public IdentifiableStore createDictionaryStore(CentralRegistry centr DICTIONARIES.storeInfo(), this::closeStore, this::removeStore, - namespaceCollection.injectIntoNew(objectMapper) + centralRegistry.injectIntoNew(objectMapper) ); } @@ -312,27 +310,27 @@ public IdentifiableStore createDictionaryStore(CentralRegistry centr @Override public IdentifiableStore> createConceptStore(CentralRegistry centralRegistry, String pathName) { - return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, CONCEPTS), centralRegistry); + return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, CONCEPTS, centralRegistry.injectIntoNew(objectMapper)), centralRegistry); } @Override public IdentifiableStore createImportStore(CentralRegistry centralRegistry, String pathName) { - return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, IMPORTS), centralRegistry); + return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, IMPORTS, centralRegistry.injectIntoNew(objectMapper)), centralRegistry); } @Override public IdentifiableStore createCBlockStore(CentralRegistry centralRegistry, String pathName) { - return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, C_BLOCKS), centralRegistry); + return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, C_BLOCKS, centralRegistry.injectIntoNew(objectMapper)), centralRegistry); } @Override public IdentifiableStore createBucketStore(CentralRegistry centralRegistry, String pathName) { - return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, BUCKETS), centralRegistry); + return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, BUCKETS, centralRegistry.injectIntoNew(objectMapper)), centralRegistry); } @Override public SingletonStore createWorkerInformationStore(String pathName) { - return StoreMappings.singleton(createStore(findEnvironment(pathName), validator, WORKER)); + return StoreMappings.singleton(createStore(findEnvironment(pathName), validator, WORKER, objectMapper)); } @Override @@ -349,43 +347,43 @@ public SingletonStore createIdMappingStore(String pathName) { @Override public SingletonStore createWorkerToBucketsStore(String pathName) { - return StoreMappings.singleton(createStore(findEnvironment(pathName), validator, WORKER_TO_BUCKETS)); + return StoreMappings.singleton(createStore(findEnvironment(pathName), validator, WORKER_TO_BUCKETS, objectMapper)); } @Override - public SingletonStore createStructureStore(String pathName, SingletonNamespaceCollection centralRegistry) { - return StoreMappings.singleton(createStore(findEnvironment(pathName), validator, STRUCTURE), centralRegistry); + public SingletonStore createStructureStore(String pathName, CentralRegistry centralRegistry) { + return StoreMappings.singleton(createStore(findEnvironment(pathName), validator, STRUCTURE, centralRegistry.injectIntoNew(objectMapper))); } @Override public IdentifiableStore> createExecutionsStore(CentralRegistry centralRegistry, String pathName) { - return StoreMappings.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "executions")), validator, EXECUTIONS), centralRegistry); + return StoreMappings.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "executions")), validator, EXECUTIONS, centralRegistry.injectIntoNew(objectMapper)), centralRegistry); } @Override public IdentifiableStore createFormConfigStore(CentralRegistry centralRegistry, String pathName) { - return StoreMappings.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "formConfigs")), validator, FORM_CONFIG), centralRegistry); + return StoreMappings.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "formConfigs")), validator, FORM_CONFIG, centralRegistry.injectIntoNew(objectMapper)), centralRegistry); } @Override public IdentifiableStore createUserStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage) { - return StoreMappings.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "users")), validator, AUTH_USER), centralRegistry); + return StoreMappings.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "users")), validator, AUTH_USER, centralRegistry.injectIntoNew(objectMapper)), centralRegistry); } @Override public IdentifiableStore createRoleStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage) { - return StoreMappings.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "roles")), validator, AUTH_ROLE), centralRegistry); + return StoreMappings.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "roles")), validator, AUTH_ROLE, centralRegistry.injectIntoNew(objectMapper)),centralRegistry); } @Override public IdentifiableStore createGroupStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage) { - return StoreMappings.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "groups")), validator, AUTH_GROUP), centralRegistry); + return StoreMappings.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "groups")), validator, AUTH_GROUP, centralRegistry.injectIntoNew(objectMapper)),centralRegistry); } @Override - public SingletonStore createPrimaryDictionaryStore(String pathName, SingletonNamespaceCollection namespaceCollection) { - return StoreMappings.singleton(createStore(findEnvironment(pathName), validator, PRIMARY_DICTIONARY), namespaceCollection); + public SingletonStore createPrimaryDictionaryStore(String pathName, CentralRegistry centralRegistry) { + return StoreMappings.singleton(createStore(findEnvironment(pathName), validator, PRIMARY_DICTIONARY, centralRegistry.injectIntoNew(objectMapper))); } private File resolveSubDir(String... subdirs) { @@ -466,15 +464,14 @@ public static void removeEnvironmentHook(Environment env) { } } - public Store < KEY, VALUE > createStore(Environment environment, Validator validator, StoreMappings storeId){ + public Store < KEY, VALUE > createStore(Environment environment, Validator validator, StoreMappings storeId, ObjectMapper objectMapper){ final StoreInfo storeInfo = storeId.storeInfo(); synchronized (openStoresInEnv) { return new CachedStore<>( new SerializingStore<>( - new XodusStore(environment, storeInfo.getName(), this::closeStore, this::removeStore), validator, - objectMapper.copy(), + objectMapper, storeInfo.getKeyType(), storeInfo.getValueType(), this.isValidateOnWrite(), diff --git a/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java b/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java index 5577c9a024..694143b30b 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java +++ b/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java @@ -66,12 +66,7 @@ @Slf4j public class SerializationTests { - private final static MetaStorage STORAGE = new MetaStorage(); - - @BeforeAll - public static void beforAll() { - STORAGE.openStores(new NonPersistentStoreFactory()); - } + private final static MetaStorage STORAGE = new NonPersistentStoreFactory().createMetaStorage(); @Test public void dataset() throws IOException, JSONException { diff --git a/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStore.java b/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStore.java index 4385c410f1..ca162db402 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStore.java +++ b/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStore.java @@ -62,11 +62,6 @@ public Collection getAll() { return map.values(); } - @Override - public void inject(Injectable injectable) { - // Don't inject here since there is no serdes when non persistent - } - @Override public Collection getAllKeys() { return map.keySet(); diff --git a/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java b/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java index 7aae5fed4c..27037de505 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java +++ b/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java @@ -9,6 +9,7 @@ import com.bakdata.conquery.io.storage.IdentifiableStore; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.io.storage.NamespaceStorage; +import com.bakdata.conquery.io.storage.StoreMappings; import com.bakdata.conquery.io.storage.WorkerStorage; import com.bakdata.conquery.io.storage.xodus.stores.SingletonStore; import com.bakdata.conquery.models.auth.entities.Group; @@ -47,92 +48,92 @@ public Collection loadWorkerStorages() { @Override public SingletonStore createDatasetStore(String pathName) { - return DATASET.singleton(new NonPersistentStore()); + return StoreMappings.singleton(new NonPersistentStore<>()); } @Override public IdentifiableStore createSecondaryIdDescriptionStore(CentralRegistry centralRegistry, String pathName) { - return SECONDARY_IDS.identifiable(new NonPersistentStore(), centralRegistry); + return StoreMappings.identifiable(new NonPersistentStore<>(), centralRegistry); } @Override public IdentifiableStore
createTableStore(CentralRegistry centralRegistry, String pathName) { - return TABLES.identifiable(new NonPersistentStore(), centralRegistry); + return StoreMappings.identifiable(new NonPersistentStore<>(), centralRegistry); } @Override public IdentifiableStore createDictionaryStore(CentralRegistry centralRegistry, String pathName) { - return DICTIONARIES.identifiable(new NonPersistentStore(), centralRegistry); + return StoreMappings.identifiable(new NonPersistentStore<>(), centralRegistry); } @Override public IdentifiableStore> createConceptStore(CentralRegistry centralRegistry, String pathName) { - return CONCEPTS.identifiable(new NonPersistentStore(), centralRegistry); + return StoreMappings.identifiable(new NonPersistentStore(), centralRegistry); } @Override public IdentifiableStore createImportStore(CentralRegistry centralRegistry, String pathName) { - return IMPORTS.identifiable(new NonPersistentStore(), centralRegistry); + return StoreMappings.identifiable(new NonPersistentStore<>(), centralRegistry); } @Override public IdentifiableStore createCBlockStore(CentralRegistry centralRegistry, String pathName) { - return C_BLOCKS.identifiable(new NonPersistentStore(), centralRegistry); + return StoreMappings.identifiable(new NonPersistentStore(), centralRegistry); } @Override public IdentifiableStore createBucketStore(CentralRegistry centralRegistry, String pathName) { - return BUCKETS.identifiable(new NonPersistentStore(), centralRegistry); + return StoreMappings.identifiable(new NonPersistentStore<>(), centralRegistry); } @Override public SingletonStore createWorkerInformationStore(String pathName) { - return WORKER.singleton(new NonPersistentStore()); + return WORKER.singleton(new NonPersistentStore<>()); } @Override public SingletonStore createIdMappingStore(String pathName) { - return ID_MAPPING.singleton(new NonPersistentStore()); + return ID_MAPPING.singleton(new NonPersistentStore<>()); } @Override public SingletonStore createWorkerToBucketsStore(String pathName) { - return WORKER_TO_BUCKETS.singleton(new NonPersistentStore()); + return WORKER_TO_BUCKETS.singleton(new NonPersistentStore<>()); } @Override - public SingletonStore createStructureStore(String pathName, SingletonNamespaceCollection centralRegistry) { - return STRUCTURE.singleton(new NonPersistentStore(), centralRegistry); + public SingletonStore createStructureStore(String pathName, CentralRegistry centralRegistry) { + return StoreMappings.singleton(new NonPersistentStore<>()); } @Override public IdentifiableStore> createExecutionsStore(CentralRegistry centralRegistry, String pathName) { - return EXECUTIONS.identifiable(new NonPersistentStore(), centralRegistry); + return StoreMappings.identifiable(new NonPersistentStore<>(), centralRegistry); } @Override public IdentifiableStore createFormConfigStore(CentralRegistry centralRegistry, String pathName) { - return FORM_CONFIG.identifiable(new NonPersistentStore(), centralRegistry); + return StoreMappings.identifiable(new NonPersistentStore<>(), centralRegistry); } @Override public IdentifiableStore createUserStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage) { - return AUTH_USER.identifiable(new NonPersistentStore(), centralRegistry); + return StoreMappings.identifiable(new NonPersistentStore<>(), centralRegistry); } @Override public IdentifiableStore createRoleStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage) { - return AUTH_ROLE.identifiable(new NonPersistentStore(), centralRegistry); + return StoreMappings.identifiable(new NonPersistentStore<>(), centralRegistry); } @Override public IdentifiableStore createGroupStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage) { - return AUTH_GROUP.identifiable(new NonPersistentStore(), centralRegistry); + return StoreMappings.identifiable(new NonPersistentStore<>(), centralRegistry); } @Override - public SingletonStore createPrimaryDictionaryStore(String pathName, SingletonNamespaceCollection namespaceCollection) { - return DICTIONARIES.singleton(new NonPersistentStore<>()); + public SingletonStore createPrimaryDictionaryStore(String pathName, CentralRegistry centralRegistry) { + return StoreMappings.singleton(new NonPersistentStore<>()); } /** From 417c0b188fbbd53520505d24ed73393cf9cd0a6d Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Thu, 23 Sep 2021 14:56:55 +0200 Subject: [PATCH 25/35] fix GroovyIndexedTest: prep serdes --- .../conquery/io/jackson/Injectable.java | 4 ++-- .../concepts/tree/GroovyIndexedTest.java | 20 ++++++++++++++++--- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/Injectable.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/Injectable.java index 717eb08efe..3cc0a8556c 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jackson/Injectable.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jackson/Injectable.java @@ -29,12 +29,12 @@ default ObjectMapper injectIntoNew(ObjectMapper mapper) { .setInjectableValues(inject(new MutableInjectableValues())); } - default void injectInto(ObjectMapper mapper) { + default ObjectMapper injectInto(ObjectMapper mapper) { // If is already MutableInjectable, add my values to other, else begin from scratch. if (mapper.getInjectableValues() instanceof MutableInjectableValues) { mapper.setInjectableValues(inject(((MutableInjectableValues) mapper.getInjectableValues()))); - return; + return mapper; } throw new IllegalStateException("Cannot add additional injectables if the mapper does not provide MutableInjectableValues"); } diff --git a/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/tree/GroovyIndexedTest.java b/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/tree/GroovyIndexedTest.java index bbb4eccc5d..55f9df7d3b 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/tree/GroovyIndexedTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/tree/GroovyIndexedTest.java @@ -9,7 +9,11 @@ import java.util.function.Supplier; import java.util.stream.Stream; +import javax.validation.Validator; + +import com.bakdata.conquery.io.jackson.Injectable; import com.bakdata.conquery.io.jackson.Jackson; +import com.bakdata.conquery.io.jackson.MutableInjectableValues; import com.bakdata.conquery.models.datasets.concepts.Concept; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.Dataset; @@ -20,6 +24,7 @@ import com.bakdata.conquery.models.identifiable.CentralRegistry; import com.bakdata.conquery.models.worker.SingletonNamespaceCollection; import com.bakdata.conquery.util.CalculatedValue; +import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.databind.node.ObjectNode; import com.github.powerlibraries.io.In; import io.dropwizard.jersey.validation.Validators; @@ -81,16 +86,25 @@ public static void init() throws IOException, JSONException, ConfigurationExcept registry.register(table); registry.register(column); - // load tree twice to to avoid references - indexedConcept = new SingletonNamespaceCollection(registry).injectIntoNew(dataset.injectIntoNew(Jackson.MAPPER.readerFor(Concept.class))).readValue(node); + // Prepare Serdes injections + final Validator validator = Validators.newValidator(); + final ObjectReader conceptReader = new Injectable(){ + @Override + public MutableInjectableValues inject(MutableInjectableValues values) { + return values.add(Validator.class, validator); + } + }.injectInto(registry.injectIntoNew(dataset.injectIntoNew(Jackson.MAPPER))).readerFor(Concept.class); + + // load tree twice to to avoid references + indexedConcept = conceptReader.readValue(node); indexedConcept.setDataset(dataset); indexedConcept.initElements(); TreeChildPrefixIndex.putIndexInto(indexedConcept); - oldConcept = new SingletonNamespaceCollection(registry).injectIntoNew(dataset.injectIntoNew(Jackson.MAPPER.readerFor(Concept.class))).readValue(node); + oldConcept = conceptReader.readValue(node); oldConcept.setDataset(dataset); oldConcept.initElements(); From a91e91bf4a3e933867818fd3d3e23bc0c212bff7 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Thu, 23 Sep 2021 14:59:44 +0200 Subject: [PATCH 26/35] add comment --- .../main/java/com/bakdata/conquery/io/jackson/Injectable.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/Injectable.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/Injectable.java index 3cc0a8556c..2c49abf85f 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jackson/Injectable.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jackson/Injectable.java @@ -24,7 +24,7 @@ default ObjectMapper injectIntoNew(ObjectMapper mapper) { return mapper.copy() .setInjectableValues(inject(((MutableInjectableValues) mapper.getInjectableValues()).copy())); } - // TODO unsuse + // TODO unsure if overriding is expected here from the user return mapper.copy() .setInjectableValues(inject(new MutableInjectableValues())); } From 0d157fa0b546b37c7ca8aa67ed46f39655dd31a3 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Mon, 27 Sep 2021 17:49:39 +0200 Subject: [PATCH 27/35] add documentation --- .../com/bakdata/conquery/apiv1/auth/ProtoUser.java | 2 +- .../com/bakdata/conquery/io/jackson/Injectable.java | 11 +++++++++++ .../basic/UserAuthenticationManagementProcessor.java | 5 +---- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/auth/ProtoUser.java b/backend/src/main/java/com/bakdata/conquery/apiv1/auth/ProtoUser.java index 584370c805..9678440017 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/auth/ProtoUser.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/auth/ProtoUser.java @@ -63,7 +63,7 @@ public User createOrOverwriteUser(@NonNull MetaStorage storage) { } @JsonIgnore - public UserId getId() { + public UserId createId() { return new UserId(name); } } diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/Injectable.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/Injectable.java index 2c49abf85f..5e46b3a2dd 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jackson/Injectable.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jackson/Injectable.java @@ -8,6 +8,9 @@ */ public interface Injectable { + /** + * See {@link Injectable#injectIntoNew(ObjectMapper)} + */ default ObjectReader injectIntoNew(ObjectReader reader) { // If is already MutableInjectable, add my values to other, else begin from scratch. if (reader.getInjectableValues() instanceof MutableInjectableValues) { @@ -17,6 +20,11 @@ default ObjectReader injectIntoNew(ObjectReader reader) { return reader.with(inject(new MutableInjectableValues())); } + /** + * Creates a copy of the provided mapper and its injected values and adds the caller to the new copy + * @param mapper the blueprint mapper to use which remains untouched + * @return a new mapper with this injected + */ default ObjectMapper injectIntoNew(ObjectMapper mapper) { // If is already MutableInjectable, add my values to other, else begin from scratch. @@ -29,6 +37,9 @@ default ObjectMapper injectIntoNew(ObjectMapper mapper) { .setInjectableValues(inject(new MutableInjectableValues())); } + /** + * Injects this to the provided mapper and returns the mapper + */ default ObjectMapper injectInto(ObjectMapper mapper) { // If is already MutableInjectable, add my values to other, else begin from scratch. diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/basic/UserAuthenticationManagementProcessor.java b/backend/src/main/java/com/bakdata/conquery/models/auth/basic/UserAuthenticationManagementProcessor.java index 88e2699219..feece2eb8f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/basic/UserAuthenticationManagementProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/basic/UserAuthenticationManagementProcessor.java @@ -2,7 +2,6 @@ import com.bakdata.conquery.apiv1.auth.ProtoUser; import com.bakdata.conquery.io.storage.MetaStorage; -import com.bakdata.conquery.models.auth.AuthorizationController; import com.bakdata.conquery.models.auth.AuthorizationHelper; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.identifiable.ids.specific.UserId; @@ -10,8 +9,6 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import java.util.Optional; - /** * Business logic for the {@link UserAuthenticationManagementResource}. */ @@ -24,7 +21,7 @@ public class UserAuthenticationManagementProcessor { public boolean tryRegister(ProtoUser pUser) { // Throws an exception if it would override the existing user - final UserId id = pUser.getId(); + final UserId id = pUser.createId(); User user = storage.getUser(id); if (user == null) { log.warn("Unable to add new user {}. Probably already existed.", pUser); From d93c09774be01b6fa04425522c07c63ee9e783f6 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 28 Sep 2021 13:37:48 +0200 Subject: [PATCH 28/35] fix loading of executions and FormConfigs --- .../com/bakdata/conquery/commands/ManagerNode.java | 2 +- .../bakdata/conquery/commands/StandaloneCommand.java | 10 ++++++++++ .../com/bakdata/conquery/io/storage/MetaStorage.java | 7 +++++-- .../bakdata/conquery/models/config/StoreFactory.java | 5 ++--- .../conquery/models/config/XodusStoreFactory.java | 9 +++++---- .../conquery/api/form/config/FormConfigTest.java | 4 +--- .../conquery/integration/tests/RestartTest.java | 4 ++++ .../storage/xodus/stores/SerializingStoreDumpTest.java | 2 +- .../com/bakdata/conquery/models/auth/CopyUserTest.java | 3 +-- .../conquery/models/execution/DefaultLabelTest.java | 2 +- .../conquery/tasks/PermissionCleanupTaskTest.java | 7 +------ .../bakdata/conquery/tasks/QueryCleanupTaskTest.java | 7 +------ .../conquery/util/NonPersistentStoreFactory.java | 8 ++++---- .../bakdata/conquery/util/support/TestConquery.java | 3 +-- 14 files changed, 38 insertions(+), 35 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java b/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java index cd2e21a83b..b1df5682fb 100644 --- a/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java +++ b/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java @@ -105,7 +105,7 @@ public void run(ConqueryConfig config, Environment environment) throws Interrupt // Instantiate DatasetRegistry and MetaStorage so they are ready for injection into the object mapper (API + Storage) datasetRegistry = new DatasetRegistry(config.getCluster().getEntityBucketSize()); - storage = new MetaStorage(); + storage = new MetaStorage(datasetRegistry); datasetRegistry.injectInto(environment.getObjectMapper()); storage.injectInto(environment.getObjectMapper()); diff --git a/backend/src/main/java/com/bakdata/conquery/commands/StandaloneCommand.java b/backend/src/main/java/com/bakdata/conquery/commands/StandaloneCommand.java index 54683dd211..afd84212c5 100644 --- a/backend/src/main/java/com/bakdata/conquery/commands/StandaloneCommand.java +++ b/backend/src/main/java/com/bakdata/conquery/commands/StandaloneCommand.java @@ -18,6 +18,7 @@ import io.dropwizard.setup.Bootstrap; import io.dropwizard.setup.Environment; import lombok.Getter; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import net.sourceforge.argparse4j.inf.Namespace; @@ -131,4 +132,13 @@ protected void startStandalone(Environment environment, Namespace namespace, Con ConqueryMDC.setLocation(null); super.run(environment, namespace, config); } + + @SneakyThrows + public void shutdown(){ + manager.stop(); + for (ShardNode shardNode : shardNodes) { + shardNode.stop(); + } + + } } diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java index d44e065b4d..7051430ec4 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java @@ -22,10 +22,12 @@ import com.bakdata.conquery.models.worker.DatasetRegistry; import com.google.common.base.Preconditions; import lombok.Getter; +import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @Slf4j +@RequiredArgsConstructor public class MetaStorage implements ConqueryStorage, Injectable { private IdentifiableStore> executions; @@ -37,14 +39,15 @@ public class MetaStorage implements ConqueryStorage, Injectable { @Getter protected final CentralRegistry centralRegistry = new CentralRegistry(); + protected final DatasetRegistry datasetRegistry; public void openStores(StoreFactory storageFactory) { authUser = storageFactory.createUserStore(centralRegistry, "meta", this); authRole = storageFactory.createRoleStore(centralRegistry, "meta", this); authGroup = storageFactory.createGroupStore(centralRegistry, "meta", this); // Executions depend on users - executions = storageFactory.createExecutionsStore(centralRegistry, "meta"); - formConfigs = storageFactory.createFormConfigStore(centralRegistry, "meta"); + executions = storageFactory.createExecutionsStore(centralRegistry, datasetRegistry, "meta"); + formConfigs = storageFactory.createFormConfigStore(centralRegistry, datasetRegistry, "meta"); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java index 5159a0a3e3..d2e29ce3cf 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java @@ -27,7 +27,6 @@ import com.bakdata.conquery.models.identifiable.CentralRegistry; import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; import com.bakdata.conquery.models.worker.DatasetRegistry; -import com.bakdata.conquery.models.worker.SingletonNamespaceCollection; import com.bakdata.conquery.models.worker.WorkerInformation; import com.bakdata.conquery.models.worker.WorkerToBucketsMap; import com.fasterxml.jackson.annotation.JsonTypeInfo; @@ -62,8 +61,8 @@ public interface StoreFactory { SingletonStore createStructureStore(String pathName, CentralRegistry centralRegistry); // MetaStorage - IdentifiableStore> createExecutionsStore(CentralRegistry centralRegistry, String pathName); - IdentifiableStore createFormConfigStore(CentralRegistry centralRegistry, String pathName); + IdentifiableStore> createExecutionsStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName); + IdentifiableStore createFormConfigStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName); IdentifiableStore createUserStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage); IdentifiableStore createRoleStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage); IdentifiableStore createGroupStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage); diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java index 72c0d3f2c0..38146cc545 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java @@ -59,6 +59,7 @@ import com.bakdata.conquery.models.identifiable.CentralRegistry; import com.bakdata.conquery.models.identifiable.ids.IId; import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; +import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.WorkerInformation; import com.bakdata.conquery.models.worker.WorkerToBucketsMap; import com.bakdata.conquery.util.io.ConqueryMDC; @@ -365,13 +366,13 @@ public SingletonStore createStructureStore(String pathName, Cen } @Override - public IdentifiableStore> createExecutionsStore(CentralRegistry centralRegistry, String pathName) { - return StoreMappings.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "executions")), validator, EXECUTIONS, centralRegistry.injectIntoNew(objectMapper)), centralRegistry); + public IdentifiableStore> createExecutionsStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName) { + return StoreMappings.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "executions")), validator, EXECUTIONS, datasetRegistry.injectInto(centralRegistry.injectIntoNew(objectMapper))), centralRegistry); } @Override - public IdentifiableStore createFormConfigStore(CentralRegistry centralRegistry, String pathName) { - return StoreMappings.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "formConfigs")), validator, FORM_CONFIG, centralRegistry.injectIntoNew(objectMapper)), centralRegistry); + public IdentifiableStore createFormConfigStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName) { + return StoreMappings.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "formConfigs")), validator, FORM_CONFIG, datasetRegistry.injectInto(centralRegistry.injectIntoNew(objectMapper))), centralRegistry); } @Override diff --git a/backend/src/test/java/com/bakdata/conquery/api/form/config/FormConfigTest.java b/backend/src/test/java/com/bakdata/conquery/api/form/config/FormConfigTest.java index 2dec54032b..bd76bc3176 100644 --- a/backend/src/test/java/com/bakdata/conquery/api/form/config/FormConfigTest.java +++ b/backend/src/test/java/com/bakdata/conquery/api/form/config/FormConfigTest.java @@ -117,9 +117,7 @@ else if (id.equals(datasetId1)) { when(namespacesMock.injectIntoNew(any(ObjectMapper.class))).thenCallRealMethod(); when(namespacesMock.inject(any(MutableInjectableValues.class))).thenCallRealMethod(); - storage = new MetaStorage(); - storage.openStores(new NonPersistentStoreFactory()); - + storage = new NonPersistentStoreFactory().createMetaStorage(); ((MutableInjectableValues)FormConfigProcessor.getMAPPER().getInjectableValues()) .add(IdResolveContext.class, namespacesMock); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/RestartTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/RestartTest.java index b8936b10be..654baf8a96 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/RestartTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/RestartTest.java @@ -61,6 +61,8 @@ public void execute(String name, TestConquery testConquery) throws Exception { test.executeTest(conquery); + final int numberOfExecutions = conquery.getMetaStorage().getAllExecutions().size(); + // IDMapping Testing NamespaceStorage namespaceStorage = conquery.getNamespaceStorage(); @@ -145,6 +147,8 @@ public void execute(String name, TestConquery testConquery) throws Exception { DatasetRegistry datasetRegistry = support.getDatasetsProcessor().getDatasetRegistry(); + assertThat(support.getMetaStorage().getAllExecutions().size()).as("Executions after restart").isEqualTo(numberOfExecutions); + test.executeTest(support); {// Auth actual tests diff --git a/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStoreDumpTest.java b/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStoreDumpTest.java index 824f61a3a0..965a7ae79b 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStoreDumpTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStoreDumpTest.java @@ -37,7 +37,7 @@ public class SerializingStoreDumpTest { public static final StoreInfo USER_STORE_ID = StoreMappings.AUTH_USER.storeInfo(); - private final static MetaStorage STORAGE = new MetaStorage(); + private final static MetaStorage STORAGE = new NonPersistentStoreFactory().createMetaStorage(); private File tmpDir; private Environment env; private XodusStoreFactory config; diff --git a/backend/src/test/java/com/bakdata/conquery/models/auth/CopyUserTest.java b/backend/src/test/java/com/bakdata/conquery/models/auth/CopyUserTest.java index 8be2d2c23d..0eaa449e02 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/auth/CopyUserTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/auth/CopyUserTest.java @@ -20,8 +20,7 @@ void testUserCopy(){ final DatasetRegistry registry = new DatasetRegistry(0); - MetaStorage storage = new MetaStorage(); - storage.openStores(new NonPersistentStoreFactory()); + MetaStorage storage = new NonPersistentStoreFactory().createMetaStorage(); registry.setMetaStorage(storage); diff --git a/backend/src/test/java/com/bakdata/conquery/models/execution/DefaultLabelTest.java b/backend/src/test/java/com/bakdata/conquery/models/execution/DefaultLabelTest.java index f06270d7d3..d273bf00f8 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/execution/DefaultLabelTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/execution/DefaultLabelTest.java @@ -34,7 +34,7 @@ public class DefaultLabelTest { - private final static MetaStorage STORAGE = new MetaStorage(); + private final static MetaStorage STORAGE = new NonPersistentStoreFactory().createMetaStorage(); private static final DatasetRegistry DATASET_REGISTRY = Mockito.mock(DatasetRegistry.class); private static final Dataset DATASET = new Dataset("dataset"); diff --git a/backend/src/test/java/com/bakdata/conquery/tasks/PermissionCleanupTaskTest.java b/backend/src/test/java/com/bakdata/conquery/tasks/PermissionCleanupTaskTest.java index 401ceb662e..e84a66bf74 100644 --- a/backend/src/test/java/com/bakdata/conquery/tasks/PermissionCleanupTaskTest.java +++ b/backend/src/test/java/com/bakdata/conquery/tasks/PermissionCleanupTaskTest.java @@ -29,12 +29,7 @@ class PermissionCleanupTaskTest { - private static final MetaStorage STORAGE = new MetaStorage(); - - @BeforeAll - public static void beforeAll() { - STORAGE.openStores(new NonPersistentStoreFactory()); - } + private static final MetaStorage STORAGE = new NonPersistentStoreFactory().createMetaStorage(); @AfterEach public void teardownAfterEach() { diff --git a/backend/src/test/java/com/bakdata/conquery/tasks/QueryCleanupTaskTest.java b/backend/src/test/java/com/bakdata/conquery/tasks/QueryCleanupTaskTest.java index 5b681d1262..3ece75e8ef 100644 --- a/backend/src/test/java/com/bakdata/conquery/tasks/QueryCleanupTaskTest.java +++ b/backend/src/test/java/com/bakdata/conquery/tasks/QueryCleanupTaskTest.java @@ -44,12 +44,7 @@ private ManagedQuery createManagedQuery() { return managedQuery; } - private static final MetaStorage STORAGE = new MetaStorage(); - - @BeforeAll - public static void beforeAll() { - STORAGE.openStores(new NonPersistentStoreFactory()); - } + private static final MetaStorage STORAGE = new NonPersistentStoreFactory().createMetaStorage(); @AfterEach diff --git a/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java b/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java index 27037de505..ff719c3c14 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java +++ b/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java @@ -29,7 +29,7 @@ import com.bakdata.conquery.models.forms.configs.FormConfig; import com.bakdata.conquery.models.identifiable.CentralRegistry; import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; -import com.bakdata.conquery.models.worker.SingletonNamespaceCollection; +import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.WorkerInformation; import com.bakdata.conquery.models.worker.WorkerToBucketsMap; @@ -107,12 +107,12 @@ public SingletonStore createStructureStore(String pathName, Cen } @Override - public IdentifiableStore> createExecutionsStore(CentralRegistry centralRegistry, String pathName) { + public IdentifiableStore> createExecutionsStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName) { return StoreMappings.identifiable(new NonPersistentStore<>(), centralRegistry); } @Override - public IdentifiableStore createFormConfigStore(CentralRegistry centralRegistry, String pathName) { + public IdentifiableStore createFormConfigStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName) { return StoreMappings.identifiable(new NonPersistentStore<>(), centralRegistry); } @@ -140,7 +140,7 @@ public SingletonStore createPrimaryDictionaryStore(String pathName, * @implNote intended for Unit-tests */ public MetaStorage createMetaStorage() { - final MetaStorage metaStorage = new MetaStorage(); + final MetaStorage metaStorage = new MetaStorage(null); metaStorage.openStores(this); return metaStorage; } diff --git a/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java b/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java index f5bb6ac468..56aa2b7701 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java +++ b/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java @@ -183,8 +183,7 @@ public synchronized StandaloneSupport getSupport(String name) { @SneakyThrows public synchronized void shutdown() { - standaloneCommand.getManager().getDatasetRegistry().close(); - standaloneCommand.getManager().getStorage().close(); + standaloneCommand.shutdown(); openSupports.clear(); } From f37f2590e6bfd7c66b87a6eefcef42ac49171e81 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 28 Sep 2021 13:45:52 +0200 Subject: [PATCH 29/35] removes comment --- .../models/auth/basic/UserAuthenticationManagementProcessor.java | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/basic/UserAuthenticationManagementProcessor.java b/backend/src/main/java/com/bakdata/conquery/models/auth/basic/UserAuthenticationManagementProcessor.java index feece2eb8f..9723706fdf 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/basic/UserAuthenticationManagementProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/basic/UserAuthenticationManagementProcessor.java @@ -20,7 +20,6 @@ public class UserAuthenticationManagementProcessor { private final MetaStorage storage; public boolean tryRegister(ProtoUser pUser) { - // Throws an exception if it would override the existing user final UserId id = pUser.createId(); User user = storage.getUser(id); if (user == null) { From 477ccd1f75c4e6cdd1a14a107483676af6cc8937 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 28 Sep 2021 11:49:01 +0000 Subject: [PATCH 30/35] Update AutoDoc --- docs/Config JSON.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/Config JSON.md b/docs/Config JSON.md index 51ffe4fe1c..fc3e8c4e30 100644 --- a/docs/Config JSON.md +++ b/docs/Config JSON.md @@ -47,7 +47,7 @@ An `AuthenticationConfig` is used to define how specific realms for authenticati Different types of AuthenticationRealmFactory can be used by setting `type` to one of the following values: -### DEVELOPMENT [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/auth/develop/DevAuthConfig.java#L11-L14) +### DEVELOPMENT [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/auth/develop/DevAuthConfig.java#L13-L16) Default configuration for the auth system. Sets up all other default components. This configuration causes that every request is handled as invoked by the super user.
Details

@@ -406,7 +406,7 @@ Supported Fields: | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusConfig.java#L51) | treeMaxPageSize | `int` or `null` | `null` | | |

-### Type XodusStoreFactory [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L84) +### Type XodusStoreFactory [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L88)
Details

@@ -417,12 +417,12 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L94) | directory | `Path` | `"file://./storage"` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L105) | nThreads | `@javax.validation.constraints.Min(1) int` | ␀ | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L108-L110) | removeUnreadableFromStore | `boolean` | `false` | | Flag for the {@link SerializingStore} whether to delete values from the underlying store, that cannot be mapped to an object anymore. | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L113-L115) | unreadableDataDumpDirectory | `File` | `null` | | When set, all values that could not be deserialized from the persistent store, are dump into individual files. | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L101) | useWeakDictionaryCaching | `boolean` | `false` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L96) | validateOnWrite | `boolean` | `false` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L102) | weakCacheDuration | `@NotNull Duration` | `"48 hours"` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L97) | xodus | [XodusConfig](#Type-XodusConfig) | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L127) | directory | `Path` | `"file://./storage"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L138) | nThreads | `@javax.validation.constraints.Min(1) int` | ␀ | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L141-L143) | removeUnreadableFromStore | `boolean` | `false` | | Flag for the {@link SerializingStore} whether to delete values from the underlying store, that cannot be mapped to an object anymore. | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L146-L148) | unreadableDataDumpDirectory | `File` | `null` | | When set, all values that could not be deserialized from the persistent store, are dump into individual files. | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L134) | useWeakDictionaryCaching | `boolean` | `false` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L129) | validateOnWrite | `boolean` | `false` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L135) | weakCacheDuration | `@NotNull Duration` | `"48 hours"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L130) | xodus | [XodusConfig](#Type-XodusConfig) | | | |

From 22379fe6d4c74eec59f8ea0bcdf6110e35275c47 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 28 Sep 2021 13:51:31 +0200 Subject: [PATCH 31/35] adds comment about checking the storages --- .../bakdata/conquery/models/config/XodusStoreFactory.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java index 38146cc545..23cdc01fa8 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java @@ -95,6 +95,11 @@ @CPSType(id = "XODUS", base = StoreFactory.class) public class XodusStoreFactory implements StoreFactory { + + /** + * The store names are created by hand here because the abstraction of {@link BigStore} + * creates two stores. Defining the expected stores like this, does not require a lot or complicated logic. + */ public static final Set NAMESPACED_STORES = Set.of( DATASET.storeInfo().getName(), SECONDARY_IDS.storeInfo().getName(), From 2e9bf07e79653ed55cb73b9cd8d72687f4464e3b Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 28 Sep 2021 14:47:53 +0200 Subject: [PATCH 32/35] revert clean shutdown because of port binding problems --- .../com/bakdata/conquery/commands/StandaloneCommand.java | 9 --------- .../com/bakdata/conquery/util/support/TestConquery.java | 8 +++++++- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/commands/StandaloneCommand.java b/backend/src/main/java/com/bakdata/conquery/commands/StandaloneCommand.java index afd84212c5..33bc8cdb07 100644 --- a/backend/src/main/java/com/bakdata/conquery/commands/StandaloneCommand.java +++ b/backend/src/main/java/com/bakdata/conquery/commands/StandaloneCommand.java @@ -132,13 +132,4 @@ protected void startStandalone(Environment environment, Namespace namespace, Con ConqueryMDC.setLocation(null); super.run(environment, namespace, config); } - - @SneakyThrows - public void shutdown(){ - manager.stop(); - for (ShardNode shardNode : shardNodes) { - shardNode.stop(); - } - - } } diff --git a/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java b/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java index 56aa2b7701..0a0e1bc615 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java +++ b/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java @@ -30,6 +30,7 @@ import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.Namespace; +import com.bakdata.conquery.models.worker.Worker; import com.bakdata.conquery.util.Wait; import com.bakdata.conquery.util.io.Cloner; import com.google.common.util.concurrent.Uninterruptibles; @@ -183,7 +184,12 @@ public synchronized StandaloneSupport getSupport(String name) { @SneakyThrows public synchronized void shutdown() { - standaloneCommand.shutdown(); + + // Don't call ManagerNode::stop, under linux the port is not released instantly, so a consecutive restart probably fails + standaloneCommand.getManager().getDatasetRegistry().close(); + standaloneCommand.getManager().getStorage().close(); + + standaloneCommand.getShardNodes().stream().flatMap(s -> s.getWorkers().getWorkers().values().stream()).forEach(Worker::close); openSupports.clear(); } From 753ffa987ae843810658053591bdb9977698c6dd Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 28 Sep 2021 15:24:36 +0200 Subject: [PATCH 33/35] fix removing of datasets in restart test and repeated closing of stores --- .../conquery/io/storage/xodus/stores/XodusStore.java | 8 ++++---- .../bakdata/conquery/integration/tests/RestartTest.java | 4 ++-- .../tests/deletion/ConceptUpdateAndDeletionTest.java | 2 -- .../integration/tests/deletion/DatasetDeletionTest.java | 3 +-- .../integration/tests/deletion/ImportDeletionTest.java | 2 -- .../integration/tests/deletion/TableDeletionTest.java | 2 -- .../com/bakdata/conquery/util/support/TestConquery.java | 9 ++------- 7 files changed, 9 insertions(+), 21 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/XodusStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/XodusStore.java index 31caae40d7..6d3522f36c 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/XodusStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/XodusStore.java @@ -99,10 +99,10 @@ public void clear() { } public void remove() { - if (!environment.isOpen()) { - log.debug("While removing store: Environment is already closed for {}", this); - return; - } +// if (!environment.isOpen()) { +// log.debug("While removing store: Environment is already closed for {}", this); +// return; +// } log.debug("Removing store {} from environment {}", store.getName(), environment.getLocation()); environment.executeInTransaction(t -> environment.removeStore(store.getName(),t)); storeRemoveHook.accept(this); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/RestartTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/RestartTest.java index 654baf8a96..1cc0b95608 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/RestartTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/RestartTest.java @@ -135,8 +135,6 @@ public void execute(String name, TestConquery testConquery) throws Exception { testConquery.shutdown(); - //stop dropwizard directly so ConquerySupport does not delete the tmp directory - testConquery.getDropwizard().after(); log.info("Restarting"); testConquery.beforeAll(); @@ -180,6 +178,8 @@ public void execute(String name, TestConquery testConquery) throws Exception { .getIdMapping(); assertThat(entityIdMapAfterRestart).isEqualTo(entityIdMap); + // We need to reassign the dataset processor because the instance prio to the restart became invalid + adminDatasetProcessor = testConquery.getStandaloneCommand().getManager().getAdmin().getAdminDatasetProcessor(); // Cleanup adminDatasetProcessor.deleteDataset(dataset1); adminDatasetProcessor.deleteDataset(dataset2); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ConceptUpdateAndDeletionTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ConceptUpdateAndDeletionTest.java index d6c86d5df1..1440502d83 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ConceptUpdateAndDeletionTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ConceptUpdateAndDeletionTest.java @@ -202,8 +202,6 @@ public void execute(String name, TestConquery testConquery) throws Exception { { testConquery.shutdown(); - //stop dropwizard directly so ConquerySupport does not delete the tmp directory - testConquery.getDropwizard().after(); //restart testConquery.beforeAll(); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/DatasetDeletionTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/DatasetDeletionTest.java index 9dd593fe82..06f0ef83fe 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/DatasetDeletionTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/DatasetDeletionTest.java @@ -201,8 +201,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { // Finally, restart conquery and assert again, that the data is correct. { testConquery.shutdown(); - //stop dropwizard directly so ConquerySupport does not delete the tmp directory - testConquery.getDropwizard().after(); + //restart testConquery.beforeAll(); final StandaloneSupport conqueryRestart = testConquery.openDataset(conqueryReimport.getDataset().getId()); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ImportDeletionTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ImportDeletionTest.java index f8614c15d1..2dc04f2689 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ImportDeletionTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/ImportDeletionTest.java @@ -272,8 +272,6 @@ public void execute(String name, TestConquery testConquery) throws Exception { { testConquery.shutdown(); - //stop dropwizard directly so ConquerySupport does not delete the tmp directory - testConquery.getDropwizard().after(); //restart testConquery.beforeAll(); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/TableDeletionTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/TableDeletionTest.java index 1d3b870109..90796fae00 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/TableDeletionTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/deletion/TableDeletionTest.java @@ -246,8 +246,6 @@ public void execute(String name, TestConquery testConquery) throws Exception { { testConquery.shutdown(); - //stop dropwizard directly so ConquerySupport does not delete the tmp directory - testConquery.getDropwizard().after(); //restart testConquery.beforeAll(); diff --git a/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java b/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java index 0a0e1bc615..9267ab3a3c 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java +++ b/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java @@ -184,13 +184,8 @@ public synchronized StandaloneSupport getSupport(String name) { @SneakyThrows public synchronized void shutdown() { - - // Don't call ManagerNode::stop, under linux the port is not released instantly, so a consecutive restart probably fails - standaloneCommand.getManager().getDatasetRegistry().close(); - standaloneCommand.getManager().getStorage().close(); - - standaloneCommand.getShardNodes().stream().flatMap(s -> s.getWorkers().getWorkers().values().stream()).forEach(Worker::close); - + //stop dropwizard directly so ConquerySupport does not delete the tmp directory + getDropwizard().after(); openSupports.clear(); } From 8067cf731a55cff6114016b1b353f49576822790 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 28 Sep 2021 13:26:53 +0000 Subject: [PATCH 34/35] Update AutoDoc --- docs/Config JSON.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/Config JSON.md b/docs/Config JSON.md index fc3e8c4e30..9086ef3939 100644 --- a/docs/Config JSON.md +++ b/docs/Config JSON.md @@ -417,12 +417,12 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L127) | directory | `Path` | `"file://./storage"` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L138) | nThreads | `@javax.validation.constraints.Min(1) int` | ␀ | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L141-L143) | removeUnreadableFromStore | `boolean` | `false` | | Flag for the {@link SerializingStore} whether to delete values from the underlying store, that cannot be mapped to an object anymore. | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L146-L148) | unreadableDataDumpDirectory | `File` | `null` | | When set, all values that could not be deserialized from the persistent store, are dump into individual files. | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L134) | useWeakDictionaryCaching | `boolean` | `false` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L129) | validateOnWrite | `boolean` | `false` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L135) | weakCacheDuration | `@NotNull Duration` | `"48 hours"` | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L130) | xodus | [XodusConfig](#Type-XodusConfig) | | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L132) | directory | `Path` | `"file://./storage"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L143) | nThreads | `@javax.validation.constraints.Min(1) int` | ␀ | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L146-L148) | removeUnreadableFromStore | `boolean` | `false` | | Flag for the {@link SerializingStore} whether to delete values from the underlying store, that cannot be mapped to an object anymore. | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L151-L153) | unreadableDataDumpDirectory | `File` | `null` | | When set, all values that could not be deserialized from the persistent store, are dump into individual files. | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L139) | useWeakDictionaryCaching | `boolean` | `false` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L134) | validateOnWrite | `boolean` | `false` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L140) | weakCacheDuration | `@NotNull Duration` | `"48 hours"` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java#L135) | xodus | [XodusConfig](#Type-XodusConfig) | | | |

From 62ba40663bf7c9de36eddaa258e5e9b4c791656e Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 28 Sep 2021 15:44:20 +0200 Subject: [PATCH 35/35] makes exception to warning --- .../java/com/bakdata/conquery/models/jobs/JobExecutor.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/jobs/JobExecutor.java b/backend/src/main/java/com/bakdata/conquery/models/jobs/JobExecutor.java index 0753953b05..70ed43de74 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/jobs/JobExecutor.java +++ b/backend/src/main/java/com/bakdata/conquery/models/jobs/JobExecutor.java @@ -33,7 +33,8 @@ public JobExecutor(String name, boolean failOnError) { public void add(Job job) { if(closed.get()) { - throw new IllegalStateException("Tried to add a job to a closed JobManager"); + log.warn("Tried to add a job to a closed JobManager: {}", job.getLabel()); + return; } jobs.add(job); }