diff --git a/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java b/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java index 004048186d..d1c5686040 100644 --- a/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java +++ b/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java @@ -46,7 +46,7 @@ import org.glassfish.jersey.internal.inject.AbstractBinder; /** - * Central node of Conquery. Hosts the frontend, api, meta data and takes care of query distribution to + * Central node of Conquery. Hosts the frontend, api, metadata and takes care of query distribution to * {@link ShardNode}s and respectively the {@link Worker}s hosted on them. The {@link ManagerNode} can also * forward queries or results to statistic backends. Finally, it collects the results of queries for access over the api. */ @@ -113,7 +113,7 @@ public void run(Manager manager) throws InterruptedException { // Create AdminServlet first to make it available to the realms admin = new AdminServlet(this); - authController = new AuthorizationController(getStorage(), config, environment, admin); + authController = new AuthorizationController(getMetaStorage(), config, environment, admin); environment.lifecycle().manage(authController); // Register default components for the admin interface @@ -145,14 +145,14 @@ public void run(Manager manager) throws InterruptedException { private void registerTasks(Manager manager, Environment environment, ConqueryConfig config) { environment.admin().addTask(formScanner); environment.admin().addTask( - new QueryCleanupTask(getStorage(), Duration.of( + new QueryCleanupTask(getMetaStorage(), Duration.of( config.getQueries().getOldQueriesTime().getQuantity(), config.getQueries().getOldQueriesTime().getUnit().toChronoUnit() ))); - environment.admin().addTask(new PermissionCleanupTask(getStorage())); + environment.admin().addTask(new PermissionCleanupTask(getMetaStorage())); manager.getAdminTasks().forEach(environment.admin()::addTask); - environment.admin().addTask(new ReloadMetaStorageTask(getStorage())); + environment.admin().addTask(new ReloadMetaStorageTask(getMetaStorage())); final ShutdownTask shutdown = new ShutdownTask(); environment.admin().addTask(shutdown); @@ -164,7 +164,7 @@ private void configureApiServlet(ConqueryConfig config, DropwizardResourceConfig jerseyConfig.register(new AbstractBinder() { @Override protected void configure() { - bind(getStorage()).to(MetaStorage.class); + bind(getMetaStorage()).to(MetaStorage.class); bind(getDatasetRegistry()).to(DatasetRegistry.class); } }); @@ -203,7 +203,7 @@ public void customizeApiObjectMapper(ObjectMapper objectMapper) { injectableValues.add(Validator.class, getValidator()); getDatasetRegistry().injectInto(objectMapper); - getStorage().injectInto(objectMapper); + getMetaStorage().injectInto(objectMapper); getConfig().injectInto(objectMapper); } @@ -219,10 +219,10 @@ public ObjectMapper createInternalObjectMapper(Class viewClass) private void loadMetaStorage() { log.info("Opening MetaStorage"); - getStorage().openStores(getInternalObjectMapperCreator().createInternalObjectMapper(View.Persistence.Manager.class)); + getMetaStorage().openStores(getInternalObjectMapperCreator().createInternalObjectMapper(View.Persistence.Manager.class)); log.info("Loading MetaStorage"); - getStorage().loadData(); - log.info("MetaStorage loaded {}", getStorage()); + getMetaStorage().loadData(); + log.info("MetaStorage loaded {}", getMetaStorage()); } @SneakyThrows(InterruptedException.class) @@ -236,7 +236,7 @@ public void loadNamespaces() { final Collection namespaceStorages = getConfig().getStorage().discoverNamespaceStorages(); for (NamespaceStorage namespaceStorage : namespaceStorages) { loaders.submit(() -> { - registry.createNamespace(namespaceStorage); + registry.createNamespace(namespaceStorage, getMetaStorage()); }); } @@ -262,16 +262,16 @@ public void stop() throws Exception { provider.close(); } catch (Exception e) { - log.error(provider + " could not be closed", e); + log.error("{} could not be closed", provider, e); } } try { - getStorage().close(); + getMetaStorage().close(); } catch (Exception e) { - log.error("{} could not be closed", getStorage(), e); + log.error("{} could not be closed", getMetaStorage(), e); } } diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/MetaIdReferenceDeserializer.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/MetaIdReferenceDeserializer.java index 0023d97757..e08b026eea 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/MetaIdReferenceDeserializer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/MetaIdReferenceDeserializer.java @@ -4,18 +4,14 @@ import java.util.InputMismatchException; import java.util.Optional; +import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.models.identifiable.CentralRegistry; import com.bakdata.conquery.models.identifiable.Identifiable; import com.bakdata.conquery.models.identifiable.ids.Id; import com.bakdata.conquery.models.identifiable.ids.IdUtil; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonToken; -import com.fasterxml.jackson.databind.BeanDescription; -import com.fasterxml.jackson.databind.BeanProperty; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JavaType; -import com.fasterxml.jackson.databind.JsonDeserializer; -import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.*; import com.fasterxml.jackson.databind.deser.ContextualDeserializer; import com.fasterxml.jackson.databind.deser.SettableBeanProperty; import com.fasterxml.jackson.databind.jsontype.TypeDeserializer; @@ -42,7 +38,7 @@ public T deserialize(JsonParser parser, DeserializationContext ctxt) throws IOEx ID id = ctxt.readValue(parser, idClass); try { - final CentralRegistry centralRegistry = CentralRegistry.get(ctxt); + final CentralRegistry centralRegistry = MetaStorage.get(ctxt).getCentralRegistry(); // Not all Components have registries, we leave it up to the validator to be angry. if (centralRegistry == null) { diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java index d8c12a95b8..d0cf222cef 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java @@ -11,13 +11,9 @@ import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.forms.configs.FormConfig; import com.bakdata.conquery.models.identifiable.CentralRegistry; -import com.bakdata.conquery.models.identifiable.ids.specific.FormConfigId; -import com.bakdata.conquery.models.identifiable.ids.specific.GroupId; -import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; -import com.bakdata.conquery.models.identifiable.ids.specific.RoleId; -import com.bakdata.conquery.models.identifiable.ids.specific.UserId; -import com.bakdata.conquery.models.worker.DatasetRegistry; -import com.bakdata.conquery.models.worker.Namespace; +import com.bakdata.conquery.models.identifiable.ids.specific.*; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -34,8 +30,6 @@ public class MetaStorage extends ConqueryStorage implements Injectable { protected final CentralRegistry centralRegistry = new CentralRegistry(); private final StoreFactory storageFactory; - @Getter - protected final DatasetRegistry datasetRegistry; private IdentifiableStore executions; private IdentifiableStore formConfigs; private IdentifiableStore authUser; @@ -47,8 +41,8 @@ public void openStores(ObjectMapper mapper) { authRole = storageFactory.createRoleStore(centralRegistry, "meta", this, mapper); authGroup = storageFactory.createGroupStore(centralRegistry, "meta", this, mapper); // Executions depend on users - executions = storageFactory.createExecutionsStore(centralRegistry, datasetRegistry, "meta", mapper); - formConfigs = storageFactory.createFormConfigStore(centralRegistry, datasetRegistry, "meta", mapper); + executions = storageFactory.createExecutionsStore(centralRegistry, "meta", mapper); + formConfigs = storageFactory.createFormConfigStore(centralRegistry, "meta", mapper); } @@ -196,4 +190,8 @@ public void addFormConfig(FormConfig formConfig) { public MutableInjectableValues inject(MutableInjectableValues values) { return values.add(MetaStorage.class, this); } + + public static MetaStorage get(DeserializationContext ctxt) throws JsonMappingException { + return (MetaStorage) ctxt.findInjectableValue(MetaStorage.class.getName(), null, null); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java index f84b4dc544..7b29656e6d 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java @@ -17,7 +17,6 @@ import com.bakdata.conquery.models.worker.WorkerToBucketsMap; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; -import jakarta.validation.Validator; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -32,7 +31,7 @@ public class NamespaceStorage extends NamespacedStorage { protected CachedStore entity2Bucket; - public NamespaceStorage(StoreFactory storageFactory, String pathName, Validator validator) { + public NamespaceStorage(StoreFactory storageFactory, String pathName) { super(storageFactory, pathName); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java index a4a4c07d54..a9a7378760 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java @@ -19,6 +19,7 @@ import com.bakdata.conquery.models.identifiable.ids.specific.ImportId; import com.bakdata.conquery.models.identifiable.ids.specific.SecondaryIdDescriptionId; import com.bakdata.conquery.models.identifiable.ids.specific.TableId; +import com.bakdata.conquery.models.worker.SingletonNamespaceCollection; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import lombok.Getter; @@ -56,7 +57,8 @@ public NamespacedStorage(StoreFactory storageFactory, String pathName) { } public void openStores(ObjectMapper objectMapper) { - + // Before we start to parse the stores we need to replace the injected value for the IdResolveContext (from DatasetRegistry to this centralRegistry) + new SingletonNamespaceCollection(centralRegistry).injectInto(objectMapper); dataset = storageFactory.createDatasetStore(pathName, objectMapper); secondaryIds = storageFactory.createSecondaryIdDescriptionStore(centralRegistry, pathName, objectMapper); diff --git a/backend/src/main/java/com/bakdata/conquery/mode/DelegateManager.java b/backend/src/main/java/com/bakdata/conquery/mode/DelegateManager.java index f4a402adce..53ada11ccc 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/DelegateManager.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/DelegateManager.java @@ -24,6 +24,7 @@ public class DelegateManager implements Manager { ConqueryConfig config; Environment environment; DatasetRegistry datasetRegistry; + MetaStorage storage; ImportHandler importHandler; StorageListener storageListener; Supplier> nodeProvider; @@ -42,7 +43,7 @@ public void stop() throws Exception { } @Override - public MetaStorage getStorage() { - return datasetRegistry.getMetaStorage(); + public MetaStorage getMetaStorage() { + return storage; } } diff --git a/backend/src/main/java/com/bakdata/conquery/mode/InternalObjectMapperCreator.java b/backend/src/main/java/com/bakdata/conquery/mode/InternalObjectMapperCreator.java index 64d255dad8..8fdcf63076 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/InternalObjectMapperCreator.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/InternalObjectMapperCreator.java @@ -23,13 +23,12 @@ @RequiredArgsConstructor public class InternalObjectMapperCreator { private final ConqueryConfig config; + private final MetaStorage storage; private final Validator validator; private DatasetRegistry datasetRegistry = null; - private MetaStorage storage = null; public void init(DatasetRegistry datasetRegistry) { this.datasetRegistry = datasetRegistry; - this.storage = datasetRegistry.getMetaStorage(); } public ObjectMapper createInternalObjectMapper(@Nullable Class viewClass) { diff --git a/backend/src/main/java/com/bakdata/conquery/mode/Manager.java b/backend/src/main/java/com/bakdata/conquery/mode/Manager.java index bef1a3c444..f8de4d3035 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/Manager.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/Manager.java @@ -10,9 +10,9 @@ import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.models.worker.ShardNodeInformation; +import io.dropwizard.core.setup.Environment; import io.dropwizard.lifecycle.Managed; import io.dropwizard.servlets.tasks.Task; -import io.dropwizard.core.setup.Environment; /** * A manager provides the implementations that differ by running mode. @@ -27,5 +27,6 @@ public interface Manager extends Managed { List getAdminTasks(); InternalObjectMapperCreator getInternalObjectMapperCreator(); JobManager getJobManager(); - MetaStorage getStorage(); + + MetaStorage getMetaStorage(); } diff --git a/backend/src/main/java/com/bakdata/conquery/mode/ManagerProvider.java b/backend/src/main/java/com/bakdata/conquery/mode/ManagerProvider.java index c25f63a08e..fe45f4ecbe 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/ManagerProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/ManagerProvider.java @@ -23,8 +23,8 @@ static JobManager newJobManager(ConqueryConfig config) { return new JobManager(JOB_MANAGER_NAME, config.isFailOnError()); } - static InternalObjectMapperCreator newInternalObjectMapperCreator(ConqueryConfig config, Validator validator) { - return new InternalObjectMapperCreator(config, validator); + static InternalObjectMapperCreator newInternalObjectMapperCreator(ConqueryConfig config, MetaStorage metaStorage, Validator validator) { + return new InternalObjectMapperCreator(config, metaStorage, validator); } static DatasetRegistry createDatasetRegistry( @@ -33,16 +33,13 @@ static DatasetRegistry createDatasetRegistry( InternalObjectMapperCreator creator ) { final IndexService indexService = new IndexService(config.getCsv().createCsvParserSettings(), config.getIndex().getEmptyLabel()); - DatasetRegistry datasetRegistry = new DatasetRegistry<>( + return new DatasetRegistry<>( config.getCluster().getEntityBucketSize(), config, creator, namespaceHandler, indexService ); - MetaStorage storage = new MetaStorage(config.getStorage(), datasetRegistry); - datasetRegistry.setMetaStorage(storage); - return datasetRegistry; } } diff --git a/backend/src/main/java/com/bakdata/conquery/mode/NamespaceHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/NamespaceHandler.java index 8f3e4eb1e4..fef5334580 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/NamespaceHandler.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/NamespaceHandler.java @@ -4,6 +4,7 @@ import java.util.List; import com.bakdata.conquery.io.jackson.Injectable; +import com.bakdata.conquery.io.jackson.Jackson; import com.bakdata.conquery.io.jackson.View; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.io.storage.NamespaceStorage; @@ -32,6 +33,7 @@ public interface NamespaceHandler { static NamespaceSetupData createNamespaceSetup(NamespaceStorage storage, final ConqueryConfig config, final InternalObjectMapperCreator mapperCreator, IndexService indexService) { List injectables = new ArrayList<>(); injectables.add(indexService); + ObjectMapper persistenceMapper = mapperCreator.createInternalObjectMapper(View.Persistence.Manager.class); ObjectMapper communicationMapper = mapperCreator.createInternalObjectMapper(View.InternalCommunication.class); ObjectMapper preprocessMapper = mapperCreator.createInternalObjectMapper(null); @@ -40,8 +42,9 @@ static NamespaceSetupData createNamespaceSetup(NamespaceStorage storage, final C injectables.forEach(i -> i.injectInto(communicationMapper)); injectables.forEach(i -> i.injectInto(preprocessMapper)); - // Open and load the stores - storage.openStores(persistenceMapper); + + // Each store needs its own mapper because each injects its own registry + storage.openStores(Jackson.copyMapperAndInjectables(persistenceMapper)); storage.loadData(); JobManager jobManager = new JobManager(storage.getDataset().getName(), config.isFailOnError()); diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterManagerProvider.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterManagerProvider.java index 1579acf868..cafc855713 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterManagerProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterManagerProvider.java @@ -4,6 +4,7 @@ import java.util.List; import java.util.function.Supplier; +import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.mode.*; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.jobs.JobManager; @@ -19,7 +20,8 @@ public class ClusterManagerProvider implements ManagerProvider { public ClusterManager provideManager(ConqueryConfig config, Environment environment) { final JobManager jobManager = ManagerProvider.newJobManager(config); - final InternalObjectMapperCreator creator = ManagerProvider.newInternalObjectMapperCreator(config, environment.getValidator()); + final MetaStorage storage = new MetaStorage(config.getStorage()); + final InternalObjectMapperCreator creator = ManagerProvider.newInternalObjectMapperCreator(config, storage, environment.getValidator()); final ClusterState clusterState = new ClusterState(); final NamespaceHandler namespaceHandler = new ClusterNamespaceHandler(clusterState, config, creator); final DatasetRegistry datasetRegistry = ManagerProvider.createDatasetRegistry(namespaceHandler, config, creator); @@ -35,7 +37,7 @@ public ClusterManager provideManager(ConqueryConfig config, Environment environm final DelegateManager delegate = - new DelegateManager<>(config, environment, datasetRegistry, importHandler, extension, nodeProvider, adminTasks, creator, jobManager); + new DelegateManager<>(config, environment, datasetRegistry, storage, importHandler, extension, nodeProvider, adminTasks, creator, jobManager); environment.healthChecks().register("cluster", new ClusterHealthCheck(clusterState)); diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java index 762a754eb8..41d3895cf3 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java @@ -5,6 +5,7 @@ import java.util.List; import java.util.function.Supplier; +import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.mode.DelegateManager; import com.bakdata.conquery.mode.InternalObjectMapperCreator; import com.bakdata.conquery.mode.ManagerProvider; @@ -32,15 +33,18 @@ public LocalManagerProvider(SqlDialectFactory dialectFactory) { public DelegateManager provideManager(ConqueryConfig config, Environment environment) { - InternalObjectMapperCreator creator = ManagerProvider.newInternalObjectMapperCreator(config, environment.getValidator()); - NamespaceHandler namespaceHandler = new LocalNamespaceHandler(config, creator, dialectFactory); - DatasetRegistry datasetRegistry = ManagerProvider.createDatasetRegistry(namespaceHandler, config, creator); + final MetaStorage storage = new MetaStorage(config.getStorage()); + final InternalObjectMapperCreator creator = ManagerProvider.newInternalObjectMapperCreator(config, storage, environment.getValidator()); + final NamespaceHandler namespaceHandler = new LocalNamespaceHandler(config, creator, dialectFactory); + final DatasetRegistry datasetRegistry = ManagerProvider.createDatasetRegistry(namespaceHandler, config, creator); + creator.init(datasetRegistry); return new DelegateManager<>( config, environment, datasetRegistry, + storage, new FailingImportHandler(), new LocalStorageListener(), EMPTY_NODE_PROVIDER, diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java index f79907fa0a..78ec8aecea 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java @@ -12,11 +12,7 @@ import com.bakdata.conquery.models.auth.entities.Group; import com.bakdata.conquery.models.auth.entities.Role; import com.bakdata.conquery.models.auth.entities.User; -import com.bakdata.conquery.models.datasets.Dataset; -import com.bakdata.conquery.models.datasets.Import; -import com.bakdata.conquery.models.datasets.PreviewConfig; -import com.bakdata.conquery.models.datasets.SecondaryIdDescription; -import com.bakdata.conquery.models.datasets.Table; +import com.bakdata.conquery.models.datasets.*; import com.bakdata.conquery.models.datasets.concepts.Concept; import com.bakdata.conquery.models.datasets.concepts.StructureNode; import com.bakdata.conquery.models.events.Bucket; @@ -27,7 +23,6 @@ import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; import com.bakdata.conquery.models.index.InternToExternMapper; import com.bakdata.conquery.models.index.search.SearchIndex; -import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.WorkerInformation; import com.bakdata.conquery.models.worker.WorkerToBucketsMap; import com.fasterxml.jackson.annotation.JsonTypeInfo; @@ -67,9 +62,9 @@ public interface StoreFactory { SingletonStore createStructureStore(String pathName, CentralRegistry centralRegistry, ObjectMapper objectMapper); // MetaStorage - IdentifiableStore createExecutionsStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName, ObjectMapper objectMapper); + IdentifiableStore createExecutionsStore(CentralRegistry centralRegistry, String pathName, ObjectMapper objectMapper); - IdentifiableStore createFormConfigStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName, ObjectMapper objectMapper); + IdentifiableStore createFormConfigStore(CentralRegistry centralRegistry, String pathName, ObjectMapper objectMapper); IdentifiableStore createUserStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage, ObjectMapper objectMapper); diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java index ae74ba9f23..7529e92934 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java @@ -52,7 +52,6 @@ import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; import com.bakdata.conquery.models.index.InternToExternMapper; import com.bakdata.conquery.models.index.search.SearchIndex; -import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.WorkerInformation; import com.bakdata.conquery.models.worker.WorkerToBucketsMap; import com.bakdata.conquery.util.io.ConqueryMDC; @@ -191,7 +190,7 @@ public ExecutorService getReaderExecutorService() { @Override public Collection discoverNamespaceStorages() { - return loadNamespacedStores("dataset_", (storePath) -> new NamespaceStorage(this, storePath, getValidator()), NAMESPACE_STORES); + return loadNamespacedStores("dataset_", (storePath) -> new NamespaceStorage(this, storePath), NAMESPACE_STORES); } @Override @@ -258,22 +257,22 @@ public SingletonStore createDatasetStore(String pathName, ObjectMapper @Override public IdentifiableStore createSecondaryIdDescriptionStore(CentralRegistry centralRegistry, String pathName, ObjectMapper objectMapper) { - return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, SECONDARY_IDS, centralRegistry.injectIntoNew(objectMapper)), centralRegistry); + return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, SECONDARY_IDS, objectMapper), centralRegistry); } @Override public IdentifiableStore createInternToExternMappingStore(String pathName, CentralRegistry centralRegistry, ObjectMapper objectMapper) { - return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, INTERN_TO_EXTERN, centralRegistry.injectIntoNew(objectMapper)), centralRegistry); + return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, INTERN_TO_EXTERN, objectMapper), centralRegistry); } @Override public IdentifiableStore createSearchIndexStore(String pathName, CentralRegistry centralRegistry, ObjectMapper objectMapper) { - return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, SEARCH_INDEX, centralRegistry.injectIntoNew(objectMapper)), centralRegistry); + return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, SEARCH_INDEX, objectMapper), centralRegistry); } @Override public SingletonStore createPreviewStore(String pathName, CentralRegistry centralRegistry, ObjectMapper objectMapper) { - return StoreMappings.singleton(createStore(findEnvironment(pathName), validator, ENTITY_PREVIEW, centralRegistry.injectIntoNew(objectMapper))); + return StoreMappings.singleton(createStore(findEnvironment(pathName), validator, ENTITY_PREVIEW, objectMapper)); } @Override @@ -283,27 +282,27 @@ public CachedStore createEntity2BucketStore(String pathName, Ob @Override public IdentifiableStore createTableStore(CentralRegistry centralRegistry, String pathName, ObjectMapper objectMapper) { - return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, TABLES, centralRegistry.injectIntoNew(objectMapper)), centralRegistry); + return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, TABLES, objectMapper), centralRegistry); } @Override public IdentifiableStore> createConceptStore(CentralRegistry centralRegistry, String pathName, ObjectMapper objectMapper) { - return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, CONCEPTS, centralRegistry.injectIntoNew(objectMapper)), centralRegistry); + return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, CONCEPTS, objectMapper), centralRegistry); } @Override public IdentifiableStore createImportStore(CentralRegistry centralRegistry, String pathName, ObjectMapper objectMapper) { - return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, IMPORTS, centralRegistry.injectIntoNew(objectMapper)), centralRegistry); + return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, IMPORTS, objectMapper), centralRegistry); } @Override public IdentifiableStore createCBlockStore(CentralRegistry centralRegistry, String pathName, ObjectMapper objectMapper) { - return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, C_BLOCKS, centralRegistry.injectIntoNew(objectMapper)), centralRegistry); + return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, C_BLOCKS, objectMapper), centralRegistry); } @Override public IdentifiableStore createBucketStore(CentralRegistry centralRegistry, String pathName, ObjectMapper objectMapper) { - return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, BUCKETS, centralRegistry.injectIntoNew(objectMapper)), centralRegistry); + return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, BUCKETS, objectMapper), centralRegistry); } @Override @@ -332,17 +331,17 @@ public SingletonStore createWorkerToBucketsStore(String path @Override public SingletonStore createStructureStore(String pathName, CentralRegistry centralRegistry, ObjectMapper objectMapper) { - return StoreMappings.singleton(createStore(findEnvironment(pathName), validator, STRUCTURE, centralRegistry.injectIntoNew(objectMapper))); + return StoreMappings.singleton(createStore(findEnvironment(pathName), validator, STRUCTURE, objectMapper)); } @Override - public IdentifiableStore createExecutionsStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName, ObjectMapper objectMapper) { - return StoreMappings.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "executions")), validator, EXECUTIONS, datasetRegistry.injectInto(centralRegistry.injectIntoNew(objectMapper))), centralRegistry); + public IdentifiableStore createExecutionsStore(CentralRegistry centralRegistry, String pathName, ObjectMapper objectMapper) { + return StoreMappings.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "executions")), validator, EXECUTIONS, objectMapper), centralRegistry); } @Override - public IdentifiableStore createFormConfigStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName, ObjectMapper objectMapper) { - return StoreMappings.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "formConfigs")), validator, FORM_CONFIG, datasetRegistry.injectInto(centralRegistry.injectIntoNew(objectMapper))), centralRegistry); + public IdentifiableStore createFormConfigStore(CentralRegistry centralRegistry, String pathName, ObjectMapper objectMapper) { + return StoreMappings.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "formConfigs")), validator, FORM_CONFIG, objectMapper), centralRegistry); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/CentralRegistry.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/CentralRegistry.java index f03b7a69b4..ae74f0de63 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/CentralRegistry.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/CentralRegistry.java @@ -5,12 +5,8 @@ import java.util.concurrent.ConcurrentMap; import java.util.function.Function; -import com.bakdata.conquery.io.jackson.Injectable; -import com.bakdata.conquery.io.jackson.MutableInjectableValues; import com.bakdata.conquery.models.error.ConqueryError.ExecutionCreationResolveError; import com.bakdata.conquery.models.identifiable.ids.Id; -import com.bakdata.conquery.models.worker.IdResolveContext; -import com.bakdata.conquery.models.worker.SingletonNamespaceCollection; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.JsonMappingException; import lombok.NoArgsConstructor; @@ -22,7 +18,7 @@ @SuppressWarnings({"rawtypes", "unchecked"}) @NoArgsConstructor @ToString(of = "map") -public class CentralRegistry implements Injectable { +public class CentralRegistry { private final IdMap map = new IdMap<>(); private final ConcurrentMap, Function> cacheables = new ConcurrentHashMap<>(); @@ -70,24 +66,8 @@ public synchronized void remove(Identifiable ident) { map.remove(id); } - @Override - public MutableInjectableValues inject(MutableInjectableValues values) { - return values.add(CentralRegistry.class, this) - // Possibly overriding mapping for DatasetRegistry - .add(IdResolveContext.class, new SingletonNamespaceCollection(this)); - } - public static CentralRegistry get(DeserializationContext ctxt) throws JsonMappingException { - CentralRegistry result = (CentralRegistry) ctxt.findInjectableValue(CentralRegistry.class.getName(), null, null); - if (result != null) { - return result; - } - - IdResolveContext alternative = (IdResolveContext) ctxt.findInjectableValue(IdResolveContext.class.getName(), null, null); - if (alternative == null) { - return null; - } - return alternative.getMetaRegistry(); + return (CentralRegistry) ctxt.findInjectableValue(CentralRegistry.class.getName(), null, null); } public void clear() { diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/NamespacedId.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/NamespacedId.java index 936a4c313f..d19838ffc8 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/NamespacedId.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/NamespacedId.java @@ -1,12 +1,13 @@ package com.bakdata.conquery.models.identifiable.ids; import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; -import com.bakdata.conquery.models.worker.IdResolveContext; import com.fasterxml.jackson.annotation.JsonIgnore; import org.apache.commons.lang3.StringUtils; /** - * Marker interface for {@link Id}s that are loaded via Namespaced CentralRegistry (see {@link com.bakdata.conquery.models.worker.IdResolveContext#findRegistry(DatasetId)}, as opposed to MetaRegistry (see {@link IdResolveContext#getMetaRegistry()}). + * Marker interface for {@link Id}s that are loaded via Namespaced CentralRegistry + * (see {@link com.bakdata.conquery.models.worker.IdResolveContext#findRegistry(DatasetId)}, + * as opposed to Registry in the {@link com.bakdata.conquery.io.storage.MetaStorage} */ public interface NamespacedId { diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/DatasetRegistry.java b/backend/src/main/java/com/bakdata/conquery/models/worker/DatasetRegistry.java index afc490de52..64606760ff 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/DatasetRegistry.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/DatasetRegistry.java @@ -10,6 +10,7 @@ import java.util.concurrent.ConcurrentMap; import java.util.stream.Collectors; +import com.bakdata.conquery.io.jackson.Jackson; import com.bakdata.conquery.io.jackson.MutableInjectableValues; import com.bakdata.conquery.io.jackson.View; import com.bakdata.conquery.io.storage.MetaStorage; @@ -27,10 +28,8 @@ import com.fasterxml.jackson.annotation.JsonIgnoreType; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.cache.CacheStats; -import jakarta.validation.Validator; import lombok.Getter; import lombok.RequiredArgsConstructor; -import lombok.Setter; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -47,30 +46,27 @@ public class DatasetRegistry extends IdResolveContext imple private final InternalObjectMapperCreator internalObjectMapperCreator; - @Getter - @Setter - private MetaStorage metaStorage; - private final NamespaceHandler namespaceHandler; private final IndexService indexService; - public N createNamespace(Dataset dataset, Validator validator) throws IOException { + public N createNamespace(Dataset dataset, MetaStorage metaStorage) throws IOException { // Prepare empty storage - NamespaceStorage datasetStorage = new NamespaceStorage(config.getStorage(), "dataset_" + dataset.getName(), validator); + NamespaceStorage datasetStorage = new NamespaceStorage(config.getStorage(), "dataset_" + dataset.getName()); final ObjectMapper persistenceMapper = internalObjectMapperCreator.createInternalObjectMapper(View.Persistence.Manager.class); - datasetStorage.openStores(persistenceMapper); + // Each store injects its own IdResolveCtx so each needs its own mapper + datasetStorage.openStores(Jackson.copyMapperAndInjectables((persistenceMapper))); datasetStorage.loadData(); datasetStorage.updateDataset(dataset); datasetStorage.updateIdMapping(new EntityIdMap()); datasetStorage.setPreviewConfig(new PreviewConfig()); datasetStorage.close(); - return createNamespace(datasetStorage); + return createNamespace(datasetStorage, metaStorage); } - public N createNamespace(NamespaceStorage datasetStorage) { + public N createNamespace(NamespaceStorage datasetStorage, MetaStorage metaStorage) { final N namespace = namespaceHandler.createNamespace(datasetStorage, metaStorage, indexService); add(namespace); return namespace; @@ -88,7 +84,6 @@ public void removeNamespace(DatasetId id) { N removed = datasets.remove(id); if (removed != null) { - metaStorage.getCentralRegistry().remove(removed.getDataset()); namespaceHandler.removeNamespace(id, removed); removed.remove(); } @@ -103,12 +98,6 @@ public CentralRegistry findRegistry(DatasetId dataset) throws NoSuchElementExcep return datasets.get(dataset).getStorage().getCentralRegistry(); } - @Override - public CentralRegistry getMetaRegistry() { - return metaStorage.getCentralRegistry(); - } - - public List getAllDatasets() { return datasets.values().stream().map(Namespace::getStorage).map(NamespaceStorage::getDataset).collect(Collectors.toList()); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/IdResolveContext.java b/backend/src/main/java/com/bakdata/conquery/models/worker/IdResolveContext.java index 04fec43874..d65f7850c0 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/IdResolveContext.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/IdResolveContext.java @@ -10,7 +10,6 @@ import com.bakdata.conquery.models.identifiable.ids.Id; import com.bakdata.conquery.models.identifiable.ids.NamespacedId; import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; -import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.JsonMappingException; import io.dropwizard.jackson.Jackson; @@ -36,8 +35,6 @@ public MutableInjectableValues inject(MutableInjectableValues values) { } public abstract CentralRegistry findRegistry(DatasetId dataset) throws NoSuchElementException; - @JsonIgnore - public abstract CentralRegistry getMetaRegistry(); public & NamespacedId, T extends Identifiable> T resolve(ID id) { return findRegistry(id.getDataset()).resolve(id); diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/Namespace.java b/backend/src/main/java/com/bakdata/conquery/models/worker/Namespace.java index 191265b8b2..1c952d0de7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/Namespace.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/Namespace.java @@ -122,12 +122,6 @@ public CentralRegistry findRegistry(DatasetId dataset) throws NoSuchElementExcep return storage.getCentralRegistry(); } - @Override - public CentralRegistry getMetaRegistry() { - throw new UnsupportedOperationException(); - } - - /** * Issues a job that initializes the search that is used by the frontend for recommendations in the filter interface of a concept. */ diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/SingletonNamespaceCollection.java b/backend/src/main/java/com/bakdata/conquery/models/worker/SingletonNamespaceCollection.java index 78bc0c5b52..4f1d3799b3 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/SingletonNamespaceCollection.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/SingletonNamespaceCollection.java @@ -8,21 +8,11 @@ @RequiredArgsConstructor public class SingletonNamespaceCollection extends IdResolveContext { - public SingletonNamespaceCollection(CentralRegistry registry) { - this(registry, null); - } - @NonNull private final CentralRegistry registry; - private final CentralRegistry metaRegistry; @Override public CentralRegistry findRegistry(DatasetId dataset) { return registry; } - - @Override - public CentralRegistry getMetaRegistry() { - return metaRegistry; - } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/Worker.java b/backend/src/main/java/com/bakdata/conquery/models/worker/Worker.java index 853830d3de..096554e5c5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/Worker.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/Worker.java @@ -125,11 +125,6 @@ public MessageToManagerNode transform(NamespaceMessage message) { return new ForwardToNamespace(getInfo().getDataset(), message); } - public ObjectMapper inject(ObjectMapper binaryMapper) { - return new SingletonNamespaceCollection(storage.getCentralRegistry()) - .injectIntoNew(binaryMapper); - } - @Override public void close() { // We do not close the executorService here because it does not belong to this class diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/Workers.java b/backend/src/main/java/com/bakdata/conquery/models/worker/Workers.java index 99ddb7e633..395551aa64 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/Workers.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/Workers.java @@ -119,11 +119,6 @@ public CentralRegistry findRegistry(DatasetId dataset) { return dataset2Worker.get(dataset).getStorage().getCentralRegistry(); } - @Override - public CentralRegistry getMetaRegistry() { - return null; // Workers simply have no MetaRegistry. - } - public void removeWorkerFor(DatasetId dataset) { final Worker worker = dataset2Worker.get(dataset); diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/AdminServlet.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/AdminServlet.java index 40a03b87c4..a41e6e8600 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/AdminServlet.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/AdminServlet.java @@ -4,6 +4,7 @@ import java.nio.charset.StandardCharsets; import java.util.Collections; +import jakarta.validation.Validator; import com.bakdata.conquery.commands.ManagerNode; import com.bakdata.conquery.io.freemarker.Freemarker; @@ -18,35 +19,14 @@ import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.jobs.JobManager; import com.bakdata.conquery.models.worker.DatasetRegistry; -import com.bakdata.conquery.resources.admin.rest.AdminConceptsResource; -import com.bakdata.conquery.resources.admin.rest.AdminDatasetProcessor; -import com.bakdata.conquery.resources.admin.rest.AdminDatasetResource; -import com.bakdata.conquery.resources.admin.rest.AdminDatasetsResource; -import com.bakdata.conquery.resources.admin.rest.AdminProcessor; -import com.bakdata.conquery.resources.admin.rest.AdminResource; -import com.bakdata.conquery.resources.admin.rest.AdminTablesResource; -import com.bakdata.conquery.resources.admin.rest.AuthOverviewResource; -import com.bakdata.conquery.resources.admin.rest.GroupResource; -import com.bakdata.conquery.resources.admin.rest.PermissionResource; -import com.bakdata.conquery.resources.admin.rest.RoleResource; -import com.bakdata.conquery.resources.admin.rest.UIProcessor; -import com.bakdata.conquery.resources.admin.rest.UserResource; -import com.bakdata.conquery.resources.admin.ui.AdminUIResource; -import com.bakdata.conquery.resources.admin.ui.AuthOverviewUIResource; -import com.bakdata.conquery.resources.admin.ui.ConceptsUIResource; -import com.bakdata.conquery.resources.admin.ui.DatasetsUIResource; -import com.bakdata.conquery.resources.admin.ui.GroupUIResource; -import com.bakdata.conquery.resources.admin.ui.IndexServiceUIResource; -import com.bakdata.conquery.resources.admin.ui.RoleUIResource; -import com.bakdata.conquery.resources.admin.ui.TablesUIResource; -import com.bakdata.conquery.resources.admin.ui.UserUIResource; +import com.bakdata.conquery.resources.admin.rest.*; +import com.bakdata.conquery.resources.admin.ui.*; import com.bakdata.conquery.resources.admin.ui.model.ConnectorUIResource; import io.dropwizard.core.setup.AdminEnvironment; import io.dropwizard.jersey.DropwizardResourceConfig; import io.dropwizard.jersey.jackson.JacksonMessageBodyProvider; import io.dropwizard.servlets.assets.AssetServlet; import io.dropwizard.views.common.ViewMessageBodyWriter; -import jakarta.validation.Validator; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.glassfish.hk2.utilities.binding.AbstractBinder; @@ -89,7 +69,7 @@ public AdminServlet(ManagerNode manager) { adminProcessor = new AdminProcessor( manager, manager.getConfig(), - manager.getStorage(), + manager.getMetaStorage(), manager.getDatasetRegistry(), manager.getJobManager(), manager.getMaintenanceService(), @@ -101,6 +81,7 @@ public AdminServlet(ManagerNode manager) { manager.getConfig(), manager.getValidator(), manager.getDatasetRegistry(), + manager.getMetaStorage(), manager.getJobManager(), manager.getImportHandler(), manager.getStorageListener() @@ -110,7 +91,7 @@ public AdminServlet(ManagerNode manager) { @Override protected void configure() { bind(manager.getDatasetRegistry()).to(DatasetRegistry.class); - bind(manager.getStorage()).to(MetaStorage.class); + bind(manager.getMetaStorage()).to(MetaStorage.class); bind(manager.getValidator()).to(Validator.class); bind(manager.getJobManager()).to(JobManager.class); bind(manager.getConfig()).to(ConqueryConfig.class); @@ -134,7 +115,7 @@ protected void configure() { bind(adminProcessor).to(AdminProcessor.class); bindAsContract(UIProcessor.class); bind(manager.getDatasetRegistry()).to(DatasetRegistry.class); - bind(manager.getStorage()).to(MetaStorage.class); + bind(manager.getMetaStorage()).to(MetaStorage.class); bind(manager.getConfig()).to(ConqueryConfig.class); } }) diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java index a8adad0a1e..8248842abc 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java @@ -9,6 +9,7 @@ import java.util.Set; import java.util.stream.Collectors; +import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.mode.ImportHandler; import com.bakdata.conquery.mode.StorageListener; import com.bakdata.conquery.models.config.ConqueryConfig; @@ -59,6 +60,7 @@ public class AdminDatasetProcessor { private final ConqueryConfig config; private final Validator validator; private final DatasetRegistry datasetRegistry; + private final MetaStorage metaStorage; private final JobManager jobManager; private final ImportHandler importHandler; private final StorageListener storageListener; @@ -74,7 +76,7 @@ public synchronized Dataset addDataset(Dataset dataset) throws IOException { throw new WebApplicationException("Dataset already exists", Response.Status.CONFLICT); } - return datasetRegistry.createNamespace(dataset, getValidator()).getDataset(); + return datasetRegistry.createNamespace(dataset, metaStorage).getDataset(); } /** diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/ConqueryTestSpec.java b/backend/src/test/java/com/bakdata/conquery/integration/json/ConqueryTestSpec.java index 07e8aea28a..e96c80c276 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/ConqueryTestSpec.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/ConqueryTestSpec.java @@ -4,15 +4,12 @@ import java.util.ArrayList; import java.util.List; import java.util.function.Consumer; - import javax.annotation.Nullable; import com.bakdata.conquery.integration.IntegrationTest; import com.bakdata.conquery.io.cps.CPSBase; import com.bakdata.conquery.io.jackson.Jackson; -import com.bakdata.conquery.io.jackson.MutableInjectableValues; import com.bakdata.conquery.io.jackson.View; -import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.config.Dialect; import com.bakdata.conquery.models.config.IdColumnConfig; @@ -21,7 +18,6 @@ import com.bakdata.conquery.models.identifiable.Identifiable; import com.bakdata.conquery.models.identifiable.ids.Id; import com.bakdata.conquery.models.identifiable.ids.IdUtil; -import com.bakdata.conquery.models.worker.SingletonNamespaceCollection; import com.bakdata.conquery.util.NonPersistentStoreFactory; import com.bakdata.conquery.util.support.StandaloneSupport; import com.bakdata.conquery.util.support.TestSupport; @@ -94,16 +90,12 @@ public static T parseSubTree(TestSupport support, JsonNode node, JavaType ex } public static T parseSubTree(TestSupport support, JsonNode node, JavaType expectedType, Consumer modifierBeforeValidation) throws IOException { - final ObjectMapper om = Jackson.MAPPER.copy(); - ObjectMapper mapper = support.getDataset().injectIntoNew( - new SingletonNamespaceCollection(support.getNamespace().getStorage().getCentralRegistry(), support.getMetaStorage().getCentralRegistry()) - .injectIntoNew( - om.addHandler(new DatasetPlaceHolderFiller(support)) - ) - ); - final MutableInjectableValues injectableValues = (MutableInjectableValues) mapper.getInjectableValues(); - injectableValues.add(ConqueryConfig.class, support.getConfig()); - injectableValues.add(MetaStorage.class, support.getMetaStorage()); + final ObjectMapper mapper = Jackson.copyMapperAndInjectables(Jackson.MAPPER); + support.getDataset().injectInto(mapper); + support.getNamespace().injectInto(mapper); + support.getMetaStorage().injectInto(mapper); + support.getConfig().injectInto(mapper); + mapper.addHandler(new DatasetPlaceHolderFiller(support)); T result = mapper.readerFor(expectedType).readValue(node); @@ -117,13 +109,12 @@ public static T parseSubTree(TestSupport support, JsonNode node, JavaType ex public static List parseSubTreeList(TestSupport support, ArrayNode node, Class expectedType, Consumer modifierBeforeValidation) throws IOException { - final ObjectMapper om = Jackson.MAPPER.copy(); - ObjectMapper mapper = support.getDataset().injectInto( - new SingletonNamespaceCollection(support.getNamespace().getStorage().getCentralRegistry()).injectIntoNew( - om.addHandler(new DatasetPlaceHolderFiller(support)) - ) - ); - support.getNamespace().getInjectables().forEach(i -> i.injectInto(mapper)); + final ObjectMapper mapper = Jackson.copyMapperAndInjectables(Jackson.MAPPER); + support.getDataset().injectInto(mapper); + support.getNamespace().injectInto(mapper); + support.getMetaStorage().injectInto(mapper); + support.getConfig().injectInto(mapper); + mapper.addHandler(new DatasetPlaceHolderFiller(support)); mapper.setConfig(mapper.getDeserializationConfig().withView(View.Api.class)); diff --git a/backend/src/test/java/com/bakdata/conquery/io/AbstractSerializationTest.java b/backend/src/test/java/com/bakdata/conquery/io/AbstractSerializationTest.java index 5c88386bde..8d25bc0197 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/AbstractSerializationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/AbstractSerializationTest.java @@ -3,6 +3,8 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.*; +import jakarta.validation.Validator; + import com.bakdata.conquery.commands.ManagerNode; import com.bakdata.conquery.commands.ShardNode; import com.bakdata.conquery.io.jackson.Jackson; @@ -18,7 +20,6 @@ import com.bakdata.conquery.util.NonPersistentStoreFactory; import com.fasterxml.jackson.databind.ObjectMapper; import io.dropwizard.jersey.validation.Validators; -import jakarta.validation.Validator; import lombok.Getter; import org.junit.jupiter.api.BeforeEach; @@ -37,12 +38,11 @@ public abstract class AbstractSerializationTest { @BeforeEach public void before() { - InternalObjectMapperCreator creator = new InternalObjectMapperCreator(config, validator); + metaStorage = new MetaStorage(new NonPersistentStoreFactory()); + InternalObjectMapperCreator creator = new InternalObjectMapperCreator(config, metaStorage, validator); final IndexService indexService = new IndexService(config.getCsv().createCsvParserSettings(), "emptyDefaultLabel"); final ClusterNamespaceHandler clusterNamespaceHandler = new ClusterNamespaceHandler(new ClusterState(), config, creator); - datasetRegistry = new DatasetRegistry<>(0, config, null, clusterNamespaceHandler, indexService); - metaStorage = new MetaStorage(new NonPersistentStoreFactory(), datasetRegistry); - datasetRegistry.setMetaStorage(metaStorage); + datasetRegistry = new DatasetRegistry<>(0, config, creator, clusterNamespaceHandler, indexService); creator.init(datasetRegistry); // Prepare manager node internal mapper @@ -50,7 +50,7 @@ public void before() { when(managerNode.getConfig()).thenReturn(config); when(managerNode.getValidator()).thenReturn(validator); doReturn(datasetRegistry).when(managerNode).getDatasetRegistry(); - when(managerNode.getStorage()).thenReturn(metaStorage); + when(managerNode.getMetaStorage()).thenReturn(metaStorage); when(managerNode.getInternalObjectMapperCreator()).thenReturn(creator); when(managerNode.createInternalObjectMapper(any())).thenCallRealMethod(); diff --git a/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/IdRefrenceTest.java b/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/IdRefrenceTest.java index 69f77b30f9..cc6cbebfde 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/IdRefrenceTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/IdRefrenceTest.java @@ -7,14 +7,11 @@ import java.util.List; import com.bakdata.conquery.io.jackson.Jackson; -import com.bakdata.conquery.io.jackson.MutableInjectableValues; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.datasets.Table; import com.bakdata.conquery.models.identifiable.CentralRegistry; -import com.bakdata.conquery.models.worker.DatasetRegistry; -import com.bakdata.conquery.models.worker.DistributedNamespace; import com.bakdata.conquery.models.worker.SingletonNamespaceCollection; import com.bakdata.conquery.util.NonPersistentStoreFactory; import com.fasterxml.jackson.annotation.JsonCreator; @@ -27,8 +24,7 @@ public class IdRefrenceTest { @Test public void testListReferences() throws IOException { - final ObjectMapper mapper = Jackson.MAPPER.copy(); - mapper.setInjectableValues(new MutableInjectableValues()); + final ObjectMapper mapper = Jackson.copyMapperAndInjectables(Jackson.MAPPER); CentralRegistry registry = new CentralRegistry(); Dataset dataset = new Dataset(); @@ -39,12 +35,9 @@ public void testListReferences() throws IOException { registry.register(dataset); registry.register(table); - final DatasetRegistry datasetRegistry = new DatasetRegistry<>(0, null, null, null, null); - - final MetaStorage metaStorage = new MetaStorage(new NonPersistentStoreFactory(),datasetRegistry); + final MetaStorage metaStorage = new MetaStorage(new NonPersistentStoreFactory()); metaStorage.openStores(null); - datasetRegistry.setMetaStorage(metaStorage); User user = new User("usermail", "userlabel", metaStorage); @@ -61,14 +54,20 @@ public void testListReferences() throws IOException { .contains("\"user.usermail\"") .contains("\"dataset.table\""); - ListHolder holder = new SingletonNamespaceCollection(registry, metaStorage.getCentralRegistry()) - .injectIntoNew(mapper.readerFor(ListHolder.class)) + new SingletonNamespaceCollection(registry) + .injectInto(mapper); + metaStorage.injectInto(mapper); + ListHolder holder = mapper + .readerFor(ListHolder.class) .readValue(json); assertThat(holder.getUsers().get(0)).isSameAs(user); assertThat(holder.getTables().get(0)).isSameAs(table); } + /** + * @implNote this needs to be a class, because jackson ignores NsIdRefCollection on records + */ @Getter @RequiredArgsConstructor(onConstructor_ = @JsonCreator) public static class ListHolder { diff --git a/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/SerializationTestUtil.java b/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/SerializationTestUtil.java index 10c92e1004..a6a5de9174 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/SerializationTestUtil.java +++ b/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/SerializationTestUtil.java @@ -8,6 +8,7 @@ import java.lang.ref.WeakReference; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.UnaryOperator; +import jakarta.validation.Validator; import com.bakdata.conquery.io.jackson.Injectable; import com.bakdata.conquery.io.jackson.Jackson; @@ -23,7 +24,6 @@ import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.databind.ObjectWriter; import io.dropwizard.jersey.validation.Validators; -import jakarta.validation.Validator; import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.Setter; @@ -116,7 +116,7 @@ public void test(T value) throws JSONException, IOException { private void test(T value, T expected, ObjectMapper mapper) throws IOException { if (registry != null) { - mapper = new SingletonNamespaceCollection(registry, registry).injectInto(mapper); + mapper = new SingletonNamespaceCollection(registry).injectInto(mapper); } for (Injectable injectable : injectables) { mapper = injectable.injectInto(mapper); diff --git a/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/tree/GroovyIndexedTest.java b/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/tree/GroovyIndexedTest.java index d5549e0723..3afe94bb22 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/tree/GroovyIndexedTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/tree/GroovyIndexedTest.java @@ -8,12 +8,10 @@ import java.util.Random; import java.util.function.Supplier; import java.util.stream.Stream; - import jakarta.validation.Validator; import com.bakdata.conquery.io.jackson.Injectable; import com.bakdata.conquery.io.jackson.Jackson; -import com.bakdata.conquery.io.jackson.MutableInjectableValues; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.datasets.Table; @@ -22,7 +20,9 @@ import com.bakdata.conquery.models.exceptions.ConfigurationException; import com.bakdata.conquery.models.exceptions.JSONException; import com.bakdata.conquery.models.identifiable.CentralRegistry; +import com.bakdata.conquery.models.worker.SingletonNamespaceCollection; import com.bakdata.conquery.util.CalculatedValue; +import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.databind.node.ObjectNode; import com.github.powerlibraries.io.In; @@ -87,15 +87,13 @@ public static void init() throws IOException, JSONException, ConfigurationExcept // Prepare Serdes injections - final Validator validator = Validators.newValidator(); - final ObjectReader conceptReader = new Injectable(){ - @Override - public MutableInjectableValues inject(MutableInjectableValues values) { - return values.add(Validator.class, validator); - } - }.injectInto(registry.injectIntoNew(dataset.injectIntoNew(Jackson.MAPPER))).readerFor(Concept.class); - - // load tree twice to to avoid references + ObjectMapper mapper = Jackson.copyMapperAndInjectables(Jackson.MAPPER); + ((Injectable) values -> values.add(Validator.class, Validators.newValidator())).injectInto(mapper); + new SingletonNamespaceCollection(registry).injectInto(mapper); + dataset.injectInto(mapper); + final ObjectReader conceptReader = mapper.readerFor(Concept.class); + + // load tree twice to avoid references indexedConcept = conceptReader.readValue(node); indexedConcept.setDataset(dataset); diff --git a/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java b/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java index c9622d73e1..ae13a560f6 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java +++ b/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java @@ -6,22 +6,14 @@ import java.util.concurrent.ConcurrentHashMap; import com.bakdata.conquery.io.cps.CPSType; -import com.bakdata.conquery.io.storage.IdentifiableStore; -import com.bakdata.conquery.io.storage.MetaStorage; -import com.bakdata.conquery.io.storage.NamespaceStorage; -import com.bakdata.conquery.io.storage.StoreMappings; -import com.bakdata.conquery.io.storage.WorkerStorage; +import com.bakdata.conquery.io.storage.*; import com.bakdata.conquery.io.storage.xodus.stores.CachedStore; import com.bakdata.conquery.io.storage.xodus.stores.SingletonStore; import com.bakdata.conquery.models.auth.entities.Group; import com.bakdata.conquery.models.auth.entities.Role; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.config.StoreFactory; -import com.bakdata.conquery.models.datasets.Dataset; -import com.bakdata.conquery.models.datasets.Import; -import com.bakdata.conquery.models.datasets.PreviewConfig; -import com.bakdata.conquery.models.datasets.SecondaryIdDescription; -import com.bakdata.conquery.models.datasets.Table; +import com.bakdata.conquery.models.datasets.*; import com.bakdata.conquery.models.datasets.concepts.Concept; import com.bakdata.conquery.models.datasets.concepts.StructureNode; import com.bakdata.conquery.models.events.Bucket; @@ -33,7 +25,6 @@ import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; import com.bakdata.conquery.models.index.InternToExternMapper; import com.bakdata.conquery.models.index.search.SearchIndex; -import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.WorkerInformation; import com.bakdata.conquery.models.worker.WorkerToBucketsMap; import com.fasterxml.jackson.databind.ObjectMapper; @@ -153,12 +144,12 @@ public SingletonStore createStructureStore(String pathName, Cen } @Override - public IdentifiableStore createExecutionsStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName, ObjectMapper objectMapper) { + public IdentifiableStore createExecutionsStore(CentralRegistry centralRegistry, String pathName, ObjectMapper objectMapper) { return StoreMappings.identifiable(executionStore.computeIfAbsent(pathName, n -> new NonPersistentStore<>()), centralRegistry); } @Override - public IdentifiableStore createFormConfigStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName, ObjectMapper objectMapper) { + public IdentifiableStore createFormConfigStore(CentralRegistry centralRegistry, String pathName, ObjectMapper objectMapper) { return StoreMappings.identifiable(formConfigStore.computeIfAbsent(pathName, n -> new NonPersistentStore<>()), centralRegistry); } @@ -181,7 +172,7 @@ public IdentifiableStore createGroupStore(CentralRegistry centralRegistry * @implNote intended for Unit-tests */ public MetaStorage createMetaStorage() { - final MetaStorage metaStorage = new MetaStorage(this, null); + final MetaStorage metaStorage = new MetaStorage(this); metaStorage.openStores(null); return metaStorage; } diff --git a/backend/src/test/java/com/bakdata/conquery/util/support/StandaloneSupport.java b/backend/src/test/java/com/bakdata/conquery/util/support/StandaloneSupport.java index 1abc9b0fc9..49042dada3 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/support/StandaloneSupport.java +++ b/backend/src/test/java/com/bakdata/conquery/util/support/StandaloneSupport.java @@ -4,6 +4,11 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import jakarta.validation.Validator; +import jakarta.ws.rs.client.Client; +import jakarta.ws.rs.client.ClientRequestContext; +import jakarta.ws.rs.client.ClientRequestFilter; +import jakarta.ws.rs.core.UriBuilder; import com.bakdata.conquery.commands.PreprocessorCommand; import com.bakdata.conquery.commands.ShardNode; @@ -23,11 +28,6 @@ import com.bakdata.conquery.resources.admin.rest.AdminProcessor; import com.google.common.util.concurrent.MoreExecutors; import io.dropwizard.core.setup.Environment; -import jakarta.validation.Validator; -import jakarta.ws.rs.client.Client; -import jakarta.ws.rs.client.ClientRequestContext; -import jakarta.ws.rs.client.ClientRequestFilter; -import jakarta.ws.rs.core.UriBuilder; import lombok.Data; import lombok.Getter; import lombok.RequiredArgsConstructor; @@ -94,7 +94,7 @@ public Validator getValidator() { } public MetaStorage getMetaStorage() { - return testConquery.getStandaloneCommand().getManagerNode().getStorage(); + return testConquery.getStandaloneCommand().getManagerNode().getMetaStorage(); } public NamespaceStorage getNamespaceStorage() { diff --git a/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java b/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java index 4346306bc3..00f9822cd3 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java +++ b/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java @@ -11,6 +11,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import jakarta.validation.Validator; +import jakarta.ws.rs.client.Client; import com.bakdata.conquery.Conquery; import com.bakdata.conquery.commands.DistributedStandaloneCommand; @@ -36,8 +38,6 @@ import io.dropwizard.client.JerseyClientBuilder; import io.dropwizard.core.cli.Command; import io.dropwizard.testing.DropwizardTestSupport; -import jakarta.validation.Validator; -import jakarta.ws.rs.client.Client; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; @@ -149,7 +149,7 @@ public void afterEach() throws Exception { } openSupports.clear(); } - this.getStandaloneCommand().getManagerNode().getStorage().clear(); + this.getStandaloneCommand().getManagerNode().getMetaStorage().clear(); waitUntilWorkDone(); } @@ -190,7 +190,7 @@ public void waitUntilWorkDone() { } public void beforeEach() { - final MetaStorage storage = standaloneCommand.getManagerNode().getStorage(); + final MetaStorage storage = standaloneCommand.getManagerNode().getMetaStorage(); testUser = standaloneCommand.getManagerNode().getConfig().getAuthorizationRealms().getInitialUsers().get(0).createOrOverwriteUser(storage); storage.updateUser(testUser); } @@ -261,7 +261,7 @@ private boolean isBusy() { boolean busy; busy = standaloneCommand.getManagerNode().getJobManager().isSlowWorkerBusy(); busy |= standaloneCommand.getManagerNode() - .getStorage() + .getMetaStorage() .getAllExecutions() .stream() .map(ManagedExecution::getState)