diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/DirectIdentifiableStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/DirectIdentifiableStore.java index c43c91fe02..a4041183a8 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/DirectIdentifiableStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/DirectIdentifiableStore.java @@ -5,6 +5,8 @@ import com.bakdata.conquery.models.identifiable.ids.IId; import com.bakdata.conquery.util.functions.ThrowingConsumer; +import java.util.Optional; + /** * Registered items are directly referenced. Compare to {@link IdentifiableCachedStore} */ @@ -46,4 +48,23 @@ protected void added(VALUE value) { throw new RuntimeException("Failed to add "+value, e); } } + + @Override + protected void updated(VALUE value) { + try { + if (value == null) { + return; + } + final Optional> old = centralRegistry.getOptional(value.getId()); + + if (old.isPresent()) { + onRemove.accept((VALUE) old.get()); + } + + centralRegistry.update(value); + onAdd.accept(value); + } catch(Exception e) { + throw new RuntimeException("Failed to add "+value, e); + } + } } diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/IdentifiableCachedStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/IdentifiableCachedStore.java index b4c97e045f..bd734547eb 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/IdentifiableCachedStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/IdentifiableCachedStore.java @@ -8,6 +8,9 @@ import lombok.Setter; import lombok.experimental.Accessors; +import java.util.Optional; +import java.util.function.Function; + /** * Registers accessors of values instead of the value itself to the central registry. * Might be useful if the object are very large and should only be loaded on demand. @@ -31,6 +34,7 @@ protected IId extractKey(VALUE value) { protected void removed(VALUE value) { try { if(value != null) { + onRemove.accept(value); centralRegistry.remove(value); } } catch(Exception e) { @@ -44,6 +48,24 @@ protected void added(VALUE value) { if(value != null) { final IId key = extractKey(value); centralRegistry.registerCacheable(key, this::get); + onAdd.accept(value); + } + } catch(Exception e) { + throw new RuntimeException("Failed to add "+value, e); + } + } + + @Override + protected void updated(VALUE value) { + try { + if(value != null) { + final IId key = extractKey(value); + final Optional oldOpt = centralRegistry.updateCacheable(key, this::get); + if (oldOpt.isPresent()) { + final VALUE old = (VALUE) oldOpt.get(); + onRemove.accept(old); + } + onAdd.accept(value); } } catch(Exception e) { throw new RuntimeException("Failed to add "+value, e); diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java index de44da278d..ee2848a523 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java @@ -71,7 +71,8 @@ public Dictionary getPrimaryDictionaryRaw() { private void decorateIdMapping(SingletonStore idMapping) { - idMapping.onAdd(mapping -> mapping.setStorage(this)); + idMapping + .onAdd(mapping -> mapping.setStorage(this)); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java index 6158a6791e..807a728128 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java @@ -113,8 +113,9 @@ public void removeStorage() { protected abstract boolean isRegisterImports(); private void decorateDatasetStore(SingletonStore store) { - store.onAdd(centralRegistry::register) - .onRemove(centralRegistry::remove); + store + .onAdd(centralRegistry::register) + .onRemove(centralRegistry::remove); } private void decorateSecondaryIdDescriptionStore(IdentifiableStore store) { diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/KeyIncludingStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/KeyIncludingStore.java index 6dfeee74e1..5178a0b466 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/KeyIncludingStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/KeyIncludingStore.java @@ -31,11 +31,8 @@ public void forEach(Consumer consumer) { } public void update(VALUE value) { - VALUE old = get(extractKey(value)); - if(old != null) - removed(old); + updated(value); store.update(extractKey(value), value); - added(value); } public void remove(KEY key) { @@ -69,6 +66,8 @@ public String toString() { protected abstract void added(VALUE value); + protected abstract void updated(VALUE value); + public void clear() { store.clear(); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SingletonStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SingletonStore.java index ad96ad4bed..1835e9f7db 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SingletonStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SingletonStore.java @@ -49,7 +49,8 @@ public void remove(Boolean key) { public void remove() { super.remove(Boolean.TRUE); } - + + @Override protected void removed(VALUE value) { try { if(value != null) { @@ -60,6 +61,7 @@ protected void removed(VALUE value) { } } + @Override protected void added(VALUE value) { try { if(value != null) { @@ -69,4 +71,19 @@ protected void added(VALUE value) { throw new RuntimeException("Failed to add "+value, e); } } + + @Override + protected void updated(VALUE value) { + try { + if(value != null) { + final VALUE old = get(); + if (old != null) { + onRemove.accept(old); + } + onAdd.accept(value); + } + } catch(Exception e) { + throw new RuntimeException("Failed to update "+value, e); + } + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java index c3ce35368b..cd81fe434b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java @@ -263,6 +263,7 @@ public IdentifiableStore createDictionaryStore(CentralRegistry centr final Store, Dictionary> result; + // TODO this looks like dictionaries are double cached if (useWeakDictionaryCaching) { result = new WeakCachedStore<>(bigStore, getWeakCacheDuration()); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/CentralRegistry.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/CentralRegistry.java index 00fb54b811..c9966c02a2 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/CentralRegistry.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/CentralRegistry.java @@ -29,21 +29,8 @@ public synchronized CentralRegistry register(Identifiable ident) { return this; } - public synchronized void registerCacheable(IId id, Function supplier) { - cacheables.put(id, supplier); - } - - protected > T get(IId name) { - Object res = map.get(name); - if (res != null) { - return (T) res; - } - - Function supplier = cacheables.get(name); - if (supplier == null) { - return null; - } - return (T) supplier.apply(name); + public synchronized Function registerCacheable(IId id, Function supplier) { + return cacheables.put(id, supplier); } public > T resolve(IId name) { @@ -56,16 +43,28 @@ public > T resolve(IId name) { return result; } + public Identifiable update(Identifiable ident){ + return map.update(ident); + } + + public synchronized Optional updateCacheable(IId id, Function supplier) { + Function old = cacheables.put(id, supplier); + if (old != null) { + // If the cacheable was still there, the Object was never cached. + return Optional.empty(); + } + // The supplier might have been invoked already and the object gone into the IdMap + // So we invalidate it + return Optional.ofNullable(map.remove(id)); + } + public > Optional getOptional(IId name) { return Optional.ofNullable(get(name)); } - public void remove(Identifiable ident) { + public synchronized void remove(Identifiable ident) { IId id = ident.getId(); - synchronized (this) { - map.remove(id); - cacheables.remove(id); - } + map.remove(id); } @Override @@ -89,4 +88,32 @@ public void clear() { map.clear(); cacheables.clear(); } + + /** + * Needs to be protected in order to be overwritten by {@link InjectingCentralRegistry} + */ + protected > T get(IId name) { + Object res = map.get(name); + if (res != null) { + return (T) res; + } + synchronized (this) { + // Retry synchronized to make sure it has not been resolved from cacheables in the mean time + Object res2 = map.get(name); + if (res2 != null) { + return (T) res2; + } + Function supplier = cacheables.get(name); + if (supplier == null) { + return null; + } + + // Transfer object to the IdMap + final T apply = (T) supplier.apply(name); + register(apply); + cacheables.remove(name); + } + + return (T) map.get(name); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/IdMap.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/IdMap.java index 6c48293a8c..847d27c273 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/IdMap.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/IdMap.java @@ -79,8 +79,8 @@ public boolean add(V entry) { return true; } - public void update(V entry) { - map.put(entry.getId(), entry); + public V update(V entry) { + return map.put((ID)entry.getId(), entry); } public V remove(ID id) {