From 64091d28004578de76c423a00be7bde822f49a8e Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 27 Jul 2022 11:49:11 +0200 Subject: [PATCH 1/3] precoumpte totals to give exact response fast --- .../bakdata/conquery/apiv1/FilterSearch.java | 27 +++++++++++++++++-- .../resources/api/ConceptsProcessor.java | 9 ++----- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/FilterSearch.java b/backend/src/main/java/com/bakdata/conquery/apiv1/FilterSearch.java index c0c94c343c..5b3ab47678 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/FilterSearch.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/FilterSearch.java @@ -26,14 +26,18 @@ import com.bakdata.conquery.models.jobs.SimpleJob; import com.bakdata.conquery.util.search.TrieSearch; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.google.common.base.Functions; import com.google.common.collect.Sets; -import lombok.Value; +import it.unimi.dsi.fastutil.objects.Object2LongAVLTreeMap; +import it.unimi.dsi.fastutil.objects.Object2LongMap; +import it.unimi.dsi.fastutil.objects.Object2LongMaps; +import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.time.StopWatch; @Slf4j -@Value +@Data public class FilterSearch { private final NamespaceStorage storage; @@ -48,6 +52,7 @@ public class FilterSearch { */ @JsonIgnore private final Map> searchCache = new HashMap<>(); + private Object2LongMap> totals = Object2LongMaps.emptyMap(); /** * From a given {@link FEValue} extract all relevant keywords. @@ -75,6 +80,10 @@ public List> getSearchesFor(SelectFilter filter) { .collect(Collectors.toList()); } + public long getTotal(SelectFilter filter) { + return totals.getOrDefault(filter, 0); + } + /** * Scan all SelectFilters and submit {@link SimpleJob}s to create interactive searches for them. @@ -159,6 +168,20 @@ public void updateSearch() { log.debug("Still waiting for {} to finish.", Sets.difference(collectedSearchables, synchronizedResult.keySet())); } + // Precompute totals as that can be extremly slow. + totals = new Object2LongAVLTreeMap<>( + allSelectFilters.parallelStream() + .collect(Collectors.toMap( + Functions.identity(), + filter -> filter.getSearchReferences().stream() + .map(synchronizedResult::get) + .flatMap(TrieSearch::stream) + .mapToInt(FEValue::hashCode) + .distinct() + .count() + ))); + + log.debug("DONE loading SourceSearch"); } )); diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java index 4611bee134..88efe59c87 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java @@ -15,9 +15,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import javax.validation.Validator; - import javax.inject.Inject; +import javax.validation.Validator; import com.bakdata.conquery.apiv1.IdLabel; import com.bakdata.conquery.apiv1.frontend.FEList; @@ -54,7 +53,6 @@ import lombok.AllArgsConstructor; import lombok.Data; import lombok.Getter; -import lombok.NoArgsConstructor; import lombok.RequiredArgsConstructor; import lombok.Setter; import lombok.ToString; @@ -273,10 +271,7 @@ private long countAllValues(SelectFilter filter) { final Namespace namespace = namespaces.get(filter.getDataset().getId()); - return namespace.getFilterSearch() - .getSearchesFor(filter).stream() - .mapToLong(TrieSearch::calculateSize) - .sum(); + return namespace.getFilterSearch().getTotal(filter); } /** From 5caa993b8be04dfbba5857de1c628830f20e06ec Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 27 Jul 2022 09:52:05 +0000 Subject: [PATCH 2/3] Update AutoDoc --- docs/REST API JSONs.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/REST API JSONs.md b/docs/REST API JSONs.md index ba5d5289a6..d9857c8d23 100644 --- a/docs/REST API JSONs.md +++ b/docs/REST API JSONs.md @@ -1297,7 +1297,7 @@ Supported Fields: | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/ExecutionStatus.java#L25) | tags | list of `String` | `null` | | |

-### Type ResolvedConceptsResult [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L359) +### Type ResolvedConceptsResult [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L354)
Details

@@ -1308,12 +1308,12 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L364) | resolvedConcepts | `Set>` | ? | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L365) | resolvedFilter | [ResolvedFilterResult](#Type-ResolvedFilterResult) | ? | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L366) | unknownCodes | `Collection` | ? | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L359) | resolvedConcepts | `Set>` | ? | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L360) | resolvedFilter | [ResolvedFilterResult](#Type-ResolvedFilterResult) | ? | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L361) | unknownCodes | `Collection` | ? | | |

-### Type ResolvedFilterResult [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L349) +### Type ResolvedFilterResult [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L344)
Details

@@ -1324,9 +1324,9 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L355) | filterId | ID of `Filter` | ? | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L354) | tableId | ID of `Connector` | ? | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L356) | value | `Collection` | ? | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L350) | filterId | ID of `Filter` | ? | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L349) | tableId | ID of `Connector` | ? | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L351) | value | `Collection` | ? | | |

### Type ValidityDateContainer [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/ValidityDateContainer.java#L9) From c69d430deec5018bbf0d2ed5ea8a9d3b14864b48 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 27 Jul 2022 12:24:42 +0200 Subject: [PATCH 3/3] add logging --- .../src/main/java/com/bakdata/conquery/apiv1/FilterSearch.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/FilterSearch.java b/backend/src/main/java/com/bakdata/conquery/apiv1/FilterSearch.java index 5b3ab47678..0339f0d2b6 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/FilterSearch.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/FilterSearch.java @@ -168,6 +168,9 @@ public void updateSearch() { log.debug("Still waiting for {} to finish.", Sets.difference(collectedSearchables, synchronizedResult.keySet())); } + log.debug("BEGIN counting Search totals."); + + // Precompute totals as that can be extremly slow. totals = new Object2LongAVLTreeMap<>( allSelectFilters.parallelStream()