diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/FilterSearch.java b/backend/src/main/java/com/bakdata/conquery/apiv1/FilterSearch.java index c0c94c343c..0339f0d2b6 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/FilterSearch.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/FilterSearch.java @@ -26,14 +26,18 @@ import com.bakdata.conquery.models.jobs.SimpleJob; import com.bakdata.conquery.util.search.TrieSearch; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.google.common.base.Functions; import com.google.common.collect.Sets; -import lombok.Value; +import it.unimi.dsi.fastutil.objects.Object2LongAVLTreeMap; +import it.unimi.dsi.fastutil.objects.Object2LongMap; +import it.unimi.dsi.fastutil.objects.Object2LongMaps; +import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.time.StopWatch; @Slf4j -@Value +@Data public class FilterSearch { private final NamespaceStorage storage; @@ -48,6 +52,7 @@ public class FilterSearch { */ @JsonIgnore private final Map> searchCache = new HashMap<>(); + private Object2LongMap> totals = Object2LongMaps.emptyMap(); /** * From a given {@link FEValue} extract all relevant keywords. @@ -75,6 +80,10 @@ public List> getSearchesFor(SelectFilter filter) { .collect(Collectors.toList()); } + public long getTotal(SelectFilter filter) { + return totals.getOrDefault(filter, 0); + } + /** * Scan all SelectFilters and submit {@link SimpleJob}s to create interactive searches for them. @@ -159,6 +168,23 @@ public void updateSearch() { log.debug("Still waiting for {} to finish.", Sets.difference(collectedSearchables, synchronizedResult.keySet())); } + log.debug("BEGIN counting Search totals."); + + + // Precompute totals as that can be extremly slow. + totals = new Object2LongAVLTreeMap<>( + allSelectFilters.parallelStream() + .collect(Collectors.toMap( + Functions.identity(), + filter -> filter.getSearchReferences().stream() + .map(synchronizedResult::get) + .flatMap(TrieSearch::stream) + .mapToInt(FEValue::hashCode) + .distinct() + .count() + ))); + + log.debug("DONE loading SourceSearch"); } )); diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java index 4611bee134..88efe59c87 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java @@ -15,9 +15,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import javax.validation.Validator; - import javax.inject.Inject; +import javax.validation.Validator; import com.bakdata.conquery.apiv1.IdLabel; import com.bakdata.conquery.apiv1.frontend.FEList; @@ -54,7 +53,6 @@ import lombok.AllArgsConstructor; import lombok.Data; import lombok.Getter; -import lombok.NoArgsConstructor; import lombok.RequiredArgsConstructor; import lombok.Setter; import lombok.ToString; @@ -273,10 +271,7 @@ private long countAllValues(SelectFilter filter) { final Namespace namespace = namespaces.get(filter.getDataset().getId()); - return namespace.getFilterSearch() - .getSearchesFor(filter).stream() - .mapToLong(TrieSearch::calculateSize) - .sum(); + return namespace.getFilterSearch().getTotal(filter); } /** diff --git a/docs/REST API JSONs.md b/docs/REST API JSONs.md index ba5d5289a6..d9857c8d23 100644 --- a/docs/REST API JSONs.md +++ b/docs/REST API JSONs.md @@ -1297,7 +1297,7 @@ Supported Fields: | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/ExecutionStatus.java#L25) | tags | list of `String` | `null` | | |

-### Type ResolvedConceptsResult [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L359) +### Type ResolvedConceptsResult [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L354)
Details

@@ -1308,12 +1308,12 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L364) | resolvedConcepts | `Set>` | ? | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L365) | resolvedFilter | [ResolvedFilterResult](#Type-ResolvedFilterResult) | ? | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L366) | unknownCodes | `Collection` | ? | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L359) | resolvedConcepts | `Set>` | ? | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L360) | resolvedFilter | [ResolvedFilterResult](#Type-ResolvedFilterResult) | ? | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L361) | unknownCodes | `Collection` | ? | | |

-### Type ResolvedFilterResult [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L349) +### Type ResolvedFilterResult [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L344)
Details

@@ -1324,9 +1324,9 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L355) | filterId | ID of `Filter` | ? | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L354) | tableId | ID of `Connector` | ? | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L356) | value | `Collection` | ? | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L350) | filterId | ID of `Filter` | ? | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L349) | tableId | ID of `Connector` | ? | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L351) | value | `Collection` | ? | | |

### Type ValidityDateContainer [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/ValidityDateContainer.java#L9)