Skip to content

Commit

Permalink
Merge pull request #2027 from bakdata/feature/resolve-with-prefetch
Browse files Browse the repository at this point in the history 
Adds option to list all elements instead of doing a search
  • Loading branch information
@thoniTUB
thoniTUB authored Aug 27, 2021
2 parents 6d1a472 + 313acc8 commit 4fe38bc
Show file tree
Hide file tree
Showing 7 changed files with 495 additions and 414 deletions.
20 changes: 17 additions & 3 deletions backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FEValue.java
View file
Original file line number Diff line number Diff line change
@@ -1,23 +1,37 @@
package com.bakdata.conquery.apiv1.frontend;

import java.util.Comparator;
import java.util.Map;

import lombok.AllArgsConstructor;
import lombok.Data;
import org.jetbrains.annotations.NotNull;

/**
* This class represents a values of a SELECT filter.
*/
@Data @AllArgsConstructor
public class FEValue {

@Data
@AllArgsConstructor
public class FEValue implements Comparable<FEValue> {
private static final Comparator<FEValue> COMPARATOR = Comparator.comparing(FEValue::getLabel).thenComparing(FEValue::getValue);

@NotNull
private final String label;

@NotNull
private final String value;

private Map<String, String> templateValues;

private String optionValue;

public FEValue(String label, String value) {
this.label = label;
this.value = value;
}

@Override
public int compareTo(@NotNull FEValue o) {
return COMPARATOR.compare(this, o);
}
}
100 changes: 57 additions & 43 deletions backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package com.bakdata.conquery.resources.api;

import java.time.Duration;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
Expand All @@ -18,10 +17,10 @@
import com.bakdata.conquery.apiv1.FilterSearch;
import com.bakdata.conquery.apiv1.FilterSearchItem;
import com.bakdata.conquery.apiv1.IdLabel;
import com.bakdata.conquery.io.storage.NamespaceStorage;
import com.bakdata.conquery.apiv1.frontend.FEList;
import com.bakdata.conquery.apiv1.frontend.FERoot;
import com.bakdata.conquery.apiv1.frontend.FEValue;
import com.bakdata.conquery.io.storage.NamespaceStorage;
import com.bakdata.conquery.models.auth.entities.User;
import com.bakdata.conquery.models.auth.permissions.Ability;
import com.bakdata.conquery.models.datasets.Dataset;
Expand All @@ -39,12 +38,13 @@
import com.bakdata.conquery.util.CalculatedValue;
import com.bakdata.conquery.util.search.QuickSearch;
import com.bakdata.conquery.util.search.SearchScorer;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.collect.ImmutableList;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.Setter;
Expand All @@ -56,7 +56,7 @@
@Slf4j
@RequiredArgsConstructor
public class ConceptsProcessor {

private final DatasetRegistry namespaces;

private final LoadingCache<Concept<?>, FEList> nodeCache =
Expand All @@ -72,7 +72,7 @@ public FEList load(Concept<?> concept) throws Exception {

private final LoadingCache<Pair<AbstractSelectFilter<?>, String>, List<FEValue>> searchCache =
CacheBuilder.newBuilder()
.expireAfterAccess(Duration.ofMinutes(2))
.softValues()
.build(new CacheLoader<>() {

@Override
Expand All @@ -84,18 +84,18 @@ public List<FEValue> load(Pair<AbstractSelectFilter<?>, String> filterAndSearch)
}

});

public FERoot getRoot(NamespaceStorage storage, User user) {

return FrontEndConceptBuilder.createRoot(storage, user);
}

public FEList getNode(Concept<?> concept) {
try {
return nodeCache.get(concept);
}
catch (ExecutionException e) {
throw new RuntimeException("failed to create frontend node for "+concept, e);
throw new RuntimeException("failed to create frontend node for " + concept, e);
}
}

Expand All @@ -117,24 +117,24 @@ public ResolvedConceptsResult resolveFilterValues(AbstractSelectFilter<?> filter

//search in the full text engine
Set<String> searchResult = createSourceSearchResult(filter.getSourceSearch(), searchTerms, OptionalInt.empty(), filter.getSearchType()::score)
.stream()
.map(FEValue::getValue)
.collect(Collectors.toSet());
.stream()
.map(FEValue::getValue)
.collect(Collectors.toSet());

Set<String> openSearchTerms = new HashSet<>(searchTerms);
openSearchTerms.removeAll(searchResult);

// Iterate over all unresolved search terms. Gather all that match labels into searchResults. Keep the unresolvable ones.
for (Iterator<String> it = openSearchTerms.iterator(); it.hasNext();) {
for (Iterator<String> it = openSearchTerms.iterator(); it.hasNext(); ) {
String searchTerm = it.next();
// Test if any of the values occurs directly in the filter's values or their labels (for when we don't have a provided file).
if(filter.getValues().contains(searchTerm)) {
if (filter.getValues().contains(searchTerm)) {
searchResult.add(searchTerm);
it.remove();
}
else {
String matchingValue = filter.getLabels().inverse().get(searchTerm);
if(matchingValue != null) {
if (matchingValue != null) {
searchResult.add(matchingValue);
it.remove();
}
Expand All @@ -158,41 +158,56 @@ public ResolvedConceptsResult resolveFilterValues(AbstractSelectFilter<?> filter
public List<FEValue> autocompleteTextFilter(AbstractSelectFilter<?> filter, String text, OptionalInt pageNumberOpt, OptionalInt itemsPerPageOpt) {
int pageNumber = pageNumberOpt.orElse(0);
int itemsPerPage = itemsPerPageOpt.orElse(50);

if(pageNumber < 0) {
throw new IllegalArgumentException("Page number must be 0 or a positive integer");
}
if(itemsPerPage < 1) {
throw new IllegalArgumentException("Items per page number must be larger than 0");
}
log.trace("Try to generate serach result page {} (with {} results per page) for the term \"{}\".", pageNumber, itemsPerPage, text);


Preconditions.checkArgument(pageNumber > 0, "Page number must be 0 or a positive integer.");
Preconditions.checkArgument(itemsPerPage > 1, "Must at least have one item per page.");

log.trace("Searching for for the term \"{}\". (Page = {}, Items = {})", text, pageNumber, itemsPerPage);

List<FEValue> fullResult = null;
try{
try {
fullResult = searchCache.get(Pair.of(filter, text));
} catch (ExecutionException e) {
log.warn("Could not get a search result for the term \"{}\".", text, log.isTraceEnabled()? e : null);
}
catch (ExecutionException e) {
log.warn("Failed to search for \"{}\".", text, log.isTraceEnabled() ? e : null);
return ImmutableList.of();
}
int startIncl = fullResult.isEmpty()? 0 : Math.min(itemsPerPage*pageNumber, fullResult.size());
int endExcl = Math.min(startIncl + itemsPerPage,fullResult.size());

int startIncl = fullResult.isEmpty() ? 0 : Math.min(itemsPerPage * pageNumber, fullResult.size());
int endExcl = Math.min(startIncl + itemsPerPage, fullResult.size());

log.trace("Preparing subresult for search term \"{}\" in the index range [{}-{})", text, startIncl, endExcl);
return fullResult.subList(startIncl, endExcl);
}

/**
* Autocompletion for search terms. For values of {@link AbstractSelectFilter<?>}.
* Is used by the serach cache to load missing items
*/
private static List<FEValue> autocompleteTextFilter(AbstractSelectFilter<?> filter, String text) {
if (Strings.isNullOrEmpty(text)) {
// If no text provided, we just list them
return filter.getSourceSearch().listItems()
.stream()
.map(item -> new FEValue(item.getLabel(), item.getValue(), item.getTemplateValues(), item.getOptionValue()))
.collect(Collectors.toList());
}

List<FEValue> result = new LinkedList<>();

QuickSearch<FilterSearchItem> search = filter.getSourceSearch();

if (search != null) {
result = createSourceSearchResult(filter.getSourceSearch(), Collections.singletonList(text), OptionalInt.empty(), FilterSearch.FilterSearchType.CONTAINS::score);
result = createSourceSearchResult(
filter.getSourceSearch(),
Collections.singletonList(text),
OptionalInt.empty(),
FilterSearch.FilterSearchType.CONTAINS::score
);
}

String value = filter.getValueFor(text);
if(value != null) {
if (value != null) {
result.add(new FEValue(text, value));
}

Expand All @@ -203,24 +218,23 @@ private static List<FEValue> autocompleteTextFilter(AbstractSelectFilter<?> filt
* Do a search with the supplied values.
*/
private static List<FEValue> createSourceSearchResult(QuickSearch<FilterSearchItem> search, Collection<String> values, OptionalInt numberOfTopItems, SearchScorer scorer) {
if(search == null) {
if (search == null) {
return new ArrayList<>();
}

// Quicksearch can split and also schedule for us.
List<FilterSearchItem> result;
result = search.findItems(String.join(" ", values), numberOfTopItems.orElse(Integer.MAX_VALUE), scorer);

if(numberOfTopItems.isEmpty() && result.size() == Integer.MAX_VALUE) {
List<FilterSearchItem> result = search.findItems(String.join(" ", values), numberOfTopItems.orElse(Integer.MAX_VALUE), scorer);

if (numberOfTopItems.isEmpty() && result.size() == Integer.MAX_VALUE) {
log.warn("The quick search returned the maximum number of results ({}) which probably means not all possible results are returned.", Integer.MAX_VALUE);
}

return result
.stream()
.map(item -> new FEValue(item.getLabel(), item.getValue(), item.getTemplateValues(), item.getOptionValue()))
.collect(Collectors.toList());
.stream()
.map(item -> new FEValue(item.getLabel(), item.getValue(), item.getTemplateValues(), item.getOptionValue()))
.collect(Collectors.toList());
}

public ResolvedConceptsResult resolveConceptElements(TreeConcept concept, List<String> conceptCodes) {
List<ConceptElementId<?>> resolvedCodes = new ArrayList<>();
List<String> unknownCodes = new ArrayList<>();
Expand All @@ -241,12 +255,12 @@ public ResolvedConceptsResult resolveConceptElements(TreeConcept concept, List<S
}
}
catch (ConceptConfigurationException e) {
log.error("Error while trying to resolve "+conceptCode, e);
log.error("Error while trying to resolve " + conceptCode, e);
}
}
return new ResolvedConceptsResult(resolvedCodes, null, unknownCodes);
}

@Getter
@Setter
@AllArgsConstructor
Expand Down
32 changes: 16 additions & 16 deletions backend/src/main/java/com/bakdata/conquery/resources/api/FilterResource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,12 @@
import static com.bakdata.conquery.resources.ResourceConstants.*;

import java.util.List;
import java.util.Objects;
import java.util.OptionalInt;

import javax.inject.Inject;
import javax.servlet.http.HttpServletRequest;
import javax.validation.constraints.NotNull;
import javax.validation.Valid;
import javax.ws.rs.Consumes;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
Expand All @@ -17,51 +18,50 @@
import javax.ws.rs.core.Context;
import javax.ws.rs.core.Response.Status;

import com.bakdata.conquery.io.jersey.ExtraMimeTypes;
import com.bakdata.conquery.apiv1.frontend.FEValue;
import com.bakdata.conquery.io.jersey.ExtraMimeTypes;
import com.bakdata.conquery.models.datasets.concepts.filters.specific.AbstractSelectFilter;
import com.bakdata.conquery.resources.api.ConceptsProcessor.ResolvedConceptsResult;
import com.bakdata.conquery.resources.hierarchies.HFilters;
import io.dropwizard.logback.shaded.checkerframework.checker.nullness.qual.Nullable;
import lombok.Data;
import lombok.Setter;
import org.apache.commons.lang3.StringUtils;

@Setter
@Produces({ExtraMimeTypes.JSON_STRING, ExtraMimeTypes.SMILE_STRING})
@Consumes({ExtraMimeTypes.JSON_STRING, ExtraMimeTypes.SMILE_STRING})
@Path("datasets/{" + DATASET + "}/concepts/{" + CONCEPT + "}/tables/{" + TABLE + "}/filters/{" + FILTER + "}")
public class FilterResource extends HFilters {

@Inject
protected ConceptsProcessor processor;

@POST
@Path("resolve")
public ResolvedConceptsResult resolveFilterValues(FilterValues filterValues) {
return processor.resolveFilterValues((AbstractSelectFilter<?>)filter, filterValues.getValues());
return processor.resolveFilterValues((AbstractSelectFilter<?>) filter, filterValues.getValues());
}

@POST
@Path("autocomplete")
public List<FEValue> autocompleteTextFilter(@NotNull StringContainer text, @Context HttpServletRequest req, @QueryParam("page") OptionalInt pageNumberOpt, @QueryParam("pageSize") OptionalInt itemsPerPageOpt) {
if(StringUtils.isEmpty(text.getText())) {
throw new WebApplicationException("Too short text. Requires at least 1 characters.", Status.BAD_REQUEST);
}
if(!(filter instanceof AbstractSelectFilter)) {
throw new WebApplicationException(filter.getId()+" is not a SELECT filter, but "+filter.getClass().getSimpleName()+".", Status.BAD_REQUEST);
public List<FEValue> autocompleteTextFilter(@Valid StringContainer text, @Context HttpServletRequest req, @QueryParam("page") OptionalInt pageNumberOpt, @QueryParam("pageSize") OptionalInt itemsPerPageOpt) {

if (!(filter instanceof AbstractSelectFilter)) {
throw new WebApplicationException(filter.getId() + " is not a SELECT filter, but " + filter.getClass().getSimpleName() + ".", Status.BAD_REQUEST);
}


return processor.autocompleteTextFilter((AbstractSelectFilter<?>) filter, text.getText(), pageNumberOpt, itemsPerPageOpt);

return processor.autocompleteTextFilter((AbstractSelectFilter<?>) filter, Objects.requireNonNullElse(text.getText(), ""), pageNumberOpt, itemsPerPageOpt);
}

@Data
public static class FilterValues {
private List<String> values;
}

@Data
public static class StringContainer {
@Nullable
private String text;
}
}
4 changes: 4 additions & 0 deletions backend/src/main/java/com/bakdata/conquery/util/search/QuickSearch.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -316,6 +316,10 @@ public List<T> findItems(final String searchString, final int numberOfTopItems)
return findItems(searchString, numberOfTopItems, keywordMatchScorer);
}

public List<T> listItems() {
return graph.listItems();
}

/**
* Retrieve (find) top n items matching the supplied com.bakdata.conquery.util.search string.
*
Expand Down
Loading

0 comments on commit 4fe38bc

Please sign in to comment.