Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adds option to list all elements instead of doing a search #2027

Merged
merged 8 commits into from
Aug 27, 2021
Merged

Adds option to list all elements instead of doing a search #2027

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
cfaf0d6
Adds option to list all elements instead of doing a proper search
awildturtok Aug 13, 2021
5a4dd87
Update AutoDoc
github-actions[bot] Aug 17, 2021
c97d651
push full list down into cache
awildturtok Aug 24, 2021
c5d8e34
cleanup
awildturtok Aug 24, 2021
6dbbdb1
Update AutoDoc
github-actions[bot] Aug 24, 2021
05ba4af
adds test asserting the order of output values
awildturtok Aug 24, 2021
51f5da9
sort by Value and not Keyword
awildturtok Aug 24, 2021
313acc8
Update AutoDoc
github-actions[bot] Aug 24, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 52 additions & 40 deletions backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package com.bakdata.conquery.resources.api;

import java.time.Duration;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
Expand All @@ -18,10 +17,10 @@
import com.bakdata.conquery.apiv1.FilterSearch;
import com.bakdata.conquery.apiv1.FilterSearchItem;
import com.bakdata.conquery.apiv1.IdLabel;
import com.bakdata.conquery.io.storage.NamespaceStorage;
import com.bakdata.conquery.apiv1.frontend.FEList;
import com.bakdata.conquery.apiv1.frontend.FERoot;
import com.bakdata.conquery.apiv1.frontend.FEValue;
import com.bakdata.conquery.io.storage.NamespaceStorage;
import com.bakdata.conquery.models.auth.entities.User;
import com.bakdata.conquery.models.auth.permissions.Ability;
import com.bakdata.conquery.models.datasets.Dataset;
Expand All @@ -39,12 +38,13 @@
import com.bakdata.conquery.util.CalculatedValue;
import com.bakdata.conquery.util.search.QuickSearch;
import com.bakdata.conquery.util.search.SearchScorer;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.collect.ImmutableList;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.Setter;
Expand All @@ -56,12 +56,13 @@
@Slf4j
@RequiredArgsConstructor
public class ConceptsProcessor {

private final DatasetRegistry namespaces;

private final LoadingCache<Concept<?>, FEList> nodeCache =
CacheBuilder.newBuilder()
.softValues()
.weakKeys()
.expireAfterWrite(10, TimeUnit.MINUTES)
.build(new CacheLoader<Concept<?>, FEList>() {
@Override
Expand All @@ -72,7 +73,8 @@ public FEList load(Concept<?> concept) throws Exception {

private final LoadingCache<Pair<AbstractSelectFilter<?>, String>, List<FEValue>> searchCache =
CacheBuilder.newBuilder()
.expireAfterAccess(Duration.ofMinutes(2))
.softValues()
.weakKeys()
thoniTUB marked this conversation as resolved.
Show resolved Hide resolved
.build(new CacheLoader<>() {

@Override
Expand All @@ -84,18 +86,18 @@ public List<FEValue> load(Pair<AbstractSelectFilter<?>, String> filterAndSearch)
}

});

public FERoot getRoot(NamespaceStorage storage, User user) {

return FrontEndConceptBuilder.createRoot(storage, user);
}

public FEList getNode(Concept<?> concept) {
try {
return nodeCache.get(concept);
}
catch (ExecutionException e) {
throw new RuntimeException("failed to create frontend node for "+concept, e);
throw new RuntimeException("failed to create frontend node for " + concept, e);
}
}

Expand All @@ -117,24 +119,24 @@ public ResolvedConceptsResult resolveFilterValues(AbstractSelectFilter<?> filter

//search in the full text engine
Set<String> searchResult = createSourceSearchResult(filter.getSourceSearch(), searchTerms, OptionalInt.empty(), filter.getSearchType()::score)
.stream()
.map(FEValue::getValue)
.collect(Collectors.toSet());
.stream()
.map(FEValue::getValue)
.collect(Collectors.toSet());

Set<String> openSearchTerms = new HashSet<>(searchTerms);
openSearchTerms.removeAll(searchResult);

// Iterate over all unresolved search terms. Gather all that match labels into searchResults. Keep the unresolvable ones.
for (Iterator<String> it = openSearchTerms.iterator(); it.hasNext();) {
for (Iterator<String> it = openSearchTerms.iterator(); it.hasNext(); ) {
String searchTerm = it.next();
// Test if any of the values occurs directly in the filter's values or their labels (for when we don't have a provided file).
if(filter.getValues().contains(searchTerm)) {
if (filter.getValues().contains(searchTerm)) {
searchResult.add(searchTerm);
it.remove();
}
else {
String matchingValue = filter.getLabels().inverse().get(searchTerm);
if(matchingValue != null) {
if (matchingValue != null) {
searchResult.add(matchingValue);
it.remove();
}
Expand All @@ -158,27 +160,36 @@ public ResolvedConceptsResult resolveFilterValues(AbstractSelectFilter<?> filter
public List<FEValue> autocompleteTextFilter(AbstractSelectFilter<?> filter, String text, OptionalInt pageNumberOpt, OptionalInt itemsPerPageOpt) {
int pageNumber = pageNumberOpt.orElse(0);
int itemsPerPage = itemsPerPageOpt.orElse(50);

if(pageNumber < 0) {
throw new IllegalArgumentException("Page number must be 0 or a positive integer");
}
if(itemsPerPage < 1) {
throw new IllegalArgumentException("Items per page number must be larger than 0");

Preconditions.checkArgument(pageNumber > 0, "Page number must be 0 or a positive integer.");
Preconditions.checkArgument(itemsPerPage > 1, "Must at least have one item per page.");

log.trace("Searching for for the term \"{}\". (Page = {}, Items = {})", text, pageNumber, itemsPerPage);

if (Strings.isNullOrEmpty(text)) {
// If no text provided, we just list them
return filter.getSourceSearch().listItems(pageNumber * itemsPerPage, itemsPerPage)
.stream()
.map(item -> new FEValue(item.getLabel(), item.getValue(), item.getTemplateValues(), item.getOptionValue()))
.collect(Collectors.toList());
}
log.trace("Try to generate serach result page {} (with {} results per page) for the term \"{}\".", pageNumber, itemsPerPage, text);


List<FEValue> fullResult = null;
try{
try {
fullResult = searchCache.get(Pair.of(filter, text));
} catch (ExecutionException e) {
log.warn("Could not get a search result for the term \"{}\".", text, log.isTraceEnabled()? e : null);
}
catch (ExecutionException e) {
log.warn("Failed to search for \"{}\".", text, log.isTraceEnabled() ? e : null);
return ImmutableList.of();
}
int startIncl = fullResult.isEmpty()? 0 : Math.min(itemsPerPage*pageNumber, fullResult.size());
int endExcl = Math.min(startIncl + itemsPerPage,fullResult.size());

int startIncl = fullResult.isEmpty() ? 0 : Math.min(itemsPerPage * pageNumber, fullResult.size());
int endExcl = Math.min(startIncl + itemsPerPage, fullResult.size());

log.trace("Preparing subresult for search term \"{}\" in the index range [{}-{})", text, startIncl, endExcl);
return fullResult.subList(startIncl, endExcl);
}

/**
* Autocompletion for search terms. For values of {@link AbstractSelectFilter<?>}.
* Is used by the serach cache to load missing items
Expand All @@ -188,11 +199,12 @@ private static List<FEValue> autocompleteTextFilter(AbstractSelectFilter<?> filt

QuickSearch<FilterSearchItem> search = filter.getSourceSearch();
if (search != null) {
result = createSourceSearchResult(filter.getSourceSearch(), Collections.singletonList(text), OptionalInt.empty(), FilterSearch.FilterSearchType.CONTAINS::score);
result =
createSourceSearchResult(filter.getSourceSearch(), Collections.singletonList(text), OptionalInt.empty(), FilterSearch.FilterSearchType.CONTAINS::score);
thoniTUB marked this conversation as resolved.
Show resolved Hide resolved
}

String value = filter.getValueFor(text);
if(value != null) {
if (value != null) {
result.add(new FEValue(text, value));
}

Expand All @@ -203,24 +215,24 @@ private static List<FEValue> autocompleteTextFilter(AbstractSelectFilter<?> filt
* Do a search with the supplied values.
*/
private static List<FEValue> createSourceSearchResult(QuickSearch<FilterSearchItem> search, Collection<String> values, OptionalInt numberOfTopItems, SearchScorer scorer) {
if(search == null) {
if (search == null) {
return new ArrayList<>();
}

// Quicksearch can split and also schedule for us.
List<FilterSearchItem> result;
result = search.findItems(String.join(" ", values), numberOfTopItems.orElse(Integer.MAX_VALUE), scorer);
if(numberOfTopItems.isEmpty() && result.size() == Integer.MAX_VALUE) {

if (numberOfTopItems.isEmpty() && result.size() == Integer.MAX_VALUE) {
log.warn("The quick search returned the maximum number of results ({}) which probably means not all possible results are returned.", Integer.MAX_VALUE);
}

return result
.stream()
.map(item -> new FEValue(item.getLabel(), item.getValue(), item.getTemplateValues(), item.getOptionValue()))
.collect(Collectors.toList());
.stream()
.map(item -> new FEValue(item.getLabel(), item.getValue(), item.getTemplateValues(), item.getOptionValue()))
.collect(Collectors.toList());
}

public ResolvedConceptsResult resolveConceptElements(TreeConcept concept, List<String> conceptCodes) {
List<ConceptElementId<?>> resolvedCodes = new ArrayList<>();
List<String> unknownCodes = new ArrayList<>();
Expand All @@ -241,12 +253,12 @@ public ResolvedConceptsResult resolveConceptElements(TreeConcept concept, List<S
}
}
catch (ConceptConfigurationException e) {
log.error("Error while trying to resolve "+conceptCode, e);
log.error("Error while trying to resolve " + conceptCode, e);
}
}
return new ResolvedConceptsResult(resolvedCodes, null, unknownCodes);
}

@Getter
@Setter
@AllArgsConstructor
Expand Down
32 changes: 16 additions & 16 deletions backend/src/main/java/com/bakdata/conquery/resources/api/FilterResource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,12 @@
import static com.bakdata.conquery.resources.ResourceConstants.*;

import java.util.List;
import java.util.Objects;
import java.util.OptionalInt;

import javax.inject.Inject;
import javax.servlet.http.HttpServletRequest;
import javax.validation.constraints.NotNull;
import javax.validation.Valid;
import javax.ws.rs.Consumes;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
Expand All @@ -17,51 +18,50 @@
import javax.ws.rs.core.Context;
import javax.ws.rs.core.Response.Status;

import com.bakdata.conquery.io.jersey.ExtraMimeTypes;
import com.bakdata.conquery.apiv1.frontend.FEValue;
import com.bakdata.conquery.io.jersey.ExtraMimeTypes;
import com.bakdata.conquery.models.datasets.concepts.filters.specific.AbstractSelectFilter;
import com.bakdata.conquery.resources.api.ConceptsProcessor.ResolvedConceptsResult;
import com.bakdata.conquery.resources.hierarchies.HFilters;
import io.dropwizard.logback.shaded.checkerframework.checker.nullness.qual.Nullable;
import lombok.Data;
import lombok.Setter;
import org.apache.commons.lang3.StringUtils;

@Setter
@Produces({ExtraMimeTypes.JSON_STRING, ExtraMimeTypes.SMILE_STRING})
@Consumes({ExtraMimeTypes.JSON_STRING, ExtraMimeTypes.SMILE_STRING})
@Path("datasets/{" + DATASET + "}/concepts/{" + CONCEPT + "}/tables/{" + TABLE + "}/filters/{" + FILTER + "}")
public class FilterResource extends HFilters {

@Inject
protected ConceptsProcessor processor;

@POST
@Path("resolve")
public ResolvedConceptsResult resolveFilterValues(FilterValues filterValues) {
return processor.resolveFilterValues((AbstractSelectFilter<?>)filter, filterValues.getValues());
return processor.resolveFilterValues((AbstractSelectFilter<?>) filter, filterValues.getValues());
}

@POST
@Path("autocomplete")
public List<FEValue> autocompleteTextFilter(@NotNull StringContainer text, @Context HttpServletRequest req, @QueryParam("page") OptionalInt pageNumberOpt, @QueryParam("pageSize") OptionalInt itemsPerPageOpt) {
if(StringUtils.isEmpty(text.getText())) {
throw new WebApplicationException("Too short text. Requires at least 1 characters.", Status.BAD_REQUEST);
}
if(!(filter instanceof AbstractSelectFilter)) {
throw new WebApplicationException(filter.getId()+" is not a SELECT filter, but "+filter.getClass().getSimpleName()+".", Status.BAD_REQUEST);
public List<FEValue> autocompleteTextFilter(@Valid StringContainer text, @Context HttpServletRequest req, @QueryParam("page") OptionalInt pageNumberOpt, @QueryParam("pageSize") OptionalInt itemsPerPageOpt) {

if (!(filter instanceof AbstractSelectFilter)) {
throw new WebApplicationException(filter.getId() + " is not a SELECT filter, but " + filter.getClass().getSimpleName() + ".", Status.BAD_REQUEST);
}


return processor.autocompleteTextFilter((AbstractSelectFilter<?>) filter, text.getText(), pageNumberOpt, itemsPerPageOpt);

return processor.autocompleteTextFilter((AbstractSelectFilter<?>) filter, Objects.requireNonNullElse(text.getText(), ""), pageNumberOpt, itemsPerPageOpt);
}

@Data
public static class FilterValues {
private List<String> values;
}

@Data
public static class StringContainer {
@Nullable
private String text;
}
}
8 changes: 8 additions & 0 deletions backend/src/main/java/com/bakdata/conquery/util/search/QuickSearch.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import com.bakdata.conquery.util.search.model.QuickSearchStats;
import com.bakdata.conquery.util.search.model.Result;
import com.bakdata.conquery.util.search.model.ResultItem;
import com.google.common.base.Preconditions;
import com.zigurs.karlis.utils.sort.MagicSort;


Expand Down Expand Up @@ -316,6 +317,13 @@ public List<T> findItems(final String searchString, final int numberOfTopItems)
return findItems(searchString, numberOfTopItems, keywordMatchScorer);
}

public List<T> listItems(int offset, int limit) {
Preconditions.checkArgument(offset > 0, "Offset must be positive value.");
Preconditions.checkArgument(limit > 0, "Limit must be positive value.");

return graph.listItems(offset, limit);
}

/**
* Retrieve (find) top n items matching the supplied com.bakdata.conquery.util.search string.
*
Expand Down
Loading