From cfaf0d6f7e489bbb6b109b6d14c22ec8e95a9fd5 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Fri, 13 Aug 2021 15:06:43 +0200 Subject: [PATCH 1/8] Adds option to list all elements instead of doing a proper search --- .../resources/api/ConceptsProcessor.java | 92 ++- .../resources/api/FilterResource.java | 32 +- .../conquery/util/search/QuickSearch.java | 8 + .../conquery/util/search/graph/QSGraph.java | 697 +++++++++--------- 4 files changed, 438 insertions(+), 391 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java index 5c55301c60..104b1face1 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java @@ -1,6 +1,5 @@ package com.bakdata.conquery.resources.api; -import java.time.Duration; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -18,10 +17,10 @@ import com.bakdata.conquery.apiv1.FilterSearch; import com.bakdata.conquery.apiv1.FilterSearchItem; import com.bakdata.conquery.apiv1.IdLabel; -import com.bakdata.conquery.io.storage.NamespaceStorage; import com.bakdata.conquery.apiv1.frontend.FEList; import com.bakdata.conquery.apiv1.frontend.FERoot; import com.bakdata.conquery.apiv1.frontend.FEValue; +import com.bakdata.conquery.io.storage.NamespaceStorage; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.auth.permissions.Ability; import com.bakdata.conquery.models.datasets.Dataset; @@ -39,12 +38,13 @@ import com.bakdata.conquery.util.CalculatedValue; import com.bakdata.conquery.util.search.QuickSearch; import com.bakdata.conquery.util.search.SearchScorer; +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import com.google.common.collect.ImmutableList; import lombok.AllArgsConstructor; -import lombok.Data; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.Setter; @@ -56,12 +56,13 @@ @Slf4j @RequiredArgsConstructor public class ConceptsProcessor { - + private final DatasetRegistry namespaces; private final LoadingCache, FEList> nodeCache = CacheBuilder.newBuilder() .softValues() + .weakKeys() .expireAfterWrite(10, TimeUnit.MINUTES) .build(new CacheLoader, FEList>() { @Override @@ -72,7 +73,8 @@ public FEList load(Concept concept) throws Exception { private final LoadingCache, String>, List> searchCache = CacheBuilder.newBuilder() - .expireAfterAccess(Duration.ofMinutes(2)) + .softValues() + .weakKeys() .build(new CacheLoader<>() { @Override @@ -84,18 +86,18 @@ public List load(Pair, String> filterAndSearch) } }); - + public FERoot getRoot(NamespaceStorage storage, User user) { return FrontEndConceptBuilder.createRoot(storage, user); } - + public FEList getNode(Concept concept) { try { return nodeCache.get(concept); } catch (ExecutionException e) { - throw new RuntimeException("failed to create frontend node for "+concept, e); + throw new RuntimeException("failed to create frontend node for " + concept, e); } } @@ -117,24 +119,24 @@ public ResolvedConceptsResult resolveFilterValues(AbstractSelectFilter filter //search in the full text engine Set searchResult = createSourceSearchResult(filter.getSourceSearch(), searchTerms, OptionalInt.empty(), filter.getSearchType()::score) - .stream() - .map(FEValue::getValue) - .collect(Collectors.toSet()); + .stream() + .map(FEValue::getValue) + .collect(Collectors.toSet()); Set openSearchTerms = new HashSet<>(searchTerms); openSearchTerms.removeAll(searchResult); // Iterate over all unresolved search terms. Gather all that match labels into searchResults. Keep the unresolvable ones. - for (Iterator it = openSearchTerms.iterator(); it.hasNext();) { + for (Iterator it = openSearchTerms.iterator(); it.hasNext(); ) { String searchTerm = it.next(); // Test if any of the values occurs directly in the filter's values or their labels (for when we don't have a provided file). - if(filter.getValues().contains(searchTerm)) { + if (filter.getValues().contains(searchTerm)) { searchResult.add(searchTerm); it.remove(); } else { String matchingValue = filter.getLabels().inverse().get(searchTerm); - if(matchingValue != null) { + if (matchingValue != null) { searchResult.add(matchingValue); it.remove(); } @@ -158,27 +160,36 @@ public ResolvedConceptsResult resolveFilterValues(AbstractSelectFilter filter public List autocompleteTextFilter(AbstractSelectFilter filter, String text, OptionalInt pageNumberOpt, OptionalInt itemsPerPageOpt) { int pageNumber = pageNumberOpt.orElse(0); int itemsPerPage = itemsPerPageOpt.orElse(50); - - if(pageNumber < 0) { - throw new IllegalArgumentException("Page number must be 0 or a positive integer"); - } - if(itemsPerPage < 1) { - throw new IllegalArgumentException("Items per page number must be larger than 0"); + + Preconditions.checkArgument(pageNumber > 0, "Page number must be 0 or a positive integer."); + Preconditions.checkArgument(itemsPerPage > 1, "Must at least have one item per page."); + + log.trace("Searching for for the term \"{}\". (Page = {}, Items = {})", text, pageNumber, itemsPerPage); + + if (Strings.isNullOrEmpty(text)) { + // If no text provided, we just list them + return filter.getSourceSearch().listItems(pageNumber * itemsPerPage, itemsPerPage) + .stream() + .map(item -> new FEValue(item.getLabel(), item.getValue(), item.getTemplateValues(), item.getOptionValue())) + .collect(Collectors.toList()); } - log.trace("Try to generate serach result page {} (with {} results per page) for the term \"{}\".", pageNumber, itemsPerPage, text); - + List fullResult = null; - try{ + try { fullResult = searchCache.get(Pair.of(filter, text)); - } catch (ExecutionException e) { - log.warn("Could not get a search result for the term \"{}\".", text, log.isTraceEnabled()? e : null); + } + catch (ExecutionException e) { + log.warn("Failed to search for \"{}\".", text, log.isTraceEnabled() ? e : null); return ImmutableList.of(); } - int startIncl = fullResult.isEmpty()? 0 : Math.min(itemsPerPage*pageNumber, fullResult.size()); - int endExcl = Math.min(startIncl + itemsPerPage,fullResult.size()); + + int startIncl = fullResult.isEmpty() ? 0 : Math.min(itemsPerPage * pageNumber, fullResult.size()); + int endExcl = Math.min(startIncl + itemsPerPage, fullResult.size()); + log.trace("Preparing subresult for search term \"{}\" in the index range [{}-{})", text, startIncl, endExcl); return fullResult.subList(startIncl, endExcl); } + /** * Autocompletion for search terms. For values of {@link AbstractSelectFilter}. * Is used by the serach cache to load missing items @@ -188,11 +199,12 @@ private static List autocompleteTextFilter(AbstractSelectFilter filt QuickSearch search = filter.getSourceSearch(); if (search != null) { - result = createSourceSearchResult(filter.getSourceSearch(), Collections.singletonList(text), OptionalInt.empty(), FilterSearch.FilterSearchType.CONTAINS::score); + result = + createSourceSearchResult(filter.getSourceSearch(), Collections.singletonList(text), OptionalInt.empty(), FilterSearch.FilterSearchType.CONTAINS::score); } - + String value = filter.getValueFor(text); - if(value != null) { + if (value != null) { result.add(new FEValue(text, value)); } @@ -203,24 +215,24 @@ private static List autocompleteTextFilter(AbstractSelectFilter filt * Do a search with the supplied values. */ private static List createSourceSearchResult(QuickSearch search, Collection values, OptionalInt numberOfTopItems, SearchScorer scorer) { - if(search == null) { + if (search == null) { return new ArrayList<>(); } // Quicksearch can split and also schedule for us. List result; result = search.findItems(String.join(" ", values), numberOfTopItems.orElse(Integer.MAX_VALUE), scorer); - - if(numberOfTopItems.isEmpty() && result.size() == Integer.MAX_VALUE) { + + if (numberOfTopItems.isEmpty() && result.size() == Integer.MAX_VALUE) { log.warn("The quick search returned the maximum number of results ({}) which probably means not all possible results are returned.", Integer.MAX_VALUE); } - + return result - .stream() - .map(item -> new FEValue(item.getLabel(), item.getValue(), item.getTemplateValues(), item.getOptionValue())) - .collect(Collectors.toList()); + .stream() + .map(item -> new FEValue(item.getLabel(), item.getValue(), item.getTemplateValues(), item.getOptionValue())) + .collect(Collectors.toList()); } - + public ResolvedConceptsResult resolveConceptElements(TreeConcept concept, List conceptCodes) { List> resolvedCodes = new ArrayList<>(); List unknownCodes = new ArrayList<>(); @@ -241,12 +253,12 @@ public ResolvedConceptsResult resolveConceptElements(TreeConcept concept, List)filter, filterValues.getValues()); + return processor.resolveFilterValues((AbstractSelectFilter) filter, filterValues.getValues()); } - + @POST @Path("autocomplete") - public List autocompleteTextFilter(@NotNull StringContainer text, @Context HttpServletRequest req, @QueryParam("page") OptionalInt pageNumberOpt, @QueryParam("pageSize") OptionalInt itemsPerPageOpt) { - if(StringUtils.isEmpty(text.getText())) { - throw new WebApplicationException("Too short text. Requires at least 1 characters.", Status.BAD_REQUEST); - } - if(!(filter instanceof AbstractSelectFilter)) { - throw new WebApplicationException(filter.getId()+" is not a SELECT filter, but "+filter.getClass().getSimpleName()+".", Status.BAD_REQUEST); + public List autocompleteTextFilter(@Valid StringContainer text, @Context HttpServletRequest req, @QueryParam("page") OptionalInt pageNumberOpt, @QueryParam("pageSize") OptionalInt itemsPerPageOpt) { + + if (!(filter instanceof AbstractSelectFilter)) { + throw new WebApplicationException(filter.getId() + " is not a SELECT filter, but " + filter.getClass().getSimpleName() + ".", Status.BAD_REQUEST); } - - return processor.autocompleteTextFilter((AbstractSelectFilter) filter, text.getText(), pageNumberOpt, itemsPerPageOpt); + + return processor.autocompleteTextFilter((AbstractSelectFilter) filter, Objects.requireNonNullElse(text.getText(), ""), pageNumberOpt, itemsPerPageOpt); } - + @Data public static class FilterValues { private List values; } - + @Data public static class StringContainer { + @Nullable private String text; } } diff --git a/backend/src/main/java/com/bakdata/conquery/util/search/QuickSearch.java b/backend/src/main/java/com/bakdata/conquery/util/search/QuickSearch.java index 28a3578427..2be513ed65 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/search/QuickSearch.java +++ b/backend/src/main/java/com/bakdata/conquery/util/search/QuickSearch.java @@ -34,6 +34,7 @@ import com.bakdata.conquery.util.search.model.QuickSearchStats; import com.bakdata.conquery.util.search.model.Result; import com.bakdata.conquery.util.search.model.ResultItem; +import com.google.common.base.Preconditions; import com.zigurs.karlis.utils.sort.MagicSort; @@ -316,6 +317,13 @@ public List findItems(final String searchString, final int numberOfTopItems) return findItems(searchString, numberOfTopItems, keywordMatchScorer); } + public List listItems(int offset, int limit) { + Preconditions.checkArgument(offset > 0, "Offset must be positive value."); + Preconditions.checkArgument(limit > 0, "Limit must be positive value."); + + return graph.listItems(offset, limit); + } + /** * Retrieve (find) top n items matching the supplied com.bakdata.conquery.util.search string. * diff --git a/backend/src/main/java/com/bakdata/conquery/util/search/graph/QSGraph.java b/backend/src/main/java/com/bakdata/conquery/util/search/graph/QSGraph.java index 2583ea6b3c..274933b858 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/search/graph/QSGraph.java +++ b/backend/src/main/java/com/bakdata/conquery/util/search/graph/QSGraph.java @@ -17,13 +17,16 @@ */ package com.bakdata.conquery.util.search.graph; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.concurrent.locks.StampedLock; +import java.util.stream.Collectors; import com.bakdata.conquery.util.search.ImmutableSet; import com.bakdata.conquery.util.search.SearchScorer; @@ -38,197 +41,208 @@ */ public class QSGraph> { - /** - * Map providing quick entry point to a particular node in the graph. - */ - private final Map> fragmentsNodesMap = new HashMap<>(); - - /** - * Mapping between an item and the keywords associated with it. Both a helper - * and a requirement to unmap nodes upon item removal. - */ - private final Map> itemKeywordsMap = new HashMap<>(); - - /** - * Stamped lock governing access to the graph modifying functions. - */ - private final StampedLock stampedLock = new StampedLock(); - - /* - * Public interface - */ - - /** - * Add an item to the graph and construct graph nodes for the specified keywords. - *

- * If the corresponding nodes already exist, the item will simply be added as a leaf. - * - * @param item item to add - * @param suppliedKeywords keywords to construct graph for - */ - public void registerItem(final T item, - final Set suppliedKeywords) { - long writeLock = stampedLock.writeLock(); - try { - suppliedKeywords.forEach(keyword -> createAndRegisterNode(null, keyword, item)); - - if (itemKeywordsMap.containsKey(item)) - itemKeywordsMap.put(item, ImmutableSet.fromCollections(itemKeywordsMap.get(item), suppliedKeywords)); - else - itemKeywordsMap.put(item, ImmutableSet.fromCollection(suppliedKeywords)); - } finally { - stampedLock.unlockWrite(writeLock); - } - } - - /** - * Remove an item from the map and remove any node mappings that become empty - * upon the items removal (determined using stored associated keywords of the item). - * - * @param item item to remove - */ - public void unregisterItem(final T item) { - long writeLock = stampedLock.writeLock(); - try { - if (itemKeywordsMap.containsKey(item)) { - for (String keyword : itemKeywordsMap.get(item)) { - GraphNode keywordNode = fragmentsNodesMap.get(keyword); - - keywordNode.removeItem(item); - - if (keywordNode.getItems().isEmpty()) - removeEdge(keywordNode, null); - } - } - itemKeywordsMap.remove(item); - } finally { - stampedLock.unlockWrite(writeLock); - } - } - - /** - * Walk the graph accumulating encountered items in the map with the highest score - * (according to the supplied scoring {@code BiFunction}) - * encountered at any visit to an item. - * - * @param fragment keyword or keyword fragment to start walk from - * @param scorerFunction function that will be called with the supplied fragment and node identity to score match - * - * @return map of accumulated items with their highest score encountered during walk (may be empty) - */ - public Map walkAndScore(final String fragment, - final SearchScorer scorerFunction) { - long readLock = stampedLock.readLock(); - try { - return walkAndScoreImpl(fragment, scorerFunction); - } finally { - stampedLock.unlockRead(readLock); - } - } - - /** - * Retrieve the stored keywords set associated with the item - * (or empty set if the item mapping is not recognized). - * - * @param item previously registered item - * - * @return set of associated keywords, possibly empty - */ - public Set getItemKeywords(final T item) { - /* safe to skip locking */ - ImmutableSet keywords = itemKeywordsMap.get(item); - - if (keywords == null) - return Collections.emptySet(); - - return keywords; - } - - /** - * Clear this graph. - */ - public void clear() { - long writeLock = stampedLock.writeLock(); - try { - fragmentsNodesMap.clear(); - itemKeywordsMap.clear(); - } finally { - stampedLock.unlockWrite(writeLock); - } - } - - /** - * Retrieve some basic statistics about the size of this graph. - * - * @return stats object containing sizes of internal collections - */ - public QuickSearchStats getStats() { - /* safe to ignore locking */ - return new QuickSearchStats( - itemKeywordsMap.size(), - fragmentsNodesMap.size() - ); - } - - /* - * Implementation code - */ - - private void createAndRegisterNode(final GraphNode parent, - final String identity, - final T item) { - GraphNode node = fragmentsNodesMap.get(identity); - - if (node == null) { - final String internedIdentity = identity.intern(); - - node = new GraphNode<>(internedIdentity); - fragmentsNodesMap.put(internedIdentity, node); - - // And proceed to add child nodes - if (node.getIdentity().length() > 1) { - createAndRegisterNode(node, internedIdentity.substring(0, identity.length() - 1), null); - createAndRegisterNode(node, internedIdentity.substring(1), null); - } - } - - if (item != null) - node.addItem(item); - - if (parent != null) - node.addParent(parent); - } - - private void removeEdge(final GraphNode node, - final GraphNode parent) { - if (node == null) //already removed - return; - - if (parent != null) - node.removeParent(parent); - - // No getParents or getItems means that there's nothing here to find, proceed onwards - if (node.getParents().isEmpty() && node.getItems().isEmpty()) { - fragmentsNodesMap.remove(node.getIdentity()); - - if (node.getIdentity().length() > 1) { - removeEdge(fragmentsNodesMap.get(node.getIdentity().substring(0, node.getIdentity().length() - 1)), node); - removeEdge(fragmentsNodesMap.get(node.getIdentity().substring(1)), node); - } - } - } - - /* - * Graph walking - */ - - private Map walkAndScoreImpl(final String fragment, - final SearchScorer scorerFunction) { - GraphNode root = fragmentsNodesMap.get(fragment); - - if (root == null) { - return Collections.emptyMap(); - } + /** + * Map providing quick entry point to a particular node in the graph. + */ + private final Map> fragmentsNodesMap = new HashMap<>(); + + /** + * Mapping between an item and the keywords associated with it. Both a helper + * and a requirement to unmap nodes upon item removal. + */ + private final Map> itemKeywordsMap = new HashMap<>(); + + /** + * Stamped lock governing access to the graph modifying functions. + */ + private final StampedLock stampedLock = new StampedLock(); + + /* + * Public interface + */ + + /** + * Add an item to the graph and construct graph nodes for the specified keywords. + *

+ * If the corresponding nodes already exist, the item will simply be added as a leaf. + * + * @param item item to add + * @param suppliedKeywords keywords to construct graph for + */ + public void registerItem(final T item, + final Set suppliedKeywords) { + long writeLock = stampedLock.writeLock(); + try { + suppliedKeywords.forEach(keyword -> createAndRegisterNode(null, keyword, item)); + + if (itemKeywordsMap.containsKey(item)) { + itemKeywordsMap.put(item, ImmutableSet.fromCollections(itemKeywordsMap.get(item), suppliedKeywords)); + } + else { + itemKeywordsMap.put(item, ImmutableSet.fromCollection(suppliedKeywords)); + } + } + finally { + stampedLock.unlockWrite(writeLock); + } + } + + /** + * Remove an item from the map and remove any node mappings that become empty + * upon the items removal (determined using stored associated keywords of the item). + * + * @param item item to remove + */ + public void unregisterItem(final T item) { + long writeLock = stampedLock.writeLock(); + try { + if (itemKeywordsMap.containsKey(item)) { + for (String keyword : itemKeywordsMap.get(item)) { + GraphNode keywordNode = fragmentsNodesMap.get(keyword); + + keywordNode.removeItem(item); + + if (keywordNode.getItems().isEmpty()) { + removeEdge(keywordNode, null); + } + } + } + itemKeywordsMap.remove(item); + } + finally { + stampedLock.unlockWrite(writeLock); + } + } + + /** + * Walk the graph accumulating encountered items in the map with the highest score + * (according to the supplied scoring {@code BiFunction}) + * encountered at any visit to an item. + * + * @param fragment keyword or keyword fragment to start walk from + * @param scorerFunction function that will be called with the supplied fragment and node identity to score match + * @return map of accumulated items with their highest score encountered during walk (may be empty) + */ + public Map walkAndScore(final String fragment, + final SearchScorer scorerFunction) { + long readLock = stampedLock.readLock(); + try { + return walkAndScoreImpl(fragment, scorerFunction); + } + finally { + stampedLock.unlockRead(readLock); + } + } + + /** + * Retrieve the stored keywords set associated with the item + * (or empty set if the item mapping is not recognized). + * + * @param item previously registered item + * @return set of associated keywords, possibly empty + */ + public Set getItemKeywords(final T item) { + /* safe to skip locking */ + ImmutableSet keywords = itemKeywordsMap.get(item); + + if (keywords == null) { + return Collections.emptySet(); + } + + return keywords; + } + + /** + * Clear this graph. + */ + public void clear() { + long writeLock = stampedLock.writeLock(); + try { + fragmentsNodesMap.clear(); + itemKeywordsMap.clear(); + } + finally { + stampedLock.unlockWrite(writeLock); + } + } + + /** + * Retrieve some basic statistics about the size of this graph. + * + * @return stats object containing sizes of internal collections + */ + public QuickSearchStats getStats() { + /* safe to ignore locking */ + return new QuickSearchStats( + itemKeywordsMap.size(), + fragmentsNodesMap.size() + ); + } + + /* + * Implementation code + */ + + private void createAndRegisterNode(final GraphNode parent, + final String identity, + final T item) { + GraphNode node = fragmentsNodesMap.get(identity); + + if (node == null) { + final String internedIdentity = identity.intern(); + + node = new GraphNode<>(internedIdentity); + fragmentsNodesMap.put(internedIdentity, node); + + // And proceed to add child nodes + if (node.getIdentity().length() > 1) { + createAndRegisterNode(node, internedIdentity.substring(0, identity.length() - 1), null); + createAndRegisterNode(node, internedIdentity.substring(1), null); + } + } + + if (item != null) { + node.addItem(item); + } + + if (parent != null) { + node.addParent(parent); + } + } + + private void removeEdge(final GraphNode node, + final GraphNode parent) { + if (node == null) //already removed + { + return; + } + + if (parent != null) { + node.removeParent(parent); + } + + // No getParents or getItems means that there's nothing here to find, proceed onwards + if (node.getParents().isEmpty() && node.getItems().isEmpty()) { + fragmentsNodesMap.remove(node.getIdentity()); + + if (node.getIdentity().length() > 1) { + removeEdge(fragmentsNodesMap.get(node.getIdentity().substring(0, node.getIdentity().length() - 1)), node); + removeEdge(fragmentsNodesMap.get(node.getIdentity().substring(1)), node); + } + } + } + + /* + * Graph walking + */ + + private Map walkAndScoreImpl(final String fragment, + final SearchScorer scorerFunction) { + GraphNode root = fragmentsNodesMap.get(fragment); + + if (root == null) { + return Collections.emptyMap(); + } int estResults = root.getEstimatedResultsCount() > -1 ? root.getEstimatedResultsCount() : 1024; HashMap results = new HashMap<>(estResults); @@ -242,148 +256,161 @@ private Map walkAndScoreImpl(final String fragment, root.setEstimatedResultsCount(results.size() * 2); return results; - } - - private void walkAndScore(final String originalFragment, - final GraphNode node, - final Map accumulated, - final Set visited, - final SearchScorer keywordMatchScorer) { - visited.add(node.getIdentity()); - - if (!node.getItems().isEmpty()) { - Double score = keywordMatchScorer.score(originalFragment, node.getIdentity()); - if (score > 0.0) - node.getItems().forEach(item -> accumulated.merge(item, score, (d1, d2) -> d1.compareTo(d2) > 0 ? d1 : d2)); - } - - node.getParents().forEach(parent -> { - if (!visited.contains(parent.getIdentity())) { - walkAndScore(originalFragment, parent, accumulated, visited, keywordMatchScorer); - } - }); - } - - /* - * Graph node that may contain a set of links to parent nodes - * and a set of concrete items associated with this node. - * - * The underlying idea is to have a hierarchical graph (ok, multi-root tree) - * where arbitrary nodes can have items associated with them. Each particular node - * serves as an entry point to traverse the graph upwards of it and - * operate on associated items. - */ - private static final class GraphNode> implements Comparable> { - - private final String identity; - private Set items; - private Set> parents; - - /* - * Track historical results set sizes to avoid - * excessive re-re-hashing on large result-sets - */ - - private int estimatedNodesCount = -1; - private int estimatedResultsCount = -1; - - /** - * Create a node with immutable identity string. - * - * @param fragment any string you like - */ - private GraphNode(final String fragment) { - Objects.requireNonNull(fragment); - this.identity = fragment; - this.items = new HashSet<>(); - this.parents = new HashSet<>(); - } - - /** - * Retrieve identifier. - * - * @return selected identifier - */ - private String getIdentity() { - return identity; - } - - /** - * Retrieve set containing node items. The set will likely be read only - * and you _must_ use add and remove methods to add and remove items. - * - * @return Immutable, possibly empty, set of associated items. - */ - private Set getItems() { - return items; - } - - /** - * Register an item with this node. - * - * @param item item to add. - */ - private void addItem(final V item) { - items.add(item); - } - - /** - * Remove an item from this node if it is present. - * - * @param item item to remove. - */ - private void removeItem(final V item) { - items.add(item); - } - - /** - * Retrieve set containing known node parents. The set will likely - * be read only and you _must_ use add and remove methods to ... add - * and remove parents. - * - * @return Immutable, possibly empty, set of known parent nodes. - */ - private Set> getParents() { - return parents; - } - - /** - * Add a parent node if not already known. - * - * @param parent parent to add. - */ - private void addParent(final GraphNode parent) { - parents.add(parent); - } - - /** - * Remove a parent node if known. - * - * @param parent parent to remove. - */ - private void removeParent(final GraphNode parent) { - parents.add(parent); - } - - @Override - public int compareTo(GraphNode o) { - return identity.compareTo(o.identity); - } - - private int getEstimatedNodesCount() { - return estimatedNodesCount; - } - - private void setEstimatedNodesCount(int estimatedNodesCount) { - this.estimatedNodesCount = estimatedNodesCount; - } - - private int getEstimatedResultsCount() { - return estimatedResultsCount; - } - - private void setEstimatedResultsCount(int estimatedResultsCount) { - this.estimatedResultsCount = estimatedResultsCount; - } - } + } + + private void walkAndScore(final String originalFragment, + final GraphNode node, + final Map accumulated, + final Set visited, + final SearchScorer keywordMatchScorer) { + visited.add(node.getIdentity()); + + if (!node.getItems().isEmpty()) { + Double score = keywordMatchScorer.score(originalFragment, node.getIdentity()); + if (score > 0.0) { + node.getItems().forEach(item -> accumulated.merge(item, score, (d1, d2) -> d1.compareTo(d2) > 0 ? d1 : d2)); + } + } + + node.getParents().forEach(parent -> { + if (!visited.contains(parent.getIdentity())) { + walkAndScore(originalFragment, parent, accumulated, visited, keywordMatchScorer); + } + }); + } + + public List listItems(int offset, int limit) { + return fragmentsNodesMap.entrySet() + .stream() + .sorted(Map.Entry.comparingByKey()) + .map(entry -> entry.getValue().items) + .flatMap(Collection::stream) + .distinct() + .skip(offset) + .limit(limit) + .collect(Collectors.toList()); + } + + /* + * Graph node that may contain a set of links to parent nodes + * and a set of concrete items associated with this node. + * + * The underlying idea is to have a hierarchical graph (ok, multi-root tree) + * where arbitrary nodes can have items associated with them. Each particular node + * serves as an entry point to traverse the graph upwards of it and + * operate on associated items. + */ + private static final class GraphNode> implements Comparable> { + + private final String identity; + private Set items; + private Set> parents; + + /* + * Track historical results set sizes to avoid + * excessive re-re-hashing on large result-sets + */ + + private int estimatedNodesCount = -1; + private int estimatedResultsCount = -1; + + /** + * Create a node with immutable identity string. + * + * @param fragment any string you like + */ + private GraphNode(final String fragment) { + Objects.requireNonNull(fragment); + this.identity = fragment; + this.items = new HashSet<>(); + this.parents = new HashSet<>(); + } + + /** + * Retrieve identifier. + * + * @return selected identifier + */ + private String getIdentity() { + return identity; + } + + /** + * Retrieve set containing node items. The set will likely be read only + * and you _must_ use add and remove methods to add and remove items. + * + * @return Immutable, possibly empty, set of associated items. + */ + private Set getItems() { + return items; + } + + /** + * Register an item with this node. + * + * @param item item to add. + */ + private void addItem(final V item) { + items.add(item); + } + + /** + * Remove an item from this node if it is present. + * + * @param item item to remove. + */ + private void removeItem(final V item) { + items.add(item); + } + + /** + * Retrieve set containing known node parents. The set will likely + * be read only and you _must_ use add and remove methods to ... add + * and remove parents. + * + * @return Immutable, possibly empty, set of known parent nodes. + */ + private Set> getParents() { + return parents; + } + + /** + * Add a parent node if not already known. + * + * @param parent parent to add. + */ + private void addParent(final GraphNode parent) { + parents.add(parent); + } + + /** + * Remove a parent node if known. + * + * @param parent parent to remove. + */ + private void removeParent(final GraphNode parent) { + parents.add(parent); + } + + @Override + public int compareTo(GraphNode o) { + return identity.compareTo(o.identity); + } + + private int getEstimatedNodesCount() { + return estimatedNodesCount; + } + + private void setEstimatedNodesCount(int estimatedNodesCount) { + this.estimatedNodesCount = estimatedNodesCount; + } + + private int getEstimatedResultsCount() { + return estimatedResultsCount; + } + + private void setEstimatedResultsCount(int estimatedResultsCount) { + this.estimatedResultsCount = estimatedResultsCount; + } + } } From 5a4dd879cf4bd749c3505084c837158adb6dc49e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 17 Aug 2021 09:51:04 +0000 Subject: [PATCH 2/8] Update AutoDoc --- docs/REST API JSONs.md | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/REST API JSONs.md b/docs/REST API JSONs.md index be935d0eed..e4b6fa454f 100644 --- a/docs/REST API JSONs.md +++ b/docs/REST API JSONs.md @@ -73,7 +73,7 @@ Returns: [ResolvedConceptsResult](#Type-ResolvedConceptsResult)

-### POST datasets/{dataset}/concepts/{concept}/tables/{table}/filters/{filter}/autocomplete [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/FilterResource.java#L44) +### POST datasets/{dataset}/concepts/{concept}/tables/{table}/filters/{filter}/autocomplete [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/FilterResource.java#L45)
Details

@@ -89,7 +89,7 @@ Returns: list of [FEValue](#Type-FEValue)

-### POST datasets/{dataset}/concepts/{concept}/tables/{table}/filters/{filter}/resolve [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/FilterResource.java#L38) +### POST datasets/{dataset}/concepts/{concept}/tables/{table}/filters/{filter}/resolve [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/FilterResource.java#L39)
Details

@@ -815,7 +815,7 @@ Supported Fields: | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FEValue.java#L16) | templateValues | map from `String` to `String` | ? | | |

-### Type FilterValues [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/FilterResource.java#L58) +### Type FilterValues [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/FilterResource.java#L57)
Details

@@ -826,7 +826,7 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/FilterResource.java#L60) | values | list of `String` | `null` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/FilterResource.java#L59) | values | list of `String` | `null` | | |

### Type FormConfig [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/models/forms/configs/FormConfig.java#L49) @@ -980,7 +980,7 @@ Supported Fields: | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/ExecutionStatus.java#L24) | tags | list of `String` | `null` | | |

-### Type ResolvedConceptsResult [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L260) +### Type ResolvedConceptsResult [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L272)
Details

@@ -991,12 +991,12 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L265) | resolvedConcepts | list of ID of `ConceptElement` | ? | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L266) | resolvedFilter | [ResolvedFilterResult](#Type-ResolvedFilterResult) | ? | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L267) | unknownCodes | list of `String` | ? | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L277) | resolvedConcepts | list of ID of `ConceptElement` | ? | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L278) | resolvedFilter | [ResolvedFilterResult](#Type-ResolvedFilterResult) | ? | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L279) | unknownCodes | list of `String` | ? | | |

-### Type ResolvedFilterResult [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L250) +### Type ResolvedFilterResult [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L262)
Details

@@ -1007,12 +1007,12 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L256) | filterId | ID of `Filter` | ? | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L255) | tableId | ID of `Connector` | ? | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L257) | value | list of [FEValue](#Type-FEValue) | ? | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L268) | filterId | ID of `Filter` | ? | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L267) | tableId | ID of `Connector` | ? | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L269) | value | list of [FEValue](#Type-FEValue) | ? | | |

-### Type StringContainer [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/FilterResource.java#L63) +### Type StringContainer [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/FilterResource.java#L62)
Details

@@ -1023,7 +1023,7 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/FilterResource.java#L65) | text | `String` | `null` | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/FilterResource.java#L64) | text | `String` | `null` | | |

### Type ValidityDateContainer [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/ValidityDateContainer.java#L9) From c97d65141902cd25bac0d5fab5be7f40232879d6 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Tue, 24 Aug 2021 16:09:01 +0200 Subject: [PATCH 3/8] push full list down into cache --- .../resources/api/ConceptsProcessor.java | 27 ++++++++++--------- .../conquery/util/search/QuickSearch.java | 8 ++---- .../conquery/util/search/graph/QSGraph.java | 4 +-- 3 files changed, 18 insertions(+), 21 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java index 104b1face1..3e6f9b3837 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java @@ -62,7 +62,6 @@ public class ConceptsProcessor { private final LoadingCache, FEList> nodeCache = CacheBuilder.newBuilder() .softValues() - .weakKeys() .expireAfterWrite(10, TimeUnit.MINUTES) .build(new CacheLoader, FEList>() { @Override @@ -74,7 +73,6 @@ public FEList load(Concept concept) throws Exception { private final LoadingCache, String>, List> searchCache = CacheBuilder.newBuilder() .softValues() - .weakKeys() .build(new CacheLoader<>() { @Override @@ -166,14 +164,6 @@ public List autocompleteTextFilter(AbstractSelectFilter filter, Stri log.trace("Searching for for the term \"{}\". (Page = {}, Items = {})", text, pageNumber, itemsPerPage); - if (Strings.isNullOrEmpty(text)) { - // If no text provided, we just list them - return filter.getSourceSearch().listItems(pageNumber * itemsPerPage, itemsPerPage) - .stream() - .map(item -> new FEValue(item.getLabel(), item.getValue(), item.getTemplateValues(), item.getOptionValue())) - .collect(Collectors.toList()); - } - List fullResult = null; try { fullResult = searchCache.get(Pair.of(filter, text)); @@ -195,12 +185,25 @@ public List autocompleteTextFilter(AbstractSelectFilter filter, Stri * Is used by the serach cache to load missing items */ private static List autocompleteTextFilter(AbstractSelectFilter filter, String text) { + if (Strings.isNullOrEmpty(text)) { + // If no text provided, we just list them + return filter.getSourceSearch().listItems() + .stream() + .map(item -> new FEValue(item.getLabel(), item.getValue(), item.getTemplateValues(), item.getOptionValue())) + .collect(Collectors.toList()); + } + List result = new LinkedList<>(); QuickSearch search = filter.getSourceSearch(); + if (search != null) { - result = - createSourceSearchResult(filter.getSourceSearch(), Collections.singletonList(text), OptionalInt.empty(), FilterSearch.FilterSearchType.CONTAINS::score); + result = createSourceSearchResult( + filter.getSourceSearch(), + Collections.singletonList(text), + OptionalInt.empty(), + FilterSearch.FilterSearchType.CONTAINS::score + ); } String value = filter.getValueFor(text); diff --git a/backend/src/main/java/com/bakdata/conquery/util/search/QuickSearch.java b/backend/src/main/java/com/bakdata/conquery/util/search/QuickSearch.java index 2be513ed65..3370263453 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/search/QuickSearch.java +++ b/backend/src/main/java/com/bakdata/conquery/util/search/QuickSearch.java @@ -34,7 +34,6 @@ import com.bakdata.conquery.util.search.model.QuickSearchStats; import com.bakdata.conquery.util.search.model.Result; import com.bakdata.conquery.util.search.model.ResultItem; -import com.google.common.base.Preconditions; import com.zigurs.karlis.utils.sort.MagicSort; @@ -317,11 +316,8 @@ public List findItems(final String searchString, final int numberOfTopItems) return findItems(searchString, numberOfTopItems, keywordMatchScorer); } - public List listItems(int offset, int limit) { - Preconditions.checkArgument(offset > 0, "Offset must be positive value."); - Preconditions.checkArgument(limit > 0, "Limit must be positive value."); - - return graph.listItems(offset, limit); + public List listItems() { + return graph.listItems(); } /** diff --git a/backend/src/main/java/com/bakdata/conquery/util/search/graph/QSGraph.java b/backend/src/main/java/com/bakdata/conquery/util/search/graph/QSGraph.java index 274933b858..ada7063f37 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/search/graph/QSGraph.java +++ b/backend/src/main/java/com/bakdata/conquery/util/search/graph/QSGraph.java @@ -279,15 +279,13 @@ private void walkAndScore(final String originalFragment, }); } - public List listItems(int offset, int limit) { + public List listItems() { return fragmentsNodesMap.entrySet() .stream() .sorted(Map.Entry.comparingByKey()) .map(entry -> entry.getValue().items) .flatMap(Collection::stream) .distinct() - .skip(offset) - .limit(limit) .collect(Collectors.toList()); } From c5d8e34ce5637b64e1c95bb2f7d8cd26adf48167 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Tue, 24 Aug 2021 16:10:07 +0200 Subject: [PATCH 4/8] cleanup --- .../com/bakdata/conquery/resources/api/ConceptsProcessor.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java index 3e6f9b3837..23ba7387d5 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java @@ -223,8 +223,7 @@ private static List createSourceSearchResult(QuickSearch result; - result = search.findItems(String.join(" ", values), numberOfTopItems.orElse(Integer.MAX_VALUE), scorer); + List result = search.findItems(String.join(" ", values), numberOfTopItems.orElse(Integer.MAX_VALUE), scorer); if (numberOfTopItems.isEmpty() && result.size() == Integer.MAX_VALUE) { log.warn("The quick search returned the maximum number of results ({}) which probably means not all possible results are returned.", Integer.MAX_VALUE); From 6dbbdb15a0887db10fe88733348cd2711ea8ee9f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 24 Aug 2021 14:11:54 +0000 Subject: [PATCH 5/8] Update AutoDoc --- docs/REST API JSONs.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/REST API JSONs.md b/docs/REST API JSONs.md index e4b6fa454f..063bf8a161 100644 --- a/docs/REST API JSONs.md +++ b/docs/REST API JSONs.md @@ -980,7 +980,7 @@ Supported Fields: | [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/ExecutionStatus.java#L24) | tags | list of `String` | `null` | | |

-### Type ResolvedConceptsResult [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L272) +### Type ResolvedConceptsResult [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L274)
Details

@@ -991,12 +991,12 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L277) | resolvedConcepts | list of ID of `ConceptElement` | ? | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L278) | resolvedFilter | [ResolvedFilterResult](#Type-ResolvedFilterResult) | ? | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L279) | unknownCodes | list of `String` | ? | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L279) | resolvedConcepts | list of ID of `ConceptElement` | ? | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L280) | resolvedFilter | [ResolvedFilterResult](#Type-ResolvedFilterResult) | ? | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L281) | unknownCodes | list of `String` | ? | | |

-### Type ResolvedFilterResult [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L262) +### Type ResolvedFilterResult [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L264)
Details

@@ -1007,9 +1007,9 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L268) | filterId | ID of `Filter` | ? | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L267) | tableId | ID of `Connector` | ? | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L269) | value | list of [FEValue](#Type-FEValue) | ? | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L270) | filterId | ID of `Filter` | ? | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L269) | tableId | ID of `Connector` | ? | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java#L271) | value | list of [FEValue](#Type-FEValue) | ? | | |

### Type StringContainer [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/FilterResource.java#L62) From 05ba4af313825b1440c3efbcf0a81726d2343c89 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Tue, 24 Aug 2021 16:19:09 +0200 Subject: [PATCH 6/8] adds test asserting the order of output values --- .../conquery/util/search/QuickSearchTest.java | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 backend/src/test/java/com/bakdata/conquery/util/search/QuickSearchTest.java diff --git a/backend/src/test/java/com/bakdata/conquery/util/search/QuickSearchTest.java b/backend/src/test/java/com/bakdata/conquery/util/search/QuickSearchTest.java new file mode 100644 index 0000000000..d74f4cb60c --- /dev/null +++ b/backend/src/test/java/com/bakdata/conquery/util/search/QuickSearchTest.java @@ -0,0 +1,24 @@ +package com.bakdata.conquery.util.search; + +import static org.assertj.core.api.Assertions.assertThat; + +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Test; + +@Slf4j +class QuickSearchTest { + @Test + public void listItems() { + final QuickSearch quickSearch = new QuickSearch<>(); + + quickSearch.addItem(0,"b"); + quickSearch.addItem(1,"c"); + quickSearch.addItem(2,"a"); + + assertThat(quickSearch.listItems()) + .containsExactly(2, 0, 1); + + + } + +} \ No newline at end of file From 51f5da97162fe72c4135460d9c58457738f5b2a5 Mon Sep 17 00:00:00 2001 From: Fabian Kovacs <1553491+awildturtok@users.noreply.github.com> Date: Tue, 24 Aug 2021 18:28:25 +0200 Subject: [PATCH 7/8] sort by Value and not Keyword --- .../conquery/apiv1/frontend/FEValue.java | 20 ++++++++++++++++--- .../conquery/util/search/graph/QSGraph.java | 6 +++--- .../conquery/util/search/QuickSearchTest.java | 8 ++++---- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FEValue.java b/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FEValue.java index 0e020eb140..c9e858b1d9 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FEValue.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FEValue.java @@ -1,23 +1,37 @@ package com.bakdata.conquery.apiv1.frontend; +import java.util.Comparator; import java.util.Map; import lombok.AllArgsConstructor; import lombok.Data; +import org.jetbrains.annotations.NotNull; /** * This class represents a values of a SELECT filter. */ -@Data @AllArgsConstructor -public class FEValue { - +@Data +@AllArgsConstructor +public class FEValue implements Comparable { + private static final Comparator COMPARATOR = Comparator.comparing(FEValue::getLabel).thenComparing(FEValue::getValue); + + @NotNull private final String label; + + @NotNull private final String value; + private Map templateValues; + private String optionValue; public FEValue(String label, String value) { this.label = label; this.value = value; } + + @Override + public int compareTo(@NotNull FEValue o) { + return COMPARATOR.compare(this, o); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/util/search/graph/QSGraph.java b/backend/src/main/java/com/bakdata/conquery/util/search/graph/QSGraph.java index ada7063f37..83d01608c0 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/search/graph/QSGraph.java +++ b/backend/src/main/java/com/bakdata/conquery/util/search/graph/QSGraph.java @@ -280,12 +280,12 @@ private void walkAndScore(final String originalFragment, } public List listItems() { - return fragmentsNodesMap.entrySet() + return fragmentsNodesMap.values() .stream() - .sorted(Map.Entry.comparingByKey()) - .map(entry -> entry.getValue().items) + .map(GraphNode::getItems) .flatMap(Collection::stream) .distinct() + .sorted() .collect(Collectors.toList()); } diff --git a/backend/src/test/java/com/bakdata/conquery/util/search/QuickSearchTest.java b/backend/src/test/java/com/bakdata/conquery/util/search/QuickSearchTest.java index d74f4cb60c..15a0d12213 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/search/QuickSearchTest.java +++ b/backend/src/test/java/com/bakdata/conquery/util/search/QuickSearchTest.java @@ -11,12 +11,12 @@ class QuickSearchTest { public void listItems() { final QuickSearch quickSearch = new QuickSearch<>(); - quickSearch.addItem(0,"b"); - quickSearch.addItem(1,"c"); - quickSearch.addItem(2,"a"); + quickSearch.addItem(0, "b"); + quickSearch.addItem(1, "c"); + quickSearch.addItem(2, "a"); assertThat(quickSearch.listItems()) - .containsExactly(2, 0, 1); + .containsExactly(0, 1, 2); } From 313acc89c1c137210491d6a5fa7935ff515fade7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 24 Aug 2021 16:30:04 +0000 Subject: [PATCH 8/8] Update AutoDoc --- docs/REST API JSONs.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/REST API JSONs.md b/docs/REST API JSONs.md index 063bf8a161..04283a1897 100644 --- a/docs/REST API JSONs.md +++ b/docs/REST API JSONs.md @@ -800,7 +800,7 @@ No fields can be set for this type.

-### Type FEValue [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FEValue.java#L8-L10) +### Type FEValue [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FEValue.java#L10-L12) This class represents a values of a SELECT filter.
Details

@@ -811,8 +811,8 @@ Supported Fields: | | Field | Type | Default | Example | Description | | --- | --- | --- | --- | --- | --- | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FEValue.java#L17) | optionValue | `String` | ? | | | -| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FEValue.java#L16) | templateValues | map from `String` to `String` | ? | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FEValue.java#L26) | optionValue | `String` | ? | | | +| [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FEValue.java#L24) | templateValues | map from `String` to `String` | ? | | |

### Type FilterValues [✎](https://github.com/bakdata/conquery/edit/develop/backend/src/main/java/com/bakdata/conquery/resources/api/FilterResource.java#L57)