From 41b28dcdd7c5867448aef63137bf41fb90cbcc39 Mon Sep 17 00:00:00 2001 From: Dexter Lee Date: Wed, 28 Jul 2021 23:23:40 -0700 Subject: [PATCH 1/9] Fix Browse --- ...com.linkedin.entity.entities.restspec.json | 14 +++ ...com.linkedin.entity.entities.snapshot.json | 14 +++ .../linkedin/entity/client/EntityClient.java | 16 ++++ .../resources/entity/EntityResource.java | 45 +++++----- .../metadata/search/SearchService.java | 7 ++ .../elasticsearch/ElasticSearchService.java | 5 ++ .../elasticsearch/query/ESBrowseDAO.java | 90 ++++++++++--------- .../elasticsearch/query/ESSearchDAO.java | 15 ++++ .../ElasticSearchServiceTest.java | 3 + 9 files changed, 146 insertions(+), 63 deletions(-) diff --git a/gms/api/src/main/idl/com.linkedin.entity.entities.restspec.json b/gms/api/src/main/idl/com.linkedin.entity.entities.restspec.json index 3779adfaf118c7..f5f38eed017b6d 100644 --- a/gms/api/src/main/idl/com.linkedin.entity.entities.restspec.json +++ b/gms/api/src/main/idl/com.linkedin.entity.entities.restspec.json @@ -47,6 +47,13 @@ "type" : "int" } ], "returns" : "com.linkedin.metadata.query.AutoCompleteResult" + }, { + "name" : "batchGetNumEntities", + "parameters" : [ { + "name" : "entities", + "type" : "{ \"type\" : \"array\", \"items\" : \"string\" }" + } ], + "returns" : "{ \"type\" : \"map\", \"values\" : \"long\" }" }, { "name" : "batchIngest", "parameters" : [ { @@ -80,6 +87,13 @@ "type" : "com.linkedin.common.Urn" } ], "returns" : "{ \"type\" : \"array\", \"items\" : \"string\" }" + }, { + "name" : "getNumEntities", + "parameters" : [ { + "name" : "entity", + "type" : "string" + } ], + "returns" : "long" }, { "name" : "ingest", "parameters" : [ { diff --git a/gms/api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json b/gms/api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json index 3e9a5ac509110e..4989f7ba7498ae 100644 --- a/gms/api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json +++ b/gms/api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json @@ -4340,6 +4340,13 @@ "type" : "int" } ], "returns" : "com.linkedin.metadata.query.AutoCompleteResult" + }, { + "name" : "batchGetNumEntities", + "parameters" : [ { + "name" : "entities", + "type" : "{ \"type\" : \"array\", \"items\" : \"string\" }" + } ], + "returns" : "{ \"type\" : \"map\", \"values\" : \"long\" }" }, { "name" : "batchIngest", "parameters" : [ { @@ -4373,6 +4380,13 @@ "type" : "com.linkedin.common.Urn" } ], "returns" : "{ \"type\" : \"array\", \"items\" : \"string\" }" + }, { + "name" : "getNumEntities", + "parameters" : [ { + "name" : "entity", + "type" : "string" + } ], + "returns" : "long" }, { "name" : "ingest", "parameters" : [ { diff --git a/gms/client/src/main/java/com/linkedin/entity/client/EntityClient.java b/gms/client/src/main/java/com/linkedin/entity/client/EntityClient.java index 519f74f819d04a..0e79607d8bd794 100644 --- a/gms/client/src/main/java/com/linkedin/entity/client/EntityClient.java +++ b/gms/client/src/main/java/com/linkedin/entity/client/EntityClient.java @@ -1,6 +1,8 @@ package com.linkedin.entity.client; import com.linkedin.common.urn.Urn; +import com.linkedin.entity.EntitiesDoBatchGetNumEntitiesRequestBuilder; +import com.linkedin.entity.EntitiesDoGetNumEntitiesRequestBuilder; import com.linkedin.entity.EntitiesDoSetWritableRequestBuilder; import com.linkedin.restli.client.Response; import org.slf4j.Logger; @@ -266,4 +268,18 @@ public void setWritable(boolean canWrite) throws RemoteInvocationException { ENTITIES_REQUEST_BUILDERS.actionSetWritable().valueParam(canWrite); sendClientRequest(requestBuilder.build()); } + + @Nonnull + public long getNumEntities(@Nonnull String entityName) throws RemoteInvocationException { + EntitiesDoGetNumEntitiesRequestBuilder requestBuilder = + ENTITIES_REQUEST_BUILDERS.actionGetNumEntities().entityParam(entityName); + return sendClientRequest(requestBuilder.build()).getEntity(); + } + + @Nonnull + public Map batchGetNumEntities(@Nonnull List entityName) throws RemoteInvocationException { + EntitiesDoBatchGetNumEntitiesRequestBuilder requestBuilder = + ENTITIES_REQUEST_BUILDERS.actionBatchGetNumEntities().entitiesParam(new StringArray(entityName)); + return sendClientRequest(requestBuilder.build()).getEntity(); + } } diff --git a/gms/impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java b/gms/impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java index 831eb317609679..50e8cc35538d9f 100644 --- a/gms/impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java +++ b/gms/impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java @@ -2,6 +2,7 @@ import com.linkedin.common.AuditStamp; import com.linkedin.common.urn.Urn; +import com.linkedin.data.template.LongMap; import com.linkedin.data.template.RecordTemplate; import com.linkedin.data.template.StringArray; import com.linkedin.entity.Entity; @@ -30,6 +31,7 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -101,8 +103,7 @@ public Task get(@Nonnull String urnStr, @QueryParam(PARAM_ASPECTS) @Opti @RestMethod.BatchGet @Nonnull - public Task> batchGet( - @Nonnull Set urnStrs, + public Task> batchGet(@Nonnull Set urnStrs, @QueryParam(PARAM_ASPECTS) @Optional @Nullable String[] aspectNames) throws URISyntaxException { log.info("BATCH GET {}", urnStrs.toString()); final Set urns = new HashSet<>(); @@ -153,12 +154,9 @@ public Task batchIngest(@ActionParam(PARAM_ENTITIES) @Nonnull Entity[] ent @Action(name = ACTION_SEARCH) @Nonnull - public Task search( - @ActionParam(PARAM_ENTITY) @Nonnull String entityName, - @ActionParam(PARAM_INPUT) @Nonnull String input, - @ActionParam(PARAM_FILTER) @Optional @Nullable Filter filter, - @ActionParam(PARAM_SORT) @Optional @Nullable SortCriterion sortCriterion, - @ActionParam(PARAM_START) int start, + public Task search(@ActionParam(PARAM_ENTITY) @Nonnull String entityName, + @ActionParam(PARAM_INPUT) @Nonnull String input, @ActionParam(PARAM_FILTER) @Optional @Nullable Filter filter, + @ActionParam(PARAM_SORT) @Optional @Nullable SortCriterion sortCriterion, @ActionParam(PARAM_START) int start, @ActionParam(PARAM_COUNT) int count) { log.info("GET SEARCH RESULTS for {} with query {}", entityName, input); @@ -167,24 +165,18 @@ public Task search( @Action(name = ACTION_AUTOCOMPLETE) @Nonnull - public Task autocomplete( - @ActionParam(PARAM_ENTITY) @Nonnull String entityName, - @ActionParam(PARAM_QUERY) @Nonnull String query, - @ActionParam(PARAM_FIELD) @Optional @Nullable String field, - @ActionParam(PARAM_FILTER) @Optional @Nullable Filter filter, - @ActionParam(PARAM_LIMIT) int limit) { + public Task autocomplete(@ActionParam(PARAM_ENTITY) @Nonnull String entityName, + @ActionParam(PARAM_QUERY) @Nonnull String query, @ActionParam(PARAM_FIELD) @Optional @Nullable String field, + @ActionParam(PARAM_FILTER) @Optional @Nullable Filter filter, @ActionParam(PARAM_LIMIT) int limit) { return RestliUtils.toTask(() -> _searchService.autoComplete(entityName, query, field, filter, limit)); } @Action(name = ACTION_BROWSE) @Nonnull - public Task browse( - @ActionParam(PARAM_ENTITY) @Nonnull String entityName, - @ActionParam(PARAM_PATH) @Nonnull String path, - @ActionParam(PARAM_FILTER) @Optional @Nullable Filter filter, - @ActionParam(PARAM_START) int start, - @ActionParam(PARAM_LIMIT) int limit) { + public Task browse(@ActionParam(PARAM_ENTITY) @Nonnull String entityName, + @ActionParam(PARAM_PATH) @Nonnull String path, @ActionParam(PARAM_FILTER) @Optional @Nullable Filter filter, + @ActionParam(PARAM_START) int start, @ActionParam(PARAM_LIMIT) int limit) { log.info("GET BROWSE RESULTS for {} at path {}", entityName, path); return RestliUtils.toTask(() -> _searchService.browse(entityName, path, filter, start, limit)); @@ -210,4 +202,17 @@ public Task setWriteable(@ActionParam(PARAM_VALUE) @Optional("true") @Nonn return null; }); } + + @Action(name = "getNumEntities") + @Nonnull + public Task getNumEntities(@ActionParam(PARAM_ENTITY) @Nonnull String entityName) { + return RestliUtils.toTask(() -> _searchService.docCount(entityName)); + } + + @Action(name = "batchGetNumEntities") + @Nonnull + public Task batchGetNumEntities(@ActionParam(PARAM_ENTITIES) @Nonnull String[] entityNames) { + return RestliUtils.toTask(() -> new LongMap( + Arrays.stream(entityNames).collect(Collectors.toMap(Function.identity(), _searchService::docCount)))); + } } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/SearchService.java b/metadata-io/src/main/java/com/linkedin/metadata/search/SearchService.java index 66dc64e8427f35..bb7ef91cfca1b5 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/SearchService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/SearchService.java @@ -20,6 +20,13 @@ public interface SearchService { */ void clear(); + /** + * Get the number of documents corresponding to the entity + * + * @param entityName name of the entity + */ + long docCount(@Nonnull String entityName); + /** * Updates or inserts the given search document. * diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java index 18a7c27d789cb0..32801ac5feb703 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java @@ -37,6 +37,11 @@ public void clear() { esWriteDAO.clear(); } + @Override + public long docCount(@Nonnull String entityName) { + return esSearchDAO.docCount(entityName); + } + @Override public void upsertDocument(@Nonnull String entityName, @Nonnull String document, @Nonnull String docId) { log.debug(String.format("Upserting Search document entityName: %s, document: %s, docId: %s", entityName, diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java index df3c89519eb59d..117d7069d1bb43 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java @@ -18,7 +18,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -37,7 +36,7 @@ import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregationBuilders; -import org.elasticsearch.search.aggregations.BucketOrder; +import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation; import org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude; import org.elasticsearch.search.aggregations.bucket.terms.ParsedTerms; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -57,6 +56,9 @@ public class ESBrowseDAO { private static final String URN = "urn"; private static final String REMOVED = "removed"; + private static final String GROUP_AGG = "groups"; + private static final String ALL_PATHS = "allPaths"; + /** * Gets a list of groups/entities that match given browse request. * @@ -74,14 +76,26 @@ public BrowseResult browse(@Nonnull String entityName, @Nonnull String path, @Nu try { final String indexName = indexConvention.getIndexName(entityRegistry.getEntitySpec(entityName)); + final SearchResponse groupsResponse = - client.search(constructGroupsSearchRequest(indexName, path, requestMap, 0, 1000), RequestOptions.DEFAULT); + client.search(constructGroupsSearchRequest(indexName, path, requestMap), RequestOptions.DEFAULT); + final BrowseResultMetadata browseResultMetadata = extractGroupsResponse(groupsResponse, path, from, size); + final int numGroups = browseResultMetadata.getTotalNumEntities().intValue(); + + // Based on the number of groups returned, compute the from and size to query for entities + int entityFrom = Math.max(from - numGroups, 0); + int entitySize = Math.min(Math.max(from + size - numGroups, 0), size); final SearchResponse entitiesResponse = - client.search(constructEntitiesSearchRequest(indexName, path, requestMap, from, size), + client.search(constructEntitiesSearchRequest(indexName, path, requestMap, entityFrom, entitySize), RequestOptions.DEFAULT); - final BrowseResult result = extractQueryResult(groupsResponse, entitiesResponse, path, from); - result.getMetadata().setPath(path); - return result; + final int numEntities = (int) entitiesResponse.getHits().getTotalHits().value; + final List browseResultEntityList = extractEntitiesResponse(entitiesResponse, path); + + return new BrowseResult().setMetadata(browseResultMetadata) + .setEntities(new BrowseResultEntityArray(browseResultEntityList)) + .setNumEntities(numGroups + numEntities) + .setFrom(from) + .setPageSize(size); } catch (Exception e) { log.error("Browse query failed: " + e.getMessage()); throw new ESQueryException("Browse query failed: ", e); @@ -99,11 +113,11 @@ private AggregationBuilder buildAggregations(@Nonnull String path) { final String includeFilter = ESUtils.escapeReservedCharacters(path) + "/.*"; final String excludeFilter = ESUtils.escapeReservedCharacters(path) + "/.*/.*"; - return AggregationBuilders.terms("groups") + return AggregationBuilders.terms(GROUP_AGG) .field(BROWSE_PATH) .size(Integer.MAX_VALUE) - .order(BucketOrder.count(true)) // Ascending order - .includeExclude(new IncludeExclude(includeFilter, excludeFilter)); + .includeExclude(new IncludeExclude(includeFilter, excludeFilter)) + .subAggregation(AggregationBuilders.terms(ALL_PATHS).field(BROWSE_PATH).size(Integer.MAX_VALUE)); } /** @@ -114,11 +128,10 @@ private AggregationBuilder buildAggregations(@Nonnull String path) { */ @Nonnull protected SearchRequest constructGroupsSearchRequest(@Nonnull String indexName, @Nonnull String path, - @Nonnull Map requestMap, int from, int size) { + @Nonnull Map requestMap) { final SearchRequest searchRequest = new SearchRequest(indexName); final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.from(from); - searchSourceBuilder.size(size); + searchSourceBuilder.size(0); searchSourceBuilder.query(buildQueryString(path, requestMap, true)); searchSourceBuilder.aggregation(buildAggregations(path)); searchRequest.source(searchSourceBuilder); @@ -180,29 +193,6 @@ SearchRequest constructEntitiesSearchRequest(@Nonnull String indexName, @Nonnull return searchRequest; } - /** - * Extracts search responses into browse result. - * - * @param groupsResponse groups search response - * @param entitiesResponse entity search response - * @param path the path which is being browsed - * @param from index of first entity - * @return {@link BrowseResult} - */ - @Nonnull - private BrowseResult extractQueryResult(@Nonnull SearchResponse groupsResponse, - @Nonnull SearchResponse entitiesResponse, @Nonnull String path, int from) { - final List browseResultEntityList = extractEntitiesResponse(entitiesResponse, path); - final BrowseResultMetadata browseResultMetadata = extractGroupsResponse(groupsResponse, path); - browseResultMetadata.setTotalNumEntities( - browseResultMetadata.getTotalNumEntities() + entitiesResponse.getHits().getTotalHits().value); - return new BrowseResult().setEntities(new BrowseResultEntityArray(browseResultEntityList)) - .setMetadata(browseResultMetadata) - .setFrom(from) - .setPageSize(browseResultEntityList.size()) - .setNumEntities((int) entitiesResponse.getHits().getTotalHits().value); - } - /** * Extracts group search response into browse result metadata. * @@ -211,21 +201,35 @@ private BrowseResult extractQueryResult(@Nonnull SearchResponse groupsResponse, * @return {@link BrowseResultMetadata} */ @Nonnull - private BrowseResultMetadata extractGroupsResponse(@Nonnull SearchResponse groupsResponse, @Nonnull String path) { - final ParsedTerms groups = (ParsedTerms) groupsResponse.getAggregations().getAsMap().get("groups"); + private BrowseResultMetadata extractGroupsResponse(@Nonnull SearchResponse groupsResponse, @Nonnull String path, + int from, int size) { + final ParsedTerms groups = groupsResponse.getAggregations().get(GROUP_AGG); final List groupsAgg = groups.getBuckets() .stream() + .filter(this::validateBucket) .map(group -> new BrowseResultGroup().setName(getSimpleName(group.getKeyAsString())) .setCount(group.getDocCount())) - // Sort by document count desc and then by name - .sorted(Comparator.comparing(BrowseResultGroup::getCount).reversed() - .thenComparing(Comparator.comparing(BrowseResultGroup::getName))) .collect(Collectors.toList()); - return new BrowseResultMetadata().setGroups(new BrowseResultGroupArray(groupsAgg)) - .setTotalNumEntities(groupsResponse.getHits().getTotalHits().value) + // Get the groups that are in the from to from + size range + final List paginatedGroups = + groupsAgg.size() <= from ? Collections.emptyList() : groupsAgg.subList(from, Math.min(from + size, groupsAgg.size())); + return new BrowseResultMetadata().setGroups(new BrowseResultGroupArray(paginatedGroups)) + .setTotalNumEntities(groupsAgg.size()) .setPath(path); } + /** + * Check if there are any paths that extends the matchedPath signifying that the path does not point to an entity + */ + private boolean validateBucket(@Nonnull MultiBucketsAggregation.Bucket bucket) { + final ParsedTerms groups = bucket.getAggregations().get(ALL_PATHS); + final String matchedPath = bucket.getKeyAsString(); + return groups.getBuckets() + .stream() + .map(MultiBucketsAggregation.Bucket::getKeyAsString) + .anyMatch(bucketPath -> (bucketPath.length() > matchedPath.length() && bucketPath.startsWith(matchedPath))); + } + /** * Extracts entity search response into list of browse result entities. * diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java index 8519d0d93b6be5..18d131d1fdea35 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESSearchDAO.java @@ -10,6 +10,7 @@ import com.linkedin.metadata.search.elasticsearch.query.request.AutocompleteRequestHandler; import com.linkedin.metadata.search.elasticsearch.query.request.SearchRequestHandler; import com.linkedin.metadata.utils.elasticsearch.IndexConvention; +import java.io.IOException; import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.RequiredArgsConstructor; @@ -18,6 +19,8 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.core.CountRequest; +import org.elasticsearch.index.query.QueryBuilders; /** @@ -31,6 +34,18 @@ public class ESSearchDAO { private final RestHighLevelClient client; private final IndexConvention indexConvention; + public long docCount(@Nonnull String entityName) { + EntitySpec entitySpec = entityRegistry.getEntitySpec(entityName); + CountRequest countRequest = + new CountRequest(indexConvention.getIndexName(entitySpec)).query(QueryBuilders.matchAllQuery()); + try { + return client.count(countRequest, RequestOptions.DEFAULT).getCount(); + } catch (IOException e) { + log.error("Count query failed:" + e.getMessage()); + throw new ESQueryException("Count query failed:", e); + } + } + @Nonnull private SearchResult executeAndExtract(@Nonnull EntitySpec entitySpec, @Nonnull SearchRequest searchRequest, int from, int size) { diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchServiceTest.java index 6de853732d29c1..929710358135f8 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchServiceTest.java @@ -87,6 +87,7 @@ public void testElasticSearchService() throws InterruptedException { assertEquals(searchResult.getNumEntities().intValue(), 0); BrowseResult browseResult = _elasticSearchService.browse(ENTITY_NAME, "", null, 0, 10); assertEquals(browseResult.getMetadata().getTotalNumEntities().longValue(), 0); + assertEquals(_elasticSearchService.docCount(ENTITY_NAME), 0); Urn urn = new TestEntityUrn("test", "testUrn", "VALUE_1"); ObjectNode document = JsonNodeFactory.instance.objectNode(); @@ -102,6 +103,7 @@ public void testElasticSearchService() throws InterruptedException { browseResult = _elasticSearchService.browse(ENTITY_NAME, "", null, 0, 10); assertEquals(browseResult.getMetadata().getTotalNumEntities().longValue(), 1); assertEquals(browseResult.getMetadata().getGroups().get(0).getName(), "a"); + assertEquals(_elasticSearchService.docCount(ENTITY_NAME), 1); _elasticSearchService.deleteDocument(ENTITY_NAME, urn.toString()); TimeUnit.SECONDS.sleep(5); @@ -109,5 +111,6 @@ public void testElasticSearchService() throws InterruptedException { assertEquals(searchResult.getNumEntities().intValue(), 0); browseResult = _elasticSearchService.browse(ENTITY_NAME, "", null, 0, 10); assertEquals(browseResult.getMetadata().getTotalNumEntities().longValue(), 0); + assertEquals(_elasticSearchService.docCount(ENTITY_NAME), 0); } } From 96d9d6d4052d129502e8b88da6973019332fca0b Mon Sep 17 00:00:00 2001 From: Dexter Lee Date: Thu, 29 Jul 2021 11:17:21 -0700 Subject: [PATCH 2/9] Fix to John's comments --- ...com.linkedin.entity.entities.restspec.json | 4 +-- ...com.linkedin.entity.entities.snapshot.json | 4 +-- .../linkedin/entity/client/EntityClient.java | 34 +++++++++---------- .../resources/entity/EntityResource.java | 8 ++--- .../elasticsearch/query/ESBrowseDAO.java | 3 ++ 5 files changed, 27 insertions(+), 26 deletions(-) diff --git a/gms/api/src/main/idl/com.linkedin.entity.entities.restspec.json b/gms/api/src/main/idl/com.linkedin.entity.entities.restspec.json index f5f38eed017b6d..1d66c1ba555eff 100644 --- a/gms/api/src/main/idl/com.linkedin.entity.entities.restspec.json +++ b/gms/api/src/main/idl/com.linkedin.entity.entities.restspec.json @@ -48,7 +48,7 @@ } ], "returns" : "com.linkedin.metadata.query.AutoCompleteResult" }, { - "name" : "batchGetNumEntities", + "name" : "batchGetTotalEntityCount", "parameters" : [ { "name" : "entities", "type" : "{ \"type\" : \"array\", \"items\" : \"string\" }" @@ -88,7 +88,7 @@ } ], "returns" : "{ \"type\" : \"array\", \"items\" : \"string\" }" }, { - "name" : "getNumEntities", + "name" : "getTotalEntityCount", "parameters" : [ { "name" : "entity", "type" : "string" diff --git a/gms/api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json b/gms/api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json index 4989f7ba7498ae..422578d6e77f2f 100644 --- a/gms/api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json +++ b/gms/api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json @@ -4341,7 +4341,7 @@ } ], "returns" : "com.linkedin.metadata.query.AutoCompleteResult" }, { - "name" : "batchGetNumEntities", + "name" : "batchGetTotalEntityCount", "parameters" : [ { "name" : "entities", "type" : "{ \"type\" : \"array\", \"items\" : \"string\" }" @@ -4381,7 +4381,7 @@ } ], "returns" : "{ \"type\" : \"array\", \"items\" : \"string\" }" }, { - "name" : "getNumEntities", + "name" : "getTotalEntityCount", "parameters" : [ { "name" : "entity", "type" : "string" diff --git a/gms/client/src/main/java/com/linkedin/entity/client/EntityClient.java b/gms/client/src/main/java/com/linkedin/entity/client/EntityClient.java index 0e79607d8bd794..a63958a88a2cb9 100644 --- a/gms/client/src/main/java/com/linkedin/entity/client/EntityClient.java +++ b/gms/client/src/main/java/com/linkedin/entity/client/EntityClient.java @@ -1,21 +1,17 @@ package com.linkedin.entity.client; import com.linkedin.common.urn.Urn; -import com.linkedin.entity.EntitiesDoBatchGetNumEntitiesRequestBuilder; -import com.linkedin.entity.EntitiesDoGetNumEntitiesRequestBuilder; -import com.linkedin.entity.EntitiesDoSetWritableRequestBuilder; -import com.linkedin.restli.client.Response; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import com.linkedin.data.template.DataTemplateUtil; import com.linkedin.data.template.DynamicRecordMetadata; import com.linkedin.data.template.FieldDef; import com.linkedin.data.template.StringArray; import com.linkedin.entity.EntitiesDoAutocompleteRequestBuilder; +import com.linkedin.entity.EntitiesDoBatchGetTotalEntityCountRequestBuilder; import com.linkedin.entity.EntitiesDoBrowseRequestBuilder; import com.linkedin.entity.EntitiesDoGetBrowsePathsRequestBuilder; +import com.linkedin.entity.EntitiesDoGetTotalEntityCountRequestBuilder; import com.linkedin.entity.EntitiesDoSearchRequestBuilder; +import com.linkedin.entity.EntitiesDoSetWritableRequestBuilder; import com.linkedin.entity.EntitiesRequestBuilders; import com.linkedin.entity.Entity; import com.linkedin.metadata.query.AutoCompleteResult; @@ -26,27 +22,29 @@ import com.linkedin.restli.client.BatchGetEntityRequest; import com.linkedin.restli.client.Client; import com.linkedin.restli.client.GetRequest; - import com.linkedin.restli.client.Request; +import com.linkedin.restli.client.Response; import com.linkedin.restli.client.RestLiResponseException; import com.linkedin.restli.client.RestliRequestOptions; import com.linkedin.restli.common.EmptyRecord; import com.linkedin.restli.common.ResourceSpec; import com.linkedin.restli.common.ResourceSpecImpl; -import java.util.Arrays; -import java.util.Collections; -import javax.annotation.Nonnull; import java.net.URISyntaxException; +import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; +import javax.annotation.Nonnull; import javax.annotation.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import static com.linkedin.metadata.dao.utils.QueryUtils.*; +import static com.linkedin.metadata.dao.utils.QueryUtils.newFilter; public class EntityClient { @@ -270,16 +268,16 @@ public void setWritable(boolean canWrite) throws RemoteInvocationException { } @Nonnull - public long getNumEntities(@Nonnull String entityName) throws RemoteInvocationException { - EntitiesDoGetNumEntitiesRequestBuilder requestBuilder = - ENTITIES_REQUEST_BUILDERS.actionGetNumEntities().entityParam(entityName); + public long getTotalEntityCount(@Nonnull String entityName) throws RemoteInvocationException { + EntitiesDoGetTotalEntityCountRequestBuilder requestBuilder = + ENTITIES_REQUEST_BUILDERS.actionGetTotalEntityCount().entityParam(entityName); return sendClientRequest(requestBuilder.build()).getEntity(); } @Nonnull - public Map batchGetNumEntities(@Nonnull List entityName) throws RemoteInvocationException { - EntitiesDoBatchGetNumEntitiesRequestBuilder requestBuilder = - ENTITIES_REQUEST_BUILDERS.actionBatchGetNumEntities().entitiesParam(new StringArray(entityName)); + public Map batchGetTotalEntityCount(@Nonnull List entityName) throws RemoteInvocationException { + EntitiesDoBatchGetTotalEntityCountRequestBuilder requestBuilder = + ENTITIES_REQUEST_BUILDERS.actionBatchGetTotalEntityCount().entitiesParam(new StringArray(entityName)); return sendClientRequest(requestBuilder.build()).getEntity(); } } diff --git a/gms/impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java b/gms/impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java index 50e8cc35538d9f..b170a19bddbcf2 100644 --- a/gms/impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java +++ b/gms/impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java @@ -203,15 +203,15 @@ public Task setWriteable(@ActionParam(PARAM_VALUE) @Optional("true") @Nonn }); } - @Action(name = "getNumEntities") + @Action(name = "getTotalEntityCount") @Nonnull - public Task getNumEntities(@ActionParam(PARAM_ENTITY) @Nonnull String entityName) { + public Task getTotalEntityCount(@ActionParam(PARAM_ENTITY) @Nonnull String entityName) { return RestliUtils.toTask(() -> _searchService.docCount(entityName)); } - @Action(name = "batchGetNumEntities") + @Action(name = "batchGetTotalEntityCount") @Nonnull - public Task batchGetNumEntities(@ActionParam(PARAM_ENTITIES) @Nonnull String[] entityNames) { + public Task batchGetTotalEntityCount(@ActionParam(PARAM_ENTITIES) @Nonnull String[] entityNames) { return RestliUtils.toTask(() -> new LongMap( Arrays.stream(entityNames).collect(Collectors.toMap(Function.identity(), _searchService::docCount)))); } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java index 117d7069d1bb43..22ad106e8c5ed1 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java @@ -83,6 +83,9 @@ public BrowseResult browse(@Nonnull String entityName, @Nonnull String path, @Nu final int numGroups = browseResultMetadata.getTotalNumEntities().intValue(); // Based on the number of groups returned, compute the from and size to query for entities + // Groups come before entities, so if numGroups >= from + size, we should return all groups + // if from < numGroups < from + size, we should return a mix of groups and entities + // if numGroups <= from, we should only return entities int entityFrom = Math.max(from - numGroups, 0); int entitySize = Math.min(Math.max(from + size - numGroups, 0), size); final SearchResponse entitiesResponse = From 5d90a336122b52789bb1cc927164ed886f76e236 Mon Sep 17 00:00:00 2001 From: Dexter Lee Date: Thu, 29 Jul 2021 18:37:51 -0700 Subject: [PATCH 3/9] Fix model --- .../graphql/types/chart/ChartType.java | 16 +---- .../types/dashboard/DashboardType.java | 16 +---- .../graphql/types/dataflow/DataFlowType.java | 15 +---- .../graphql/types/datajob/DataJobType.java | 15 +---- .../graphql/types/dataset/DatasetType.java | 15 +---- .../types/glossary/GlossaryTermType.java | 15 +---- .../types/mappers/BrowseResultMapper.java | 48 +++++++++++++++ .../mappers/BrowseResultMetadataMapper.java | 32 ---------- .../types/mlmodel/MLFeatureTableType.java | 15 +---- .../src/main/resources/gms.graphql | 10 ++-- .../src/app/browse/BrowseResultsPage.tsx | 2 +- datahub-web-react/src/graphql/browse.graphql | 8 +-- .../com.linkedin.chart.charts.snapshot.json | 48 ++++++++------- ...inkedin.dashboard.dashboards.snapshot.json | 48 ++++++++------- ....linkedin.dataflow.dataFlows.snapshot.json | 48 ++++++++------- ...om.linkedin.datajob.dataJobs.snapshot.json | 48 ++++++++------- ...om.linkedin.dataset.datasets.snapshot.json | 48 ++++++++------- ...com.linkedin.entity.entities.snapshot.json | 48 ++++++++------- ...kedin.glossary.glossaryTerms.snapshot.json | 48 ++++++++------- .../elasticsearch/query/ESBrowseDAO.java | 31 ++++++---- .../linkedin/metadata/query/BrowseResult.pdl | 58 +++++++++++++++++++ .../metadata/query/BrowseResultEntity.pdl | 19 ++++++ .../metadata/query/BrowseResultMetadata.pdl | 17 ++++++ 23 files changed, 386 insertions(+), 282 deletions(-) create mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/BrowseResultMapper.java delete mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/BrowseResultMetadataMapper.java create mode 100644 metadata-models/src/main/pegasus/com/linkedin/metadata/query/BrowseResult.pdl create mode 100644 metadata-models/src/main/pegasus/com/linkedin/metadata/query/BrowseResultEntity.pdl create mode 100644 metadata-models/src/main/pegasus/com/linkedin/metadata/query/BrowseResultMetadata.pdl diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/ChartType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/ChartType.java index 16e9fdc10f2dda..372164e79c4ac1 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/ChartType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/ChartType.java @@ -21,7 +21,7 @@ import com.linkedin.datahub.graphql.types.chart.mappers.ChartUpdateInputSnapshotMapper; import com.linkedin.datahub.graphql.types.mappers.AutoCompleteResultsMapper; import com.linkedin.datahub.graphql.types.mappers.BrowsePathsMapper; -import com.linkedin.datahub.graphql.types.mappers.BrowseResultMetadataMapper; +import com.linkedin.datahub.graphql.types.mappers.BrowseResultMapper; import com.linkedin.datahub.graphql.types.mappers.UrnSearchResultsMapper; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.configs.ChartSearchConfig; @@ -34,6 +34,7 @@ import com.linkedin.r2.RemoteInvocationException; import graphql.execution.DataFetcherResult; +import java.util.Optional; import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.net.URISyntaxException; @@ -134,18 +135,7 @@ public BrowseResults browse(@Nonnull List path, facetFilters, start, count); - final List urns = result.getEntities().stream().map(entity -> entity.getUrn().toString()).collect(Collectors.toList()); - final List charts = batchLoad(urns, context).stream().map(chartResult -> chartResult.getData()).collect( - Collectors.toList()); - final BrowseResults browseResults = new BrowseResults(); - browseResults.setStart(result.getFrom()); - browseResults.setCount(result.getPageSize()); - browseResults.setTotal(result.getNumEntities()); - browseResults.setMetadata(BrowseResultMetadataMapper.map(result.getMetadata())); - browseResults.setEntities(charts.stream() - .map(chart -> (com.linkedin.datahub.graphql.generated.Entity) chart) - .collect(Collectors.toList())); - return browseResults; + return BrowseResultMapper.map(result); } @Override diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java index 4d81eeba36776c..7076ba12913627 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java @@ -22,7 +22,7 @@ import com.linkedin.datahub.graphql.types.dashboard.mappers.DashboardUpdateInputSnapshotMapper; import com.linkedin.datahub.graphql.types.mappers.AutoCompleteResultsMapper; import com.linkedin.datahub.graphql.types.mappers.BrowsePathsMapper; -import com.linkedin.datahub.graphql.types.mappers.BrowseResultMetadataMapper; +import com.linkedin.datahub.graphql.types.mappers.BrowseResultMapper; import com.linkedin.datahub.graphql.types.mappers.UrnSearchResultsMapper; import com.linkedin.entity.client.EntityClient; import com.linkedin.entity.Entity; @@ -36,6 +36,7 @@ import com.linkedin.r2.RemoteInvocationException; import graphql.execution.DataFetcherResult; +import java.util.Optional; import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.net.URISyntaxException; @@ -134,18 +135,7 @@ public BrowseResults browse(@Nonnull List path, facetFilters, start, count); - final List urns = result.getEntities().stream().map(entity -> entity.getUrn().toString()).collect(Collectors.toList()); - final List dashboards = batchLoad(urns, context).stream().map(dashboardDataFetcherResult -> dashboardDataFetcherResult.getData()).collect( - Collectors.toList()); - final BrowseResults browseResults = new BrowseResults(); - browseResults.setStart(result.getFrom()); - browseResults.setCount(result.getPageSize()); - browseResults.setTotal(result.getNumEntities()); - browseResults.setMetadata(BrowseResultMetadataMapper.map(result.getMetadata())); - browseResults.setEntities(dashboards.stream() - .map(dashboard -> (com.linkedin.datahub.graphql.generated.Entity) dashboard) - .collect(Collectors.toList())); - return browseResults; + return BrowseResultMapper.map(result); } @Override diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/DataFlowType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/DataFlowType.java index b9cc96b02fdf14..50ea371a16e9b3 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/DataFlowType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/DataFlowType.java @@ -23,7 +23,7 @@ import com.linkedin.datahub.graphql.types.dataflow.mappers.DataFlowUpdateInputMapper; import com.linkedin.datahub.graphql.types.mappers.AutoCompleteResultsMapper; import com.linkedin.datahub.graphql.types.mappers.BrowsePathsMapper; -import com.linkedin.datahub.graphql.types.mappers.BrowseResultMetadataMapper; +import com.linkedin.datahub.graphql.types.mappers.BrowseResultMapper; import com.linkedin.datahub.graphql.types.mappers.UrnSearchResultsMapper; import com.linkedin.entity.client.EntityClient; import com.linkedin.entity.Entity; @@ -141,18 +141,7 @@ public BrowseResults browse(@Nonnull List path, @Nullable List urns = result.getEntities().stream().map(entity -> entity.getUrn().toString()).collect(Collectors.toList()); - final List dataFlows = batchLoad(urns, context).stream().map(dataFlow -> dataFlow.getData()).collect( - Collectors.toList()); - final BrowseResults browseResults = new BrowseResults(); - browseResults.setStart(result.getFrom()); - browseResults.setCount(result.getPageSize()); - browseResults.setTotal(result.getNumEntities()); - browseResults.setMetadata(BrowseResultMetadataMapper.map(result.getMetadata())); - browseResults.setEntities(dataFlows.stream() - .map(dataset -> (com.linkedin.datahub.graphql.generated.Entity) dataset) - .collect(Collectors.toList())); - return browseResults; + return BrowseResultMapper.map(result); } @Override diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/DataJobType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/DataJobType.java index a9bb736a73dc44..808c812a9ff426 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/DataJobType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/DataJobType.java @@ -23,7 +23,7 @@ import com.linkedin.datahub.graphql.types.MutableType; import com.linkedin.datahub.graphql.types.datajob.mappers.DataJobUpdateInputMapper; import com.linkedin.datahub.graphql.types.mappers.BrowsePathsMapper; -import com.linkedin.datahub.graphql.types.mappers.BrowseResultMetadataMapper; +import com.linkedin.datahub.graphql.types.mappers.BrowseResultMapper; import com.linkedin.datahub.graphql.types.mappers.UrnSearchResultsMapper; import com.linkedin.entity.client.EntityClient; import com.linkedin.entity.Entity; @@ -144,18 +144,7 @@ public BrowseResults browse(@Nonnull List path, @Nullable List urns = result.getEntities().stream().map(entity -> entity.getUrn().toString()).collect(Collectors.toList()); - final List dataJobs = batchLoad(urns, context).stream().map(dataFetcherResult -> dataFetcherResult.getData()).collect( - Collectors.toList()); - final BrowseResults browseResults = new BrowseResults(); - browseResults.setStart(result.getFrom()); - browseResults.setCount(result.getPageSize()); - browseResults.setTotal(result.getNumEntities()); - browseResults.setMetadata(BrowseResultMetadataMapper.map(result.getMetadata())); - browseResults.setEntities(dataJobs.stream() - .map(dataset -> (com.linkedin.datahub.graphql.generated.Entity) dataset) - .collect(Collectors.toList())); - return browseResults; + return BrowseResultMapper.map(result); } @Override diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java index eed691c61b0463..44491aa3654486 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java @@ -22,9 +22,9 @@ import com.linkedin.datahub.graphql.types.dataset.mappers.DatasetSnapshotMapper; import com.linkedin.datahub.graphql.types.mappers.AutoCompleteResultsMapper; import com.linkedin.datahub.graphql.types.mappers.BrowsePathsMapper; -import com.linkedin.datahub.graphql.types.mappers.BrowseResultMetadataMapper; import com.linkedin.datahub.graphql.types.dataset.mappers.DatasetUpdateInputMapper; import com.linkedin.datahub.graphql.resolvers.ResolverUtils; +import com.linkedin.datahub.graphql.types.mappers.BrowseResultMapper; import com.linkedin.datahub.graphql.types.mappers.UrnSearchResultsMapper; import com.linkedin.dataset.client.Datasets; import com.linkedin.entity.client.EntityClient; @@ -139,18 +139,7 @@ public BrowseResults browse(@Nonnull List path, facetFilters, start, count); - final List urns = result.getEntities().stream().map(entity -> entity.getUrn().toString()).collect(Collectors.toList()); - final List datasets = batchLoad(urns, context) - .stream().map(datasetDataFetcherResult -> datasetDataFetcherResult.getData()).collect(Collectors.toList()); - final BrowseResults browseResults = new BrowseResults(); - browseResults.setStart(result.getFrom()); - browseResults.setCount(result.getPageSize()); - browseResults.setTotal(result.getNumEntities()); - browseResults.setMetadata(BrowseResultMetadataMapper.map(result.getMetadata())); - browseResults.setEntities(datasets.stream() - .map(dataset -> (com.linkedin.datahub.graphql.generated.Entity) dataset) - .collect(Collectors.toList())); - return browseResults; + return BrowseResultMapper.map(result); } @Override diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/GlossaryTermType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/GlossaryTermType.java index 34a75097b8ad49..f228fa06553d45 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/GlossaryTermType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/GlossaryTermType.java @@ -17,8 +17,8 @@ import com.linkedin.datahub.graphql.types.glossary.mappers.GlossaryTermSnapshotMapper; import com.linkedin.datahub.graphql.types.mappers.AutoCompleteResultsMapper; import com.linkedin.datahub.graphql.types.mappers.BrowsePathsMapper; -import com.linkedin.datahub.graphql.types.mappers.BrowseResultMetadataMapper; import com.linkedin.datahub.graphql.resolvers.ResolverUtils; +import com.linkedin.datahub.graphql.types.mappers.BrowseResultMapper; import com.linkedin.datahub.graphql.types.mappers.UrnSearchResultsMapper; import com.linkedin.entity.client.EntityClient; import com.linkedin.entity.Entity; @@ -126,18 +126,7 @@ public BrowseResults browse(@Nonnull List path, facetFilters, start, count); - final List urns = result.getEntities().stream().map(entity -> entity.getUrn().toString()).collect(Collectors.toList()); - final List glossaryTerms = batchLoad(urns, context).stream().map(term -> term.getData()).collect( - Collectors.toList()); - final BrowseResults browseResults = new BrowseResults(); - browseResults.setStart(result.getFrom()); - browseResults.setCount(result.getPageSize()); - browseResults.setTotal(result.getNumEntities()); - browseResults.setMetadata(BrowseResultMetadataMapper.map(result.getMetadata())); - browseResults.setEntities(glossaryTerms.stream() - .map(glossaryTerm -> (com.linkedin.datahub.graphql.generated.Entity) glossaryTerm) - .collect(Collectors.toList())); - return browseResults; + return BrowseResultMapper.map(result); } @Override diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/BrowseResultMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/BrowseResultMapper.java new file mode 100644 index 00000000000000..0bfc0d64e104c8 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/BrowseResultMapper.java @@ -0,0 +1,48 @@ +package com.linkedin.datahub.graphql.types.mappers; + +import com.linkedin.datahub.graphql.generated.BrowseResultGroup; +import com.linkedin.datahub.graphql.generated.BrowseResultMetadata; +import com.linkedin.datahub.graphql.generated.BrowseResults; +import com.linkedin.datahub.graphql.generated.Entity; +import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper; +import java.util.List; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; + + +public class BrowseResultMapper { + + public static BrowseResults map(com.linkedin.metadata.query.BrowseResult input) { + final BrowseResults result = new BrowseResults(); + + if (!input.hasFrom() || !input.hasPageSize() || !input.hasNumElements()) { + return result; + } + + result.setStart(input.getFrom()); + result.setCount(input.getPageSize()); + result.setTotal(input.getNumElements()); + + final BrowseResultMetadata browseResultMetadata = new BrowseResultMetadata(); + browseResultMetadata.setPath(BrowsePathMapper.map(input.getMetadata().getPath()).getPath()); + browseResultMetadata.setTotalNumEntities(input.getMetadata().getTotalNumEntities()); + result.setMetadata(browseResultMetadata); + + List entities = + input.getEntities().stream().map(entity -> UrnToEntityMapper.map(entity.getUrn())).collect(Collectors.toList()); + result.setEntities(entities); + + List groups = + input.getGroups().stream().map(BrowseResultMapper::mapGroup).collect(Collectors.toList()); + result.setGroups(groups); + + return result; + } + + private static BrowseResultGroup mapGroup(@Nonnull final com.linkedin.metadata.query.BrowseResultGroup group) { + final BrowseResultGroup result = new BrowseResultGroup(); + result.setName(group.getName()); + result.setCount(group.getCount()); + return result; + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/BrowseResultMetadataMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/BrowseResultMetadataMapper.java deleted file mode 100644 index 5cce9475c55694..00000000000000 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/BrowseResultMetadataMapper.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.linkedin.datahub.graphql.types.mappers; - -import com.linkedin.datahub.graphql.generated.BrowseResultGroup; -import com.linkedin.datahub.graphql.generated.BrowseResultMetadata; - -import javax.annotation.Nonnull; -import java.util.stream.Collectors; - -public class BrowseResultMetadataMapper implements ModelMapper { - - public static final BrowseResultMetadataMapper INSTANCE = new BrowseResultMetadataMapper(); - - public static BrowseResultMetadata map(@Nonnull final com.linkedin.metadata.query.BrowseResultMetadata input) { - return INSTANCE.apply(input); - } - - @Override - public BrowseResultMetadata apply(@Nonnull final com.linkedin.metadata.query.BrowseResultMetadata input) { - final BrowseResultMetadata result = new BrowseResultMetadata(); - result.setPath(BrowsePathMapper.map(input.getPath()).getPath()); - result.setTotalNumEntities(input.getTotalNumEntities()); - result.setGroups(input.getGroups().stream().map(this::mapGroup).collect(Collectors.toList())); - return result; - } - - private BrowseResultGroup mapGroup(@Nonnull final com.linkedin.metadata.query.BrowseResultGroup group) { - final BrowseResultGroup result = new BrowseResultGroup(); - result.setName(group.getName()); - result.setCount(group.getCount()); - return result; - } -} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLFeatureTableType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLFeatureTableType.java index e83178c59ccfc1..0edd13c2566fc1 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLFeatureTableType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLFeatureTableType.java @@ -16,7 +16,7 @@ import com.linkedin.datahub.graphql.types.BrowsableEntityType; import com.linkedin.datahub.graphql.types.mappers.AutoCompleteResultsMapper; import com.linkedin.datahub.graphql.types.mappers.BrowsePathsMapper; -import com.linkedin.datahub.graphql.types.mappers.BrowseResultMetadataMapper; +import com.linkedin.datahub.graphql.types.mappers.BrowseResultMapper; import com.linkedin.datahub.graphql.types.mappers.UrnSearchResultsMapper; import com.linkedin.datahub.graphql.types.mlmodel.mappers.MLFeatureTableSnapshotMapper; import com.linkedin.entity.Entity; @@ -119,18 +119,7 @@ public BrowseResults browse(@Nonnull List path, facetFilters, start, count); - final List urns = result.getEntities().stream().map(entity -> entity.getUrn().toString()).collect(Collectors.toList()); - final List mlFeatureTables = batchLoad(urns, context) - .stream().map(mlFeatureTableDataFetcherResult -> mlFeatureTableDataFetcherResult.getData()).collect(Collectors.toList()); - final BrowseResults browseResults = new BrowseResults(); - browseResults.setStart(result.getFrom()); - browseResults.setCount(result.getPageSize()); - browseResults.setTotal(result.getNumEntities()); - browseResults.setMetadata(BrowseResultMetadataMapper.map(result.getMetadata())); - browseResults.setEntities(mlFeatureTables.stream() - .map(entity -> (com.linkedin.datahub.graphql.generated.Entity) entity) - .collect(Collectors.toList())); - return browseResults; + return BrowseResultMapper.map(result); } @Override diff --git a/datahub-graphql-core/src/main/resources/gms.graphql b/datahub-graphql-core/src/main/resources/gms.graphql index 0fe480f032b5b0..c249544c10c06d 100644 --- a/datahub-graphql-core/src/main/resources/gms.graphql +++ b/datahub-graphql-core/src/main/resources/gms.graphql @@ -1418,6 +1418,11 @@ type BrowseResults { """ entities: [Entity!]! + """ + The groups present at the provided browse path + """ + groups: [BrowseResultGroup!]! + """ The starting point of paginated results """ @@ -1444,10 +1449,7 @@ type BrowseResultMetadata { The provided path """ path: [String!]! - """ - The groups present at the provided browse path - """ - groups: [BrowseResultGroup!]! + """ The total number of entities under the provided browse path """ diff --git a/datahub-web-react/src/app/browse/BrowseResultsPage.tsx b/datahub-web-react/src/app/browse/BrowseResultsPage.tsx index 1a1e0bc6ce7ad8..38fba35a37a0ba 100644 --- a/datahub-web-react/src/app/browse/BrowseResultsPage.tsx +++ b/datahub-web-react/src/app/browse/BrowseResultsPage.tsx @@ -68,7 +68,7 @@ export const BrowseResultsPage = () => { title={path.length > 0 ? path[path.length - 1] : entityRegistry.getCollectionName(entityType)} page={page} pageSize={BrowseCfg.RESULTS_PER_PAGE} - groups={data.browse.metadata.groups} + groups={data.browse.groups} entities={data.browse.entities} totalResults={data.browse.total} onChangePage={onChangePage} diff --git a/datahub-web-react/src/graphql/browse.graphql b/datahub-web-react/src/graphql/browse.graphql index 70b1f595379c99..9ccdefac928975 100644 --- a/datahub-web-react/src/graphql/browse.graphql +++ b/datahub-web-react/src/graphql/browse.graphql @@ -123,15 +123,15 @@ query getBrowseResults($input: BrowseInput!) { } } } + groups { + name + count + } start count total metadata { path - groups { - name - count - } totalNumEntities } } diff --git a/gms/api/src/main/snapshot/com.linkedin.chart.charts.snapshot.json b/gms/api/src/main/snapshot/com.linkedin.chart.charts.snapshot.json index c72631c64ecc94..34428bea6f108b 100644 --- a/gms/api/src/main/snapshot/com.linkedin.chart.charts.snapshot.json +++ b/gms/api/src/main/snapshot/com.linkedin.chart.charts.snapshot.json @@ -647,6 +647,26 @@ } }, "doc" : "A list of entities under the queried path" + }, { + "name" : "groups", + "type" : { + "type" : "array", + "items" : { + "type" : "record", + "name" : "BrowseResultGroup", + "fields" : [ { + "name" : "name", + "type" : "string", + "doc" : "Name of the group" + }, { + "name" : "count", + "type" : "long", + "doc" : "Number of entities that can be reached from this path" + } ] + } + }, + "doc" : "A list of groups and total number of entities inside those groups under the queried path", + "default" : [ ] }, { "name" : "metadata", "type" : { @@ -657,26 +677,6 @@ "name" : "path", "type" : "string", "doc" : "Path that is being browsed" - }, { - "name" : "groups", - "type" : { - "type" : "array", - "items" : { - "type" : "record", - "name" : "BrowseResultGroup", - "fields" : [ { - "name" : "name", - "type" : "string", - "doc" : "Name of the group" - }, { - "name" : "count", - "type" : "long", - "doc" : "Number of entities that can be reached from this path" - } ] - } - }, - "doc" : "A list of groups and total number of entities inside those groups under the queried path", - "default" : [ ] }, { "name" : "totalNumEntities", "type" : "long", @@ -696,6 +696,14 @@ "name" : "numEntities", "type" : "int", "doc" : "The total number of entities directly under queried path" + }, { + "name" : "numGroups", + "type" : "int", + "doc" : "The total number of groups directly under queried path" + }, { + "name" : "numElements", + "type" : "int", + "doc" : "The total number of elements (entities + groups) directly under queried path" } ] }, "com.linkedin.metadata.query.BrowseResultEntity", "com.linkedin.metadata.query.BrowseResultGroup", "com.linkedin.metadata.query.BrowseResultMetadata", { "type" : "enum", diff --git a/gms/api/src/main/snapshot/com.linkedin.dashboard.dashboards.snapshot.json b/gms/api/src/main/snapshot/com.linkedin.dashboard.dashboards.snapshot.json index d1ac44a1309154..e940efe18a1b77 100644 --- a/gms/api/src/main/snapshot/com.linkedin.dashboard.dashboards.snapshot.json +++ b/gms/api/src/main/snapshot/com.linkedin.dashboard.dashboards.snapshot.json @@ -586,6 +586,26 @@ } }, "doc" : "A list of entities under the queried path" + }, { + "name" : "groups", + "type" : { + "type" : "array", + "items" : { + "type" : "record", + "name" : "BrowseResultGroup", + "fields" : [ { + "name" : "name", + "type" : "string", + "doc" : "Name of the group" + }, { + "name" : "count", + "type" : "long", + "doc" : "Number of entities that can be reached from this path" + } ] + } + }, + "doc" : "A list of groups and total number of entities inside those groups under the queried path", + "default" : [ ] }, { "name" : "metadata", "type" : { @@ -596,26 +616,6 @@ "name" : "path", "type" : "string", "doc" : "Path that is being browsed" - }, { - "name" : "groups", - "type" : { - "type" : "array", - "items" : { - "type" : "record", - "name" : "BrowseResultGroup", - "fields" : [ { - "name" : "name", - "type" : "string", - "doc" : "Name of the group" - }, { - "name" : "count", - "type" : "long", - "doc" : "Number of entities that can be reached from this path" - } ] - } - }, - "doc" : "A list of groups and total number of entities inside those groups under the queried path", - "default" : [ ] }, { "name" : "totalNumEntities", "type" : "long", @@ -635,6 +635,14 @@ "name" : "numEntities", "type" : "int", "doc" : "The total number of entities directly under queried path" + }, { + "name" : "numGroups", + "type" : "int", + "doc" : "The total number of groups directly under queried path" + }, { + "name" : "numElements", + "type" : "int", + "doc" : "The total number of elements (entities + groups) directly under queried path" } ] }, "com.linkedin.metadata.query.BrowseResultEntity", "com.linkedin.metadata.query.BrowseResultGroup", "com.linkedin.metadata.query.BrowseResultMetadata", { "type" : "enum", diff --git a/gms/api/src/main/snapshot/com.linkedin.dataflow.dataFlows.snapshot.json b/gms/api/src/main/snapshot/com.linkedin.dataflow.dataFlows.snapshot.json index 70c2298a73c0c9..91ea278359f76d 100644 --- a/gms/api/src/main/snapshot/com.linkedin.dataflow.dataFlows.snapshot.json +++ b/gms/api/src/main/snapshot/com.linkedin.dataflow.dataFlows.snapshot.json @@ -539,6 +539,26 @@ } }, "doc" : "A list of entities under the queried path" + }, { + "name" : "groups", + "type" : { + "type" : "array", + "items" : { + "type" : "record", + "name" : "BrowseResultGroup", + "fields" : [ { + "name" : "name", + "type" : "string", + "doc" : "Name of the group" + }, { + "name" : "count", + "type" : "long", + "doc" : "Number of entities that can be reached from this path" + } ] + } + }, + "doc" : "A list of groups and total number of entities inside those groups under the queried path", + "default" : [ ] }, { "name" : "metadata", "type" : { @@ -549,26 +569,6 @@ "name" : "path", "type" : "string", "doc" : "Path that is being browsed" - }, { - "name" : "groups", - "type" : { - "type" : "array", - "items" : { - "type" : "record", - "name" : "BrowseResultGroup", - "fields" : [ { - "name" : "name", - "type" : "string", - "doc" : "Name of the group" - }, { - "name" : "count", - "type" : "long", - "doc" : "Number of entities that can be reached from this path" - } ] - } - }, - "doc" : "A list of groups and total number of entities inside those groups under the queried path", - "default" : [ ] }, { "name" : "totalNumEntities", "type" : "long", @@ -588,6 +588,14 @@ "name" : "numEntities", "type" : "int", "doc" : "The total number of entities directly under queried path" + }, { + "name" : "numGroups", + "type" : "int", + "doc" : "The total number of groups directly under queried path" + }, { + "name" : "numElements", + "type" : "int", + "doc" : "The total number of elements (entities + groups) directly under queried path" } ] }, "com.linkedin.metadata.query.BrowseResultEntity", "com.linkedin.metadata.query.BrowseResultGroup", "com.linkedin.metadata.query.BrowseResultMetadata", { "type" : "enum", diff --git a/gms/api/src/main/snapshot/com.linkedin.datajob.dataJobs.snapshot.json b/gms/api/src/main/snapshot/com.linkedin.datajob.dataJobs.snapshot.json index 0c814cfa9237ac..50ef052a967d63 100644 --- a/gms/api/src/main/snapshot/com.linkedin.datajob.dataJobs.snapshot.json +++ b/gms/api/src/main/snapshot/com.linkedin.datajob.dataJobs.snapshot.json @@ -700,6 +700,26 @@ } }, "doc" : "A list of entities under the queried path" + }, { + "name" : "groups", + "type" : { + "type" : "array", + "items" : { + "type" : "record", + "name" : "BrowseResultGroup", + "fields" : [ { + "name" : "name", + "type" : "string", + "doc" : "Name of the group" + }, { + "name" : "count", + "type" : "long", + "doc" : "Number of entities that can be reached from this path" + } ] + } + }, + "doc" : "A list of groups and total number of entities inside those groups under the queried path", + "default" : [ ] }, { "name" : "metadata", "type" : { @@ -710,26 +730,6 @@ "name" : "path", "type" : "string", "doc" : "Path that is being browsed" - }, { - "name" : "groups", - "type" : { - "type" : "array", - "items" : { - "type" : "record", - "name" : "BrowseResultGroup", - "fields" : [ { - "name" : "name", - "type" : "string", - "doc" : "Name of the group" - }, { - "name" : "count", - "type" : "long", - "doc" : "Number of entities that can be reached from this path" - } ] - } - }, - "doc" : "A list of groups and total number of entities inside those groups under the queried path", - "default" : [ ] }, { "name" : "totalNumEntities", "type" : "long", @@ -749,6 +749,14 @@ "name" : "numEntities", "type" : "int", "doc" : "The total number of entities directly under queried path" + }, { + "name" : "numGroups", + "type" : "int", + "doc" : "The total number of groups directly under queried path" + }, { + "name" : "numElements", + "type" : "int", + "doc" : "The total number of elements (entities + groups) directly under queried path" } ] }, "com.linkedin.metadata.query.BrowseResultEntity", "com.linkedin.metadata.query.BrowseResultGroup", "com.linkedin.metadata.query.BrowseResultMetadata", { "type" : "enum", diff --git a/gms/api/src/main/snapshot/com.linkedin.dataset.datasets.snapshot.json b/gms/api/src/main/snapshot/com.linkedin.dataset.datasets.snapshot.json index 7b278264373718..ae06a176b09a23 100644 --- a/gms/api/src/main/snapshot/com.linkedin.dataset.datasets.snapshot.json +++ b/gms/api/src/main/snapshot/com.linkedin.dataset.datasets.snapshot.json @@ -1422,6 +1422,26 @@ } }, "doc" : "A list of entities under the queried path" + }, { + "name" : "groups", + "type" : { + "type" : "array", + "items" : { + "type" : "record", + "name" : "BrowseResultGroup", + "fields" : [ { + "name" : "name", + "type" : "string", + "doc" : "Name of the group" + }, { + "name" : "count", + "type" : "long", + "doc" : "Number of entities that can be reached from this path" + } ] + } + }, + "doc" : "A list of groups and total number of entities inside those groups under the queried path", + "default" : [ ] }, { "name" : "metadata", "type" : { @@ -1432,26 +1452,6 @@ "name" : "path", "type" : "string", "doc" : "Path that is being browsed" - }, { - "name" : "groups", - "type" : { - "type" : "array", - "items" : { - "type" : "record", - "name" : "BrowseResultGroup", - "fields" : [ { - "name" : "name", - "type" : "string", - "doc" : "Name of the group" - }, { - "name" : "count", - "type" : "long", - "doc" : "Number of entities that can be reached from this path" - } ] - } - }, - "doc" : "A list of groups and total number of entities inside those groups under the queried path", - "default" : [ ] }, { "name" : "totalNumEntities", "type" : "long", @@ -1471,6 +1471,14 @@ "name" : "numEntities", "type" : "int", "doc" : "The total number of entities directly under queried path" + }, { + "name" : "numGroups", + "type" : "int", + "doc" : "The total number of groups directly under queried path" + }, { + "name" : "numElements", + "type" : "int", + "doc" : "The total number of elements (entities + groups) directly under queried path" } ] }, "com.linkedin.metadata.query.BrowseResultEntity", "com.linkedin.metadata.query.BrowseResultGroup", "com.linkedin.metadata.query.BrowseResultMetadata", { "type" : "enum", diff --git a/gms/api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json b/gms/api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json index 422578d6e77f2f..c9de8f5dac92d7 100644 --- a/gms/api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json +++ b/gms/api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json @@ -4089,6 +4089,26 @@ } }, "doc" : "A list of entities under the queried path" + }, { + "name" : "groups", + "type" : { + "type" : "array", + "items" : { + "type" : "record", + "name" : "BrowseResultGroup", + "fields" : [ { + "name" : "name", + "type" : "string", + "doc" : "Name of the group" + }, { + "name" : "count", + "type" : "long", + "doc" : "Number of entities that can be reached from this path" + } ] + } + }, + "doc" : "A list of groups and total number of entities inside those groups under the queried path", + "default" : [ ] }, { "name" : "metadata", "type" : { @@ -4099,26 +4119,6 @@ "name" : "path", "type" : "string", "doc" : "Path that is being browsed" - }, { - "name" : "groups", - "type" : { - "type" : "array", - "items" : { - "type" : "record", - "name" : "BrowseResultGroup", - "fields" : [ { - "name" : "name", - "type" : "string", - "doc" : "Name of the group" - }, { - "name" : "count", - "type" : "long", - "doc" : "Number of entities that can be reached from this path" - } ] - } - }, - "doc" : "A list of groups and total number of entities inside those groups under the queried path", - "default" : [ ] }, { "name" : "totalNumEntities", "type" : "long", @@ -4138,6 +4138,14 @@ "name" : "numEntities", "type" : "int", "doc" : "The total number of entities directly under queried path" + }, { + "name" : "numGroups", + "type" : "int", + "doc" : "The total number of groups directly under queried path" + }, { + "name" : "numElements", + "type" : "int", + "doc" : "The total number of elements (entities + groups) directly under queried path" } ] }, "com.linkedin.metadata.query.BrowseResultEntity", "com.linkedin.metadata.query.BrowseResultGroup", "com.linkedin.metadata.query.BrowseResultMetadata", { "type" : "enum", diff --git a/gms/api/src/main/snapshot/com.linkedin.glossary.glossaryTerms.snapshot.json b/gms/api/src/main/snapshot/com.linkedin.glossary.glossaryTerms.snapshot.json index 6b3503f77f9ed9..7ba389639c504f 100644 --- a/gms/api/src/main/snapshot/com.linkedin.glossary.glossaryTerms.snapshot.json +++ b/gms/api/src/main/snapshot/com.linkedin.glossary.glossaryTerms.snapshot.json @@ -388,6 +388,26 @@ } }, "doc" : "A list of entities under the queried path" + }, { + "name" : "groups", + "type" : { + "type" : "array", + "items" : { + "type" : "record", + "name" : "BrowseResultGroup", + "fields" : [ { + "name" : "name", + "type" : "string", + "doc" : "Name of the group" + }, { + "name" : "count", + "type" : "long", + "doc" : "Number of entities that can be reached from this path" + } ] + } + }, + "doc" : "A list of groups and total number of entities inside those groups under the queried path", + "default" : [ ] }, { "name" : "metadata", "type" : { @@ -398,26 +418,6 @@ "name" : "path", "type" : "string", "doc" : "Path that is being browsed" - }, { - "name" : "groups", - "type" : { - "type" : "array", - "items" : { - "type" : "record", - "name" : "BrowseResultGroup", - "fields" : [ { - "name" : "name", - "type" : "string", - "doc" : "Name of the group" - }, { - "name" : "count", - "type" : "long", - "doc" : "Number of entities that can be reached from this path" - } ] - } - }, - "doc" : "A list of groups and total number of entities inside those groups under the queried path", - "default" : [ ] }, { "name" : "totalNumEntities", "type" : "long", @@ -437,6 +437,14 @@ "name" : "numEntities", "type" : "int", "doc" : "The total number of entities directly under queried path" + }, { + "name" : "numGroups", + "type" : "int", + "doc" : "The total number of groups directly under queried path" + }, { + "name" : "numElements", + "type" : "int", + "doc" : "The total number of elements (entities + groups) directly under queried path" } ] }, "com.linkedin.metadata.query.BrowseResultEntity", "com.linkedin.metadata.query.BrowseResultGroup", "com.linkedin.metadata.query.BrowseResultMetadata", { "type" : "enum", diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java index 22ad106e8c5ed1..80ca420caf3880 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java @@ -24,6 +24,7 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.RequiredArgsConstructor; +import lombok.Value; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang.StringUtils; import org.elasticsearch.action.search.SearchRequest; @@ -59,6 +60,13 @@ public class ESBrowseDAO { private static final String GROUP_AGG = "groups"; private static final String ALL_PATHS = "allPaths"; + @Value + private class BrowseGroupsResult { + List groups; + int totalGroups; + int totalNumEntities; + } + /** * Gets a list of groups/entities that match given browse request. * @@ -79,8 +87,8 @@ public BrowseResult browse(@Nonnull String entityName, @Nonnull String path, @Nu final SearchResponse groupsResponse = client.search(constructGroupsSearchRequest(indexName, path, requestMap), RequestOptions.DEFAULT); - final BrowseResultMetadata browseResultMetadata = extractGroupsResponse(groupsResponse, path, from, size); - final int numGroups = browseResultMetadata.getTotalNumEntities().intValue(); + final BrowseGroupsResult browseGroupsResult = extractGroupsResponse(groupsResponse, path, from, size); + final int numGroups = browseGroupsResult.getTotalGroups(); // Based on the number of groups returned, compute the from and size to query for entities // Groups come before entities, so if numGroups >= from + size, we should return all groups @@ -94,9 +102,13 @@ public BrowseResult browse(@Nonnull String entityName, @Nonnull String path, @Nu final int numEntities = (int) entitiesResponse.getHits().getTotalHits().value; final List browseResultEntityList = extractEntitiesResponse(entitiesResponse, path); - return new BrowseResult().setMetadata(browseResultMetadata) + return new BrowseResult().setMetadata( + new BrowseResultMetadata().setTotalNumEntities(browseGroupsResult.getTotalNumEntities()).setPath(path)) .setEntities(new BrowseResultEntityArray(browseResultEntityList)) - .setNumEntities(numGroups + numEntities) + .setGroups(new BrowseResultGroupArray(browseGroupsResult.getGroups())) + .setNumEntities(numEntities) + .setNumGroups(numGroups) + .setNumElements(numGroups + numEntities) .setFrom(from) .setPageSize(size); } catch (Exception e) { @@ -204,7 +216,7 @@ SearchRequest constructEntitiesSearchRequest(@Nonnull String indexName, @Nonnull * @return {@link BrowseResultMetadata} */ @Nonnull - private BrowseResultMetadata extractGroupsResponse(@Nonnull SearchResponse groupsResponse, @Nonnull String path, + private BrowseGroupsResult extractGroupsResponse(@Nonnull SearchResponse groupsResponse, @Nonnull String path, int from, int size) { final ParsedTerms groups = groupsResponse.getAggregations().get(GROUP_AGG); final List groupsAgg = groups.getBuckets() @@ -214,11 +226,10 @@ private BrowseResultMetadata extractGroupsResponse(@Nonnull SearchResponse group .setCount(group.getDocCount())) .collect(Collectors.toList()); // Get the groups that are in the from to from + size range - final List paginatedGroups = - groupsAgg.size() <= from ? Collections.emptyList() : groupsAgg.subList(from, Math.min(from + size, groupsAgg.size())); - return new BrowseResultMetadata().setGroups(new BrowseResultGroupArray(paginatedGroups)) - .setTotalNumEntities(groupsAgg.size()) - .setPath(path); + final List paginatedGroups = groupsAgg.size() <= from ? Collections.emptyList() + : groupsAgg.subList(from, Math.min(from + size, groupsAgg.size())); + return new BrowseGroupsResult(paginatedGroups, groupsAgg.size(), + (int) groupsResponse.getHits().getTotalHits().value); } /** diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/query/BrowseResult.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/query/BrowseResult.pdl new file mode 100644 index 00000000000000..3eec897b0da67e --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/metadata/query/BrowseResult.pdl @@ -0,0 +1,58 @@ +namespace com.linkedin.metadata.query + +/** + * The model for the result of a browse query + */ +record BrowseResult { + + /** + * A list of entities under the queried path + */ + entities: array[BrowseResultEntity] + + /** + * A list of groups and total number of entities inside those groups under the queried path + */ + groups: array[record BrowseResultGroup { + + /** + * Name of the group + */ + name: string + + /** + * Number of entities that can be reached from this path + */ + count: long + }] = [ ] + + /** + * Metadata specific to the browse result of the queried path + */ + metadata: BrowseResultMetadata + + /** + * Offset of the first entity in the result + */ + from: int + + /** + * Size of each page in the result + */ + pageSize: int + + /** + * The total number of entities directly under queried path + */ + numEntities: int + + /** + * The total number of groups directly under queried path + */ + numGroups: int + + /** + * The total number of elements (entities + groups) directly under queried path + */ + numElements: int +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/query/BrowseResultEntity.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/query/BrowseResultEntity.pdl new file mode 100644 index 00000000000000..ca9ca0f18aa72e --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/metadata/query/BrowseResultEntity.pdl @@ -0,0 +1,19 @@ +namespace com.linkedin.metadata.query + +import com.linkedin.common.Urn + +/** + * Data model for an entity returned as part of a browse query + */ +record BrowseResultEntity { + + /** + * Name of the entity + */ + name: optional string + + /** + * URN of the entity + */ + urn: Urn +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/query/BrowseResultMetadata.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/query/BrowseResultMetadata.pdl new file mode 100644 index 00000000000000..5db8fbd8ea4a45 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/metadata/query/BrowseResultMetadata.pdl @@ -0,0 +1,17 @@ +namespace com.linkedin.metadata.query + +/** + * The model for browse result metadata + */ +record BrowseResultMetadata { + + /** + * Path that is being browsed + */ + path: string + + /** + * Total number of entities we can reach from path + */ + totalNumEntities: long +} \ No newline at end of file From 096a1e0026fec9788147c3dd3d7d86a884fcc686 Mon Sep 17 00:00:00 2001 From: Dexter Lee Date: Thu, 29 Jul 2021 18:43:34 -0700 Subject: [PATCH 4/9] Fixing --- .../java/com/linkedin/datahub/graphql/types/chart/ChartType.java | 1 - 1 file changed, 1 deletion(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/ChartType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/ChartType.java index 372164e79c4ac1..30d33f558b9cb0 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/ChartType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/ChartType.java @@ -34,7 +34,6 @@ import com.linkedin.r2.RemoteInvocationException; import graphql.execution.DataFetcherResult; -import java.util.Optional; import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.net.URISyntaxException; From 8a0e92123418322f680fb858f27606af34838cd6 Mon Sep 17 00:00:00 2001 From: Dexter Lee Date: Thu, 29 Jul 2021 19:03:32 -0700 Subject: [PATCH 5/9] Fix style --- .../graphql/types/dashboard/DashboardType.java | 1 - .../graphql/types/mappers/BrowseResultMapper.java | 2 ++ .../graphql/types/mlmodel/MLModelGroupType.java | 15 ++------------- .../graphql/types/mlmodel/MLModelType.java | 15 ++------------- .../src/graphql-mock/fixtures/browsePathHelper.ts | 4 ++-- .../elasticsearch/ElasticSearchServiceTest.java | 2 +- 6 files changed, 9 insertions(+), 30 deletions(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java index 7076ba12913627..bad7b894a797e2 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java @@ -36,7 +36,6 @@ import com.linkedin.r2.RemoteInvocationException; import graphql.execution.DataFetcherResult; -import java.util.Optional; import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.net.URISyntaxException; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/BrowseResultMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/BrowseResultMapper.java index 0bfc0d64e104c8..138d5f5f6f1c4a 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/BrowseResultMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/BrowseResultMapper.java @@ -11,6 +11,8 @@ public class BrowseResultMapper { + private BrowseResultMapper() { + } public static BrowseResults map(com.linkedin.metadata.query.BrowseResult input) { final BrowseResults result = new BrowseResults(); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLModelGroupType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLModelGroupType.java index 3584bd5ac2a07f..4be080ef422efa 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLModelGroupType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLModelGroupType.java @@ -16,7 +16,7 @@ import com.linkedin.datahub.graphql.types.SearchableEntityType; import com.linkedin.datahub.graphql.types.mappers.AutoCompleteResultsMapper; import com.linkedin.datahub.graphql.types.mappers.BrowsePathsMapper; -import com.linkedin.datahub.graphql.types.mappers.BrowseResultMetadataMapper; +import com.linkedin.datahub.graphql.types.mappers.BrowseResultMapper; import com.linkedin.datahub.graphql.types.mappers.UrnSearchResultsMapper; import com.linkedin.datahub.graphql.types.mlmodel.mappers.MLModelGroupSnapshotMapper; import com.linkedin.entity.Entity; @@ -119,18 +119,7 @@ public BrowseResults browse(@Nonnull List path, facetFilters, start, count); - final List urns = result.getEntities().stream().map(entity -> entity.getUrn().toString()).collect(Collectors.toList()); - final List mlModelGroups = batchLoad(urns, context) - .stream().map(mlModelGroupDataFetcherResult -> mlModelGroupDataFetcherResult.getData()).collect(Collectors.toList()); - final BrowseResults browseResults = new BrowseResults(); - browseResults.setStart(result.getFrom()); - browseResults.setCount(result.getPageSize()); - browseResults.setTotal(result.getNumEntities()); - browseResults.setMetadata(BrowseResultMetadataMapper.map(result.getMetadata())); - browseResults.setEntities(mlModelGroups.stream() - .map(entity -> (com.linkedin.datahub.graphql.generated.Entity) entity) - .collect(Collectors.toList())); - return browseResults; + return BrowseResultMapper.map(result); } @Override diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLModelType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLModelType.java index 066fc250002f08..53dfb2ab12cf9d 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLModelType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLModelType.java @@ -2,6 +2,7 @@ import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.types.mappers.BrowseResultMapper; import com.linkedin.datahub.graphql.types.mappers.UrnSearchResultsMapper; import com.linkedin.datahub.graphql.types.mlmodel.mappers.MLModelSnapshotMapper; import com.linkedin.entity.client.EntityClient; @@ -33,7 +34,6 @@ import com.linkedin.datahub.graphql.resolvers.ResolverUtils; import com.linkedin.datahub.graphql.types.BrowsableEntityType; import com.linkedin.datahub.graphql.types.mappers.BrowsePathsMapper; -import com.linkedin.datahub.graphql.types.mappers.BrowseResultMetadataMapper; import com.linkedin.datahub.graphql.types.SearchableEntityType; import com.linkedin.datahub.graphql.types.mappers.AutoCompleteResultsMapper; import com.linkedin.metadata.query.AutoCompleteResult; @@ -121,18 +121,7 @@ public BrowseResults browse(@Nonnull List path, facetFilters, start, count); - final List urns = result.getEntities().stream().map(entity -> entity.getUrn().toString()).collect(Collectors.toList()); - final List mlModels = batchLoad(urns, context) - .stream().map(mlModelDataFetcherResult -> mlModelDataFetcherResult.getData()).collect(Collectors.toList()); - final BrowseResults browseResults = new BrowseResults(); - browseResults.setStart(result.getFrom()); - browseResults.setCount(result.getPageSize()); - browseResults.setTotal(result.getNumEntities()); - browseResults.setMetadata(BrowseResultMetadataMapper.map(result.getMetadata())); - browseResults.setEntities(mlModels.stream() - .map(entity -> (com.linkedin.datahub.graphql.generated.Entity) entity) - .collect(Collectors.toList())); - return browseResults; + return BrowseResultMapper.map(result); } @Override diff --git a/datahub-web-react/src/graphql-mock/fixtures/browsePathHelper.ts b/datahub-web-react/src/graphql-mock/fixtures/browsePathHelper.ts index 4864ea8a3536d2..412324af087312 100644 --- a/datahub-web-react/src/graphql-mock/fixtures/browsePathHelper.ts +++ b/datahub-web-react/src/graphql-mock/fixtures/browsePathHelper.ts @@ -45,12 +45,12 @@ export class BrowsePathResolver { data: { browse: { entities: [], + groups: [], start: 0, count: 0, total: 0, metadata: { path: [], - groups: [], totalNumEntities: 0, __typename: 'BrowseResultMetadata', }, @@ -102,13 +102,13 @@ export class BrowsePathResolver { browse: { ...dataBrowse, entities: chunkEntities, + groups, start: startValue, count: chunkEntities.length, total: entities.length, metadata: { ...dataBrowse.metadata, path: paths, - groups, totalNumEntities: groups.reduce(this.sumTotalEntityByGroups, 0), }, }, diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchServiceTest.java index 929710358135f8..06cc7efd8a773a 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchServiceTest.java @@ -102,7 +102,7 @@ public void testElasticSearchService() throws InterruptedException { assertEquals(searchResult.getEntities().get(0), urn); browseResult = _elasticSearchService.browse(ENTITY_NAME, "", null, 0, 10); assertEquals(browseResult.getMetadata().getTotalNumEntities().longValue(), 1); - assertEquals(browseResult.getMetadata().getGroups().get(0).getName(), "a"); + assertEquals(browseResult.getGroups().get(0).getName(), "a"); assertEquals(_elasticSearchService.docCount(ENTITY_NAME), 1); _elasticSearchService.deleteDocument(ENTITY_NAME, urn.toString()); From 9c70b0da0e17fb43d60e581d459052eb2cdb7d2b Mon Sep 17 00:00:00 2001 From: Dexter Lee Date: Thu, 29 Jul 2021 19:46:15 -0700 Subject: [PATCH 6/9] Fix models --- ...com.linkedin.entity.entities.restspec.json | 2 +- ...com.linkedin.entity.entities.snapshot.json | 74 +++++++++---------- .../metadata/resources/dashboard/Charts.java | 30 ++++++-- .../resources/dashboard/Dashboards.java | 4 +- .../metadata/resources/datajob/DataFlows.java | 44 +++++++---- .../metadata/resources/datajob/DataJobs.java | 50 ++++++++----- .../metadata/resources/dataset/Datasets.java | 34 ++++++--- .../resources/entity/EntityResource.java | 2 +- .../metadata/search/SearchService.java | 2 +- .../elasticsearch/ElasticSearchService.java | 22 +++--- .../elasticsearch/query/ESBrowseDAO.java | 12 +-- .../{query => browse}/BrowseResult.pdl | 2 +- .../{query => browse}/BrowseResultEntity.pdl | 2 +- .../BrowseResultMetadata.pdl | 2 +- metadata-utils/build.gradle | 1 + .../linkedin/metadata/utils/BrowseUtil.java | 34 +++++++++ 16 files changed, 204 insertions(+), 113 deletions(-) rename metadata-models/src/main/pegasus/com/linkedin/metadata/{query => browse}/BrowseResult.pdl (96%) rename metadata-models/src/main/pegasus/com/linkedin/metadata/{query => browse}/BrowseResultEntity.pdl (86%) rename metadata-models/src/main/pegasus/com/linkedin/metadata/{query => browse}/BrowseResultMetadata.pdl (85%) create mode 100644 metadata-utils/src/main/java/com/linkedin/metadata/utils/BrowseUtil.java diff --git a/gms/api/src/main/idl/com.linkedin.entity.entities.restspec.json b/gms/api/src/main/idl/com.linkedin.entity.entities.restspec.json index 1d66c1ba555eff..f08114729f14c7 100644 --- a/gms/api/src/main/idl/com.linkedin.entity.entities.restspec.json +++ b/gms/api/src/main/idl/com.linkedin.entity.entities.restspec.json @@ -79,7 +79,7 @@ "name" : "limit", "type" : "int" } ], - "returns" : "com.linkedin.metadata.query.BrowseResult" + "returns" : "com.linkedin.metadata.browse.BrowseResult" }, { "name" : "getBrowsePaths", "parameters" : [ { diff --git a/gms/api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json b/gms/api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json index c9de8f5dac92d7..3a2c3a7d5b87e1 100644 --- a/gms/api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json +++ b/gms/api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json @@ -4030,43 +4030,10 @@ } ] } } ] - }, "com.linkedin.glossary.GlossaryNodeInfo", "com.linkedin.glossary.GlossaryTermInfo", "com.linkedin.identity.CorpGroupInfo", "com.linkedin.identity.CorpUserEditableInfo", "com.linkedin.identity.CorpUserInfo", "com.linkedin.metadata.aspect.ChartAspect", "com.linkedin.metadata.aspect.CorpGroupAspect", "com.linkedin.metadata.aspect.CorpUserAspect", "com.linkedin.metadata.aspect.DashboardAspect", "com.linkedin.metadata.aspect.DataFlowAspect", "com.linkedin.metadata.aspect.DataJobAspect", "com.linkedin.metadata.aspect.DataPlatformAspect", "com.linkedin.metadata.aspect.DataProcessAspect", "com.linkedin.metadata.aspect.DatasetAspect", "com.linkedin.metadata.aspect.GlossaryNodeAspect", "com.linkedin.metadata.aspect.GlossaryTermAspect", "com.linkedin.metadata.aspect.MLFeatureAspect", "com.linkedin.metadata.aspect.MLFeatureTableAspect", "com.linkedin.metadata.aspect.MLModelAspect", "com.linkedin.metadata.aspect.MLModelDeploymentAspect", "com.linkedin.metadata.aspect.MLModelGroupAspect", "com.linkedin.metadata.aspect.MLPrimaryKeyAspect", "com.linkedin.metadata.aspect.TagAspect", "com.linkedin.metadata.key.ChartKey", "com.linkedin.metadata.key.CorpGroupKey", "com.linkedin.metadata.key.CorpUserKey", "com.linkedin.metadata.key.DashboardKey", "com.linkedin.metadata.key.DataFlowKey", "com.linkedin.metadata.key.DataJobKey", "com.linkedin.metadata.key.DataPlatformKey", "com.linkedin.metadata.key.DataProcessKey", "com.linkedin.metadata.key.DatasetKey", "com.linkedin.metadata.key.GlossaryNodeKey", "com.linkedin.metadata.key.GlossaryTermKey", "com.linkedin.metadata.key.MLFeatureKey", "com.linkedin.metadata.key.MLFeatureTableKey", "com.linkedin.metadata.key.MLModelDeploymentKey", "com.linkedin.metadata.key.MLModelGroupKey", "com.linkedin.metadata.key.MLModelKey", "com.linkedin.metadata.key.MLPrimaryKeyKey", "com.linkedin.metadata.key.TagKey", { - "type" : "record", - "name" : "AggregationMetadata", - "namespace" : "com.linkedin.metadata.query", - "fields" : [ { - "name" : "name", - "type" : "string", - "doc" : "The name of the aggregation, e.g, platform, origin" - }, { - "name" : "aggregations", - "type" : { - "type" : "map", - "values" : "long" - }, - "doc" : "List of aggregations showing the number of documents falling into each bucket. e.g, for platform aggregation, the bucket can be hive, kafka, etc" - } ] - }, { - "type" : "record", - "name" : "AutoCompleteResult", - "namespace" : "com.linkedin.metadata.query", - "doc" : "The model for the auto complete result", - "fields" : [ { - "name" : "query", - "type" : "string", - "doc" : "The original chars typed by user" - }, { - "name" : "suggestions", - "type" : { - "type" : "array", - "items" : "string" - }, - "doc" : "A list of typeahead suggestions" - } ] - }, { + }, "com.linkedin.glossary.GlossaryNodeInfo", "com.linkedin.glossary.GlossaryTermInfo", "com.linkedin.identity.CorpGroupInfo", "com.linkedin.identity.CorpUserEditableInfo", "com.linkedin.identity.CorpUserInfo", "com.linkedin.metadata.aspect.ChartAspect", "com.linkedin.metadata.aspect.CorpGroupAspect", "com.linkedin.metadata.aspect.CorpUserAspect", "com.linkedin.metadata.aspect.DashboardAspect", "com.linkedin.metadata.aspect.DataFlowAspect", "com.linkedin.metadata.aspect.DataJobAspect", "com.linkedin.metadata.aspect.DataPlatformAspect", "com.linkedin.metadata.aspect.DataProcessAspect", "com.linkedin.metadata.aspect.DatasetAspect", "com.linkedin.metadata.aspect.GlossaryNodeAspect", "com.linkedin.metadata.aspect.GlossaryTermAspect", "com.linkedin.metadata.aspect.MLFeatureAspect", "com.linkedin.metadata.aspect.MLFeatureTableAspect", "com.linkedin.metadata.aspect.MLModelAspect", "com.linkedin.metadata.aspect.MLModelDeploymentAspect", "com.linkedin.metadata.aspect.MLModelGroupAspect", "com.linkedin.metadata.aspect.MLPrimaryKeyAspect", "com.linkedin.metadata.aspect.TagAspect", { "type" : "record", "name" : "BrowseResult", - "namespace" : "com.linkedin.metadata.query", + "namespace" : "com.linkedin.metadata.browse", "doc" : "The model for the result of a browse query", "fields" : [ { "name" : "entities", @@ -4147,7 +4114,40 @@ "type" : "int", "doc" : "The total number of elements (entities + groups) directly under queried path" } ] - }, "com.linkedin.metadata.query.BrowseResultEntity", "com.linkedin.metadata.query.BrowseResultGroup", "com.linkedin.metadata.query.BrowseResultMetadata", { + }, "com.linkedin.metadata.browse.BrowseResultEntity", "com.linkedin.metadata.browse.BrowseResultGroup", "com.linkedin.metadata.browse.BrowseResultMetadata", "com.linkedin.metadata.key.ChartKey", "com.linkedin.metadata.key.CorpGroupKey", "com.linkedin.metadata.key.CorpUserKey", "com.linkedin.metadata.key.DashboardKey", "com.linkedin.metadata.key.DataFlowKey", "com.linkedin.metadata.key.DataJobKey", "com.linkedin.metadata.key.DataPlatformKey", "com.linkedin.metadata.key.DataProcessKey", "com.linkedin.metadata.key.DatasetKey", "com.linkedin.metadata.key.GlossaryNodeKey", "com.linkedin.metadata.key.GlossaryTermKey", "com.linkedin.metadata.key.MLFeatureKey", "com.linkedin.metadata.key.MLFeatureTableKey", "com.linkedin.metadata.key.MLModelDeploymentKey", "com.linkedin.metadata.key.MLModelGroupKey", "com.linkedin.metadata.key.MLModelKey", "com.linkedin.metadata.key.MLPrimaryKeyKey", "com.linkedin.metadata.key.TagKey", { + "type" : "record", + "name" : "AggregationMetadata", + "namespace" : "com.linkedin.metadata.query", + "fields" : [ { + "name" : "name", + "type" : "string", + "doc" : "The name of the aggregation, e.g, platform, origin" + }, { + "name" : "aggregations", + "type" : { + "type" : "map", + "values" : "long" + }, + "doc" : "List of aggregations showing the number of documents falling into each bucket. e.g, for platform aggregation, the bucket can be hive, kafka, etc" + } ] + }, { + "type" : "record", + "name" : "AutoCompleteResult", + "namespace" : "com.linkedin.metadata.query", + "doc" : "The model for the auto complete result", + "fields" : [ { + "name" : "query", + "type" : "string", + "doc" : "The original chars typed by user" + }, { + "name" : "suggestions", + "type" : { + "type" : "array", + "items" : "string" + }, + "doc" : "A list of typeahead suggestions" + } ] + }, { "type" : "enum", "name" : "Condition", "namespace" : "com.linkedin.metadata.query", @@ -4380,7 +4380,7 @@ "name" : "limit", "type" : "int" } ], - "returns" : "com.linkedin.metadata.query.BrowseResult" + "returns" : "com.linkedin.metadata.browse.BrowseResult" }, { "name" : "getBrowsePaths", "parameters" : [ { diff --git a/gms/impl/src/main/java/com/linkedin/metadata/resources/dashboard/Charts.java b/gms/impl/src/main/java/com/linkedin/metadata/resources/dashboard/Charts.java index 4972dac0b702a3..e0dc73d646e87f 100644 --- a/gms/impl/src/main/java/com/linkedin/metadata/resources/dashboard/Charts.java +++ b/gms/impl/src/main/java/com/linkedin/metadata/resources/dashboard/Charts.java @@ -35,6 +35,7 @@ import com.linkedin.metadata.search.SearchService; import com.linkedin.metadata.snapshot.ChartSnapshot; import com.linkedin.metadata.snapshot.Snapshot; +import com.linkedin.metadata.utils.BrowseUtil; import com.linkedin.parseq.Task; import com.linkedin.restli.common.ComplexResourceKey; import com.linkedin.restli.common.EmptyRecord; @@ -64,7 +65,25 @@ import javax.inject.Inject; import javax.inject.Named; -import static com.linkedin.metadata.restli.RestliConstants.*; +import static com.linkedin.metadata.restli.RestliConstants.ACTION_AUTOCOMPLETE; +import static com.linkedin.metadata.restli.RestliConstants.ACTION_BACKFILL_WITH_URNS; +import static com.linkedin.metadata.restli.RestliConstants.ACTION_BROWSE; +import static com.linkedin.metadata.restli.RestliConstants.ACTION_GET_BROWSE_PATHS; +import static com.linkedin.metadata.restli.RestliConstants.ACTION_GET_SNAPSHOT; +import static com.linkedin.metadata.restli.RestliConstants.ACTION_INGEST; +import static com.linkedin.metadata.restli.RestliConstants.FINDER_SEARCH; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_ASPECTS; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_FIELD; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_FILTER; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_INPUT; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_LIMIT; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_PATH; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_QUERY; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_SNAPSHOT; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_SORT; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_START; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_URN; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_URNS; /** @@ -318,13 +337,8 @@ public Task autocomplete(@ActionParam(PARAM_QUERY) @Nonnull public Task browse(@ActionParam(PARAM_PATH) @Nonnull String path, @ActionParam(PARAM_FILTER) @Optional @Nullable Filter filter, @ActionParam(PARAM_START) int start, @ActionParam(PARAM_LIMIT) int limit) { - return RestliUtils.toTask(() -> - _searchService.browse( - "chart", - path, - filter, - start, - limit) + return RestliUtils.toTask( + () -> BrowseUtil.convertToLegacyResult(_searchService.browse("chart", path, filter, start, limit)) ); } diff --git a/gms/impl/src/main/java/com/linkedin/metadata/resources/dashboard/Dashboards.java b/gms/impl/src/main/java/com/linkedin/metadata/resources/dashboard/Dashboards.java index 49488dddb19ce7..54a0442bd87819 100644 --- a/gms/impl/src/main/java/com/linkedin/metadata/resources/dashboard/Dashboards.java +++ b/gms/impl/src/main/java/com/linkedin/metadata/resources/dashboard/Dashboards.java @@ -34,6 +34,7 @@ import com.linkedin.metadata.search.SearchService; import com.linkedin.metadata.snapshot.DashboardSnapshot; import com.linkedin.metadata.snapshot.Snapshot; +import com.linkedin.metadata.utils.BrowseUtil; import com.linkedin.parseq.Task; import com.linkedin.restli.common.ComplexResourceKey; import com.linkedin.restli.common.EmptyRecord; @@ -319,7 +320,8 @@ public Task autocomplete(@ActionParam(PARAM_QUERY) @Nonnull public Task browse(@ActionParam(PARAM_PATH) @Nonnull String path, @ActionParam(PARAM_FILTER) @Optional @Nullable Filter filter, @ActionParam(PARAM_START) int start, @ActionParam(PARAM_LIMIT) int limit) { - return RestliUtils.toTask(() -> _searchService.browse("dashboard", path, filter, start, limit)); + return RestliUtils.toTask( + () -> BrowseUtil.convertToLegacyResult(_searchService.browse("dashboard", path, filter, start, limit))); } @Action(name = ACTION_GET_BROWSE_PATHS) diff --git a/gms/impl/src/main/java/com/linkedin/metadata/resources/datajob/DataFlows.java b/gms/impl/src/main/java/com/linkedin/metadata/resources/datajob/DataFlows.java index e6ebd0188ffe83..7ae0ff5c63570b 100644 --- a/gms/impl/src/main/java/com/linkedin/metadata/resources/datajob/DataFlows.java +++ b/gms/impl/src/main/java/com/linkedin/metadata/resources/datajob/DataFlows.java @@ -1,26 +1,25 @@ package com.linkedin.metadata.resources.datajob; import com.linkedin.common.AuditStamp; - import com.linkedin.common.GlobalTags; +import com.linkedin.common.Ownership; import com.linkedin.common.Status; import com.linkedin.common.UrnArray; -import com.linkedin.datajob.DataFlowInfo; -import com.linkedin.common.Ownership; import com.linkedin.common.urn.DataFlowUrn; import com.linkedin.common.urn.Urn; +import com.linkedin.data.template.StringArray; import com.linkedin.datajob.DataFlow; +import com.linkedin.datajob.DataFlowInfo; import com.linkedin.datajob.DataFlowKey; -import com.linkedin.data.template.StringArray; import com.linkedin.entity.Entity; import com.linkedin.metadata.PegasusUtils; import com.linkedin.metadata.aspect.DataFlowAspect; import com.linkedin.metadata.dao.BaseBrowseDAO; import com.linkedin.metadata.dao.BaseLocalDAO; import com.linkedin.metadata.dao.BaseSearchDAO; -import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.dao.utils.ModelUtils; import com.linkedin.metadata.dao.utils.QueryUtils; +import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.query.AutoCompleteResult; import com.linkedin.metadata.query.BrowseResult; import com.linkedin.metadata.query.Filter; @@ -35,6 +34,7 @@ import com.linkedin.metadata.search.SearchService; import com.linkedin.metadata.snapshot.DataFlowSnapshot; import com.linkedin.metadata.snapshot.Snapshot; +import com.linkedin.metadata.utils.BrowseUtil; import com.linkedin.parseq.Task; import com.linkedin.restli.common.ComplexResourceKey; import com.linkedin.restli.common.EmptyRecord; @@ -64,7 +64,25 @@ import javax.inject.Inject; import javax.inject.Named; -import static com.linkedin.metadata.restli.RestliConstants.*; +import static com.linkedin.metadata.restli.RestliConstants.ACTION_AUTOCOMPLETE; +import static com.linkedin.metadata.restli.RestliConstants.ACTION_BACKFILL_WITH_URNS; +import static com.linkedin.metadata.restli.RestliConstants.ACTION_BROWSE; +import static com.linkedin.metadata.restli.RestliConstants.ACTION_GET_BROWSE_PATHS; +import static com.linkedin.metadata.restli.RestliConstants.ACTION_GET_SNAPSHOT; +import static com.linkedin.metadata.restli.RestliConstants.ACTION_INGEST; +import static com.linkedin.metadata.restli.RestliConstants.FINDER_SEARCH; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_ASPECTS; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_FIELD; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_FILTER; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_INPUT; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_LIMIT; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_PATH; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_QUERY; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_SNAPSHOT; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_SORT; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_START; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_URN; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_URNS; /** * Deprecated! Use {@link EntityResource} instead. @@ -314,14 +332,9 @@ public Task autocomplete(@ActionParam(PARAM_QUERY) @Nonnull public Task browse(@ActionParam(PARAM_PATH) @Nonnull String path, @ActionParam(PARAM_FILTER) @Optional @Nullable Filter filter, @ActionParam(PARAM_START) int start, @ActionParam(PARAM_LIMIT) int limit) { - return RestliUtils.toTask(() -> - _searchService.browse( - "dataFlow", - path, - filter, - start, - limit) - ); } + return RestliUtils.toTask( + () -> BrowseUtil.convertToLegacyResult(_searchService.browse("dataFlow", path, filter, start, limit))); + } @Action(name = ACTION_GET_BROWSE_PATHS) @Override @@ -332,7 +345,8 @@ public Task getBrowsePaths( new StringArray(_searchService.getBrowsePaths( "dataFlow", urn)) - ); } + ); + } @Action(name = ACTION_INGEST) @Override diff --git a/gms/impl/src/main/java/com/linkedin/metadata/resources/datajob/DataJobs.java b/gms/impl/src/main/java/com/linkedin/metadata/resources/datajob/DataJobs.java index 2930d7ba2ce563..15988bfd935b81 100644 --- a/gms/impl/src/main/java/com/linkedin/metadata/resources/datajob/DataJobs.java +++ b/gms/impl/src/main/java/com/linkedin/metadata/resources/datajob/DataJobs.java @@ -1,27 +1,26 @@ package com.linkedin.metadata.resources.datajob; import com.linkedin.common.AuditStamp; - import com.linkedin.common.GlobalTags; +import com.linkedin.common.Ownership; import com.linkedin.common.Status; import com.linkedin.common.UrnArray; -import com.linkedin.datajob.DataJobInfo; -import com.linkedin.datajob.DataJobInputOutput; -import com.linkedin.common.Ownership; import com.linkedin.common.urn.DataJobUrn; import com.linkedin.common.urn.Urn; +import com.linkedin.data.template.StringArray; import com.linkedin.datajob.DataJob; +import com.linkedin.datajob.DataJobInfo; +import com.linkedin.datajob.DataJobInputOutput; import com.linkedin.datajob.DataJobKey; -import com.linkedin.data.template.StringArray; import com.linkedin.entity.Entity; import com.linkedin.metadata.PegasusUtils; import com.linkedin.metadata.aspect.DataJobAspect; import com.linkedin.metadata.dao.BaseBrowseDAO; import com.linkedin.metadata.dao.BaseLocalDAO; import com.linkedin.metadata.dao.BaseSearchDAO; -import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.dao.utils.ModelUtils; import com.linkedin.metadata.dao.utils.QueryUtils; +import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.query.AutoCompleteResult; import com.linkedin.metadata.query.BrowseResult; import com.linkedin.metadata.query.Filter; @@ -36,6 +35,7 @@ import com.linkedin.metadata.search.SearchService; import com.linkedin.metadata.snapshot.DataJobSnapshot; import com.linkedin.metadata.snapshot.Snapshot; +import com.linkedin.metadata.utils.BrowseUtil; import com.linkedin.parseq.Task; import com.linkedin.restli.common.ComplexResourceKey; import com.linkedin.restli.common.EmptyRecord; @@ -65,7 +65,25 @@ import javax.inject.Inject; import javax.inject.Named; -import static com.linkedin.metadata.restli.RestliConstants.*; +import static com.linkedin.metadata.restli.RestliConstants.ACTION_AUTOCOMPLETE; +import static com.linkedin.metadata.restli.RestliConstants.ACTION_BACKFILL_WITH_URNS; +import static com.linkedin.metadata.restli.RestliConstants.ACTION_BROWSE; +import static com.linkedin.metadata.restli.RestliConstants.ACTION_GET_BROWSE_PATHS; +import static com.linkedin.metadata.restli.RestliConstants.ACTION_GET_SNAPSHOT; +import static com.linkedin.metadata.restli.RestliConstants.ACTION_INGEST; +import static com.linkedin.metadata.restli.RestliConstants.FINDER_SEARCH; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_ASPECTS; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_FIELD; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_FILTER; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_INPUT; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_LIMIT; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_PATH; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_QUERY; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_SNAPSHOT; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_SORT; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_START; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_URN; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_URNS; /** * Deprecated! Use {@link EntityResource} instead. @@ -317,25 +335,17 @@ public Task autocomplete(@ActionParam(PARAM_QUERY) @Nonnull public Task browse(@ActionParam(PARAM_PATH) @Nonnull String path, @ActionParam(PARAM_FILTER) @Optional @Nullable Filter filter, @ActionParam(PARAM_START) int start, @ActionParam(PARAM_LIMIT) int limit) { - return RestliUtils.toTask(() -> - _searchService.browse( - "dataJob", - path, - filter, - start, - limit) - ); } + return RestliUtils.toTask( + () -> BrowseUtil.convertToLegacyResult(_searchService.browse("dataJob", path, filter, start, limit))); + } @Action(name = ACTION_GET_BROWSE_PATHS) @Override @Nonnull public Task getBrowsePaths( @ActionParam(value = "urn", typeref = com.linkedin.common.Urn.class) @Nonnull Urn urn) { - return RestliUtils.toTask(() -> - new StringArray(_searchService.getBrowsePaths( - "dataJob", - urn)) - ); } + return RestliUtils.toTask(() -> new StringArray(_searchService.getBrowsePaths("dataJob", urn))); + } @Action(name = ACTION_INGEST) @Override diff --git a/gms/impl/src/main/java/com/linkedin/metadata/resources/dataset/Datasets.java b/gms/impl/src/main/java/com/linkedin/metadata/resources/dataset/Datasets.java index 85c79cdffbcf08..5b7b877e015ebf 100644 --- a/gms/impl/src/main/java/com/linkedin/metadata/resources/dataset/Datasets.java +++ b/gms/impl/src/main/java/com/linkedin/metadata/resources/dataset/Datasets.java @@ -2,6 +2,7 @@ import com.linkedin.common.AuditStamp; import com.linkedin.common.GlobalTags; +import com.linkedin.common.GlossaryTerms; import com.linkedin.common.InstitutionalMemory; import com.linkedin.common.Ownership; import com.linkedin.common.Status; @@ -11,7 +12,6 @@ import com.linkedin.data.template.StringArray; import com.linkedin.dataset.Dataset; import com.linkedin.dataset.DatasetDeprecation; -import com.linkedin.common.GlossaryTerms; import com.linkedin.dataset.DatasetKey; import com.linkedin.dataset.DatasetProperties; import com.linkedin.dataset.EditableDatasetProperties; @@ -22,8 +22,8 @@ import com.linkedin.metadata.dao.BaseBrowseDAO; import com.linkedin.metadata.dao.BaseLocalDAO; import com.linkedin.metadata.dao.BaseSearchDAO; -import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.dao.utils.ModelUtils; +import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.query.AutoCompleteResult; import com.linkedin.metadata.query.BrowseResult; import com.linkedin.metadata.query.Filter; @@ -38,6 +38,7 @@ import com.linkedin.metadata.search.SearchService; import com.linkedin.metadata.snapshot.DatasetSnapshot; import com.linkedin.metadata.snapshot.Snapshot; +import com.linkedin.metadata.utils.BrowseUtil; import com.linkedin.parseq.Task; import com.linkedin.restli.common.ComplexResourceKey; import com.linkedin.restli.common.EmptyRecord; @@ -69,7 +70,25 @@ import javax.inject.Inject; import javax.inject.Named; -import static com.linkedin.metadata.restli.RestliConstants.*; +import static com.linkedin.metadata.restli.RestliConstants.ACTION_AUTOCOMPLETE; +import static com.linkedin.metadata.restli.RestliConstants.ACTION_BACKFILL; +import static com.linkedin.metadata.restli.RestliConstants.ACTION_BROWSE; +import static com.linkedin.metadata.restli.RestliConstants.ACTION_GET_BROWSE_PATHS; +import static com.linkedin.metadata.restli.RestliConstants.ACTION_GET_SNAPSHOT; +import static com.linkedin.metadata.restli.RestliConstants.ACTION_INGEST; +import static com.linkedin.metadata.restli.RestliConstants.FINDER_FILTER; +import static com.linkedin.metadata.restli.RestliConstants.FINDER_SEARCH; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_ASPECTS; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_FIELD; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_FILTER; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_INPUT; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_LIMIT; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_PATH; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_QUERY; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_SNAPSHOT; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_SORT; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_START; +import static com.linkedin.metadata.restli.RestliConstants.PARAM_URN; /** * Deprecated! Use {@link EntityResource} instead. @@ -373,13 +392,8 @@ public Task autocomplete(@ActionParam(PARAM_QUERY) @Nonnull public Task browse(@ActionParam(PARAM_PATH) @Nonnull String path, @ActionParam(PARAM_FILTER) @Optional @Nullable Filter filter, @ActionParam(PARAM_START) int start, @ActionParam(PARAM_LIMIT) int limit) { - return RestliUtils.toTask(() -> - _searchService.browse( - "dataset", - path, - filter, - start, - limit) + return RestliUtils.toTask( + () -> BrowseUtil.convertToLegacyResult(_searchService.browse("dataset", path, filter, start, limit)) ); } diff --git a/gms/impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java b/gms/impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java index b170a19bddbcf2..095262d369784a 100644 --- a/gms/impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java +++ b/gms/impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java @@ -6,10 +6,10 @@ import com.linkedin.data.template.RecordTemplate; import com.linkedin.data.template.StringArray; import com.linkedin.entity.Entity; +import com.linkedin.metadata.browse.BrowseResult; import com.linkedin.metadata.dao.utils.RecordUtils; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.query.AutoCompleteResult; -import com.linkedin.metadata.query.BrowseResult; import com.linkedin.metadata.query.Filter; import com.linkedin.metadata.query.SearchResult; import com.linkedin.metadata.query.SortCriterion; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/SearchService.java b/metadata-io/src/main/java/com/linkedin/metadata/search/SearchService.java index bb7ef91cfca1b5..857aadba9ca9b9 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/SearchService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/SearchService.java @@ -1,8 +1,8 @@ package com.linkedin.metadata.search; import com.linkedin.common.urn.Urn; +import com.linkedin.metadata.browse.BrowseResult; import com.linkedin.metadata.query.AutoCompleteResult; -import com.linkedin.metadata.query.BrowseResult; import com.linkedin.metadata.query.Filter; import com.linkedin.metadata.query.SearchResult; import com.linkedin.metadata.query.SortCriterion; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java index 32801ac5feb703..9d48717aa6b104 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchService.java @@ -1,8 +1,8 @@ package com.linkedin.metadata.search.elasticsearch; import com.linkedin.common.urn.Urn; +import com.linkedin.metadata.browse.BrowseResult; import com.linkedin.metadata.query.AutoCompleteResult; -import com.linkedin.metadata.query.BrowseResult; import com.linkedin.metadata.query.Filter; import com.linkedin.metadata.query.SearchResult; import com.linkedin.metadata.query.SortCriterion; @@ -44,8 +44,8 @@ public long docCount(@Nonnull String entityName) { @Override public void upsertDocument(@Nonnull String entityName, @Nonnull String document, @Nonnull String docId) { - log.debug(String.format("Upserting Search document entityName: %s, document: %s, docId: %s", entityName, - document, docId)); + log.debug(String.format("Upserting Search document entityName: %s, document: %s, docId: %s", entityName, document, + docId)); esWriteDAO.upsertDocument(entityName, document, docId); } @@ -59,7 +59,8 @@ public void deleteDocument(@Nonnull String entityName, @Nonnull String docId) { @Override public SearchResult search(@Nonnull String entityName, @Nonnull String input, @Nullable Filter postFilters, @Nullable SortCriterion sortCriterion, int from, int size) { - log.debug(String.format("Searching Search documents entityName: %s, input: %s, postFilters: %s, sortCriterion: %s, from: %s, size: %s", + log.debug(String.format( + "Searching Search documents entityName: %s, input: %s, postFilters: %s, sortCriterion: %s, from: %s, size: %s", entityName, input, postFilters, sortCriterion, from, size)); return esSearchDAO.search(entityName, input, postFilters, sortCriterion, from, size); } @@ -68,8 +69,9 @@ public SearchResult search(@Nonnull String entityName, @Nonnull String input, @N @Override public SearchResult filter(@Nonnull String entityName, @Nullable Filter filters, @Nullable SortCriterion sortCriterion, int from, int size) { - log.debug(String.format("Filtering Search documents entityName: %s, filters: %s, sortCriterion: %s, from: %s, size: %s", - entityName, filters, sortCriterion, from, size)); + log.debug( + String.format("Filtering Search documents entityName: %s, filters: %s, sortCriterion: %s, from: %s, size: %s", + entityName, filters, sortCriterion, from, size)); return esSearchDAO.filter(entityName, filters, sortCriterion, from, size); } @@ -86,16 +88,16 @@ public AutoCompleteResult autoComplete(@Nonnull String entityName, @Nonnull Stri @Override public BrowseResult browse(@Nonnull String entityName, @Nonnull String path, @Nullable Filter requestParams, int from, int size) { - log.debug(String.format("Browsing entities entityName: %s, path: %s, requestParams: %s, from: %s, size: %s", - entityName, path, requestParams, from, size)); + log.debug( + String.format("Browsing entities entityName: %s, path: %s, requestParams: %s, from: %s, size: %s", entityName, + path, requestParams, from, size)); return esBrowseDAO.browse(entityName, path, requestParams, from, size); } @Nonnull @Override public List getBrowsePaths(@Nonnull String entityName, @Nonnull Urn urn) { - log.debug(String.format("Getting browse paths for entity entityName: %s, urn: %s", - entityName, urn)); + log.debug(String.format("Getting browse paths for entity entityName: %s, urn: %s", entityName, urn)); return esBrowseDAO.getBrowsePaths(entityName, urn); } } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java index 80ca420caf3880..64f275fcbedd0f 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/ESBrowseDAO.java @@ -6,12 +6,12 @@ import com.linkedin.metadata.dao.utils.ESUtils; import com.linkedin.metadata.dao.utils.SearchUtils; import com.linkedin.metadata.models.registry.EntityRegistry; -import com.linkedin.metadata.query.BrowseResult; -import com.linkedin.metadata.query.BrowseResultEntity; -import com.linkedin.metadata.query.BrowseResultEntityArray; -import com.linkedin.metadata.query.BrowseResultGroup; -import com.linkedin.metadata.query.BrowseResultGroupArray; -import com.linkedin.metadata.query.BrowseResultMetadata; +import com.linkedin.metadata.browse.BrowseResult; +import com.linkedin.metadata.browse.BrowseResultEntity; +import com.linkedin.metadata.browse.BrowseResultEntityArray; +import com.linkedin.metadata.browse.BrowseResultGroup; +import com.linkedin.metadata.browse.BrowseResultGroupArray; +import com.linkedin.metadata.browse.BrowseResultMetadata; import com.linkedin.metadata.query.Filter; import com.linkedin.metadata.utils.elasticsearch.IndexConvention; import java.net.URISyntaxException; diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/query/BrowseResult.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/browse/BrowseResult.pdl similarity index 96% rename from metadata-models/src/main/pegasus/com/linkedin/metadata/query/BrowseResult.pdl rename to metadata-models/src/main/pegasus/com/linkedin/metadata/browse/BrowseResult.pdl index 3eec897b0da67e..ee0087660a0df9 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/metadata/query/BrowseResult.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/metadata/browse/BrowseResult.pdl @@ -1,4 +1,4 @@ -namespace com.linkedin.metadata.query +namespace com.linkedin.metadata.browse /** * The model for the result of a browse query diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/query/BrowseResultEntity.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/browse/BrowseResultEntity.pdl similarity index 86% rename from metadata-models/src/main/pegasus/com/linkedin/metadata/query/BrowseResultEntity.pdl rename to metadata-models/src/main/pegasus/com/linkedin/metadata/browse/BrowseResultEntity.pdl index ca9ca0f18aa72e..2b9ef20a0caee5 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/metadata/query/BrowseResultEntity.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/metadata/browse/BrowseResultEntity.pdl @@ -1,4 +1,4 @@ -namespace com.linkedin.metadata.query +namespace com.linkedin.metadata.browse import com.linkedin.common.Urn diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/query/BrowseResultMetadata.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/browse/BrowseResultMetadata.pdl similarity index 85% rename from metadata-models/src/main/pegasus/com/linkedin/metadata/query/BrowseResultMetadata.pdl rename to metadata-models/src/main/pegasus/com/linkedin/metadata/browse/BrowseResultMetadata.pdl index 5db8fbd8ea4a45..617cf7b83b2ce4 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/metadata/query/BrowseResultMetadata.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/metadata/browse/BrowseResultMetadata.pdl @@ -1,4 +1,4 @@ -namespace com.linkedin.metadata.query +namespace com.linkedin.metadata.browse /** * The model for browse result metadata diff --git a/metadata-utils/build.gradle b/metadata-utils/build.gradle index 62218aff95249a..d6ef4aeda2a7d0 100644 --- a/metadata-utils/build.gradle +++ b/metadata-utils/build.gradle @@ -5,6 +5,7 @@ dependencies { compile externalDependency.commonsLang compile externalDependency.elasticSearchRest compile externalDependency.gmaDaoApi + compile externalDependency.gmaDaoApiDataTemplate compile externalDependency.httpClient compile externalDependency.neo4jJavaDriver diff --git a/metadata-utils/src/main/java/com/linkedin/metadata/utils/BrowseUtil.java b/metadata-utils/src/main/java/com/linkedin/metadata/utils/BrowseUtil.java new file mode 100644 index 00000000000000..6489f1ed39a78f --- /dev/null +++ b/metadata-utils/src/main/java/com/linkedin/metadata/utils/BrowseUtil.java @@ -0,0 +1,34 @@ +package com.linkedin.metadata.utils; + +import com.linkedin.metadata.browse.BrowseResult; +import com.linkedin.metadata.query.BrowseResultEntity; +import com.linkedin.metadata.query.BrowseResultEntityArray; +import com.linkedin.metadata.query.BrowseResultGroup; +import com.linkedin.metadata.query.BrowseResultGroupArray; +import com.linkedin.metadata.query.BrowseResultMetadata; +import java.util.stream.Collectors; + + +public class BrowseUtil { + public static com.linkedin.metadata.query.BrowseResult convertToLegacyResult(BrowseResult browseResult) { + com.linkedin.metadata.query.BrowseResult legacyResult = new com.linkedin.metadata.query.BrowseResult(); + + legacyResult.setFrom(browseResult.getFrom()); + legacyResult.setPageSize(browseResult.getPageSize()); + legacyResult.setNumEntities(browseResult.getNumEntities()); + legacyResult.setEntities(new BrowseResultEntityArray(browseResult.getEntities() + .stream() + .map(entity -> new BrowseResultEntity(entity.data())) + .collect(Collectors.toList()))); + + BrowseResultMetadata legacyMetadata = new BrowseResultMetadata(); + legacyMetadata.setGroups(new BrowseResultGroupArray(browseResult.getGroups() + .stream() + .map(group -> new BrowseResultGroup(group.data())) + .collect(Collectors.toList()))); + legacyMetadata.setPath(browseResult.getMetadata().getPath()); + legacyMetadata.setTotalNumEntities(browseResult.getMetadata().getTotalNumEntities()); + + return legacyResult; + } +} From 983d7fc8385d4d264513a7307f0a13f7bc3d2ed1 Mon Sep 17 00:00:00 2001 From: Dexter Lee Date: Thu, 29 Jul 2021 20:29:11 -0700 Subject: [PATCH 7/9] Fix --- .../datahub/graphql/GmsGraphQLEngine.java | 9 ++++ .../load/EntityTypeBatchResolver.java | 51 +++++++++++++++++++ .../graphql/types/chart/ChartType.java | 2 +- .../types/dashboard/DashboardType.java | 2 +- .../graphql/types/dataflow/DataFlowType.java | 2 +- .../graphql/types/datajob/DataJobType.java | 2 +- .../graphql/types/dataset/DatasetType.java | 2 +- .../types/glossary/GlossaryTermType.java | 2 +- .../types/mappers/BrowseResultMapper.java | 4 +- .../types/mlmodel/MLFeatureTableType.java | 2 +- .../types/mlmodel/MLModelGroupType.java | 2 +- .../graphql/types/mlmodel/MLModelType.java | 2 +- .../linkedin/entity/client/EntityClient.java | 2 +- 13 files changed, 72 insertions(+), 12 deletions(-) create mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityTypeBatchResolver.java diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index 58a18159fbf234..313846e0ae5b39 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -2,6 +2,7 @@ import com.google.common.collect.ImmutableList; import com.linkedin.datahub.graphql.generated.Aspect; +import com.linkedin.datahub.graphql.generated.BrowseResults; import com.linkedin.datahub.graphql.generated.Chart; import com.linkedin.datahub.graphql.generated.ChartInfo; import com.linkedin.datahub.graphql.generated.DashboardInfo; @@ -29,6 +30,7 @@ import com.linkedin.datahub.graphql.generated.MLPrimaryKey; import com.linkedin.datahub.graphql.generated.MLPrimaryKeyProperties; import com.linkedin.datahub.graphql.resolvers.load.AspectResolver; +import com.linkedin.datahub.graphql.resolvers.load.EntityTypeBatchResolver; import com.linkedin.datahub.graphql.resolvers.load.EntityTypeResolver; import com.linkedin.datahub.graphql.resolvers.load.LoadableTypeBatchResolver; import com.linkedin.datahub.graphql.resolvers.load.UsageTypeResolver; @@ -391,6 +393,13 @@ private static void configureDatasetResolvers(final RuntimeWiring.Builder builde (env) -> ((SearchResult) env.getSource()).getEntity())) ) ) + .type("BrowseResults", typeWiring -> typeWiring + .dataFetcher("entities", new AuthenticatedResolver<>( + new EntityTypeBatchResolver( + ENTITY_TYPES.stream().collect(Collectors.toList()), + (env) -> ((BrowseResults) env.getSource()).getEntities())) + ) + ) .type("InstitutionalMemoryMetadata", typeWiring -> typeWiring .dataFetcher("author", new AuthenticatedResolver<>( new LoadableTypeResolver<>( diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityTypeBatchResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityTypeBatchResolver.java new file mode 100644 index 00000000000000..5c98190b736ba6 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityTypeBatchResolver.java @@ -0,0 +1,51 @@ +package com.linkedin.datahub.graphql.resolvers.load; + +import com.google.common.collect.Iterables; +import com.linkedin.datahub.graphql.generated.Entity; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.dataloader.DataLoader; + + +/** + * GraphQL resolver responsible for + * + * 1. Retrieving a single input urn. + * 2. Resolving a single Entity + * + * + */ +public class EntityTypeBatchResolver implements DataFetcher>> { + + private final List> _entityTypes; + private final Function> _entitiesProvider; + + public EntityTypeBatchResolver( + final List> entityTypes, + final Function> entitiesProvider + ) { + _entityTypes = entityTypes; + _entitiesProvider = entitiesProvider; + } + + @Override + public CompletableFuture get(DataFetchingEnvironment environment) { + final List entities = _entitiesProvider.apply(environment); + if (entities.isEmpty()) { + return CompletableFuture.completedFuture(Collections.emptyList()); + } + // Assume all entities are of the same type + final com.linkedin.datahub.graphql.types.EntityType filteredEntity = + Iterables.getOnlyElement(_entityTypes.stream() + .filter(entity -> entities.get(0).getClass().isAssignableFrom(entity.objectClass())) + .collect(Collectors.toList())); + + final DataLoader loader = environment.getDataLoaderRegistry().getDataLoader(filteredEntity.name()); + return loader.loadMany(entities.stream().map(Entity::getUrn).collect(Collectors.toList())); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/ChartType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/ChartType.java index 30d33f558b9cb0..bf6631ba469b1d 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/ChartType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/chart/ChartType.java @@ -26,8 +26,8 @@ import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.configs.ChartSearchConfig; import com.linkedin.metadata.extractor.SnapshotToAspectMap; +import com.linkedin.metadata.browse.BrowseResult; import com.linkedin.metadata.query.AutoCompleteResult; -import com.linkedin.metadata.query.BrowseResult; import com.linkedin.metadata.query.SearchResult; import com.linkedin.metadata.snapshot.ChartSnapshot; import com.linkedin.metadata.snapshot.Snapshot; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java index bad7b894a797e2..2891bfd55ce0ee 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/DashboardType.java @@ -28,8 +28,8 @@ import com.linkedin.entity.Entity; import com.linkedin.metadata.configs.DashboardSearchConfig; import com.linkedin.metadata.extractor.SnapshotToAspectMap; +import com.linkedin.metadata.browse.BrowseResult; import com.linkedin.metadata.query.AutoCompleteResult; -import com.linkedin.metadata.query.BrowseResult; import com.linkedin.metadata.query.SearchResult; import com.linkedin.metadata.snapshot.DashboardSnapshot; import com.linkedin.metadata.snapshot.Snapshot; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/DataFlowType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/DataFlowType.java index 50ea371a16e9b3..6331864d6d7c01 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/DataFlowType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/DataFlowType.java @@ -30,8 +30,8 @@ import com.linkedin.metadata.aspect.DataFlowAspect; import com.linkedin.metadata.dao.utils.ModelUtils; import com.linkedin.metadata.extractor.SnapshotToAspectMap; +import com.linkedin.metadata.browse.BrowseResult; import com.linkedin.metadata.query.AutoCompleteResult; -import com.linkedin.metadata.query.BrowseResult; import com.linkedin.metadata.query.SearchResult; import com.linkedin.metadata.snapshot.DataFlowSnapshot; import com.linkedin.metadata.snapshot.Snapshot; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/DataJobType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/DataJobType.java index 808c812a9ff426..1e0556b4b991cc 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/DataJobType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/DataJobType.java @@ -30,8 +30,8 @@ import com.linkedin.metadata.aspect.DataJobAspect; import com.linkedin.metadata.dao.utils.ModelUtils; import com.linkedin.metadata.extractor.SnapshotToAspectMap; +import com.linkedin.metadata.browse.BrowseResult; import com.linkedin.metadata.query.AutoCompleteResult; -import com.linkedin.metadata.query.BrowseResult; import com.linkedin.metadata.query.SearchResult; import com.linkedin.metadata.snapshot.DataJobSnapshot; import com.linkedin.metadata.snapshot.Snapshot; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java index 44491aa3654486..640b6bc9fd9ca6 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java @@ -30,8 +30,8 @@ import com.linkedin.entity.client.EntityClient; import com.linkedin.entity.Entity; import com.linkedin.metadata.extractor.SnapshotToAspectMap; +import com.linkedin.metadata.browse.BrowseResult; import com.linkedin.metadata.query.AutoCompleteResult; -import com.linkedin.metadata.query.BrowseResult; import com.linkedin.metadata.query.SearchResult; import com.linkedin.metadata.snapshot.Snapshot; import com.linkedin.r2.RemoteInvocationException; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/GlossaryTermType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/GlossaryTermType.java index f228fa06553d45..24d8d9630cb03a 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/GlossaryTermType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/GlossaryTermType.java @@ -23,8 +23,8 @@ import com.linkedin.entity.client.EntityClient; import com.linkedin.entity.Entity; import com.linkedin.metadata.extractor.SnapshotToAspectMap; +import com.linkedin.metadata.browse.BrowseResult; import com.linkedin.metadata.query.AutoCompleteResult; -import com.linkedin.metadata.query.BrowseResult; import com.linkedin.metadata.query.SearchResult; import graphql.execution.DataFetcherResult; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/BrowseResultMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/BrowseResultMapper.java index 138d5f5f6f1c4a..c3e74c28fe59dc 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/BrowseResultMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/BrowseResultMapper.java @@ -14,7 +14,7 @@ public class BrowseResultMapper { private BrowseResultMapper() { } - public static BrowseResults map(com.linkedin.metadata.query.BrowseResult input) { + public static BrowseResults map(com.linkedin.metadata.browse.BrowseResult input) { final BrowseResults result = new BrowseResults(); if (!input.hasFrom() || !input.hasPageSize() || !input.hasNumElements()) { @@ -41,7 +41,7 @@ public static BrowseResults map(com.linkedin.metadata.query.BrowseResult input) return result; } - private static BrowseResultGroup mapGroup(@Nonnull final com.linkedin.metadata.query.BrowseResultGroup group) { + private static BrowseResultGroup mapGroup(@Nonnull final com.linkedin.metadata.browse.BrowseResultGroup group) { final BrowseResultGroup result = new BrowseResultGroup(); result.setName(group.getName()); result.setCount(group.getCount()); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLFeatureTableType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLFeatureTableType.java index 0edd13c2566fc1..a29793b2126232 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLFeatureTableType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLFeatureTableType.java @@ -22,8 +22,8 @@ import com.linkedin.entity.Entity; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.extractor.SnapshotToAspectMap; +import com.linkedin.metadata.browse.BrowseResult; import com.linkedin.metadata.query.AutoCompleteResult; -import com.linkedin.metadata.query.BrowseResult; import com.linkedin.metadata.query.SearchResult; import graphql.execution.DataFetcherResult; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLModelGroupType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLModelGroupType.java index 4be080ef422efa..045252737c0ab7 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLModelGroupType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLModelGroupType.java @@ -22,8 +22,8 @@ import com.linkedin.entity.Entity; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.extractor.SnapshotToAspectMap; +import com.linkedin.metadata.browse.BrowseResult; import com.linkedin.metadata.query.AutoCompleteResult; -import com.linkedin.metadata.query.BrowseResult; import com.linkedin.metadata.query.SearchResult; import graphql.execution.DataFetcherResult; diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLModelType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLModelType.java index 53dfb2ab12cf9d..6654cee17df980 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLModelType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/MLModelType.java @@ -9,7 +9,7 @@ import com.linkedin.entity.Entity; import com.linkedin.metadata.extractor.SnapshotToAspectMap; import com.linkedin.metadata.query.SearchResult; -import com.linkedin.metadata.query.BrowseResult; +import com.linkedin.metadata.browse.BrowseResult; import graphql.execution.DataFetcherResult; import java.util.List; import java.util.Map; diff --git a/gms/client/src/main/java/com/linkedin/entity/client/EntityClient.java b/gms/client/src/main/java/com/linkedin/entity/client/EntityClient.java index a63958a88a2cb9..7cab74aa3e02ac 100644 --- a/gms/client/src/main/java/com/linkedin/entity/client/EntityClient.java +++ b/gms/client/src/main/java/com/linkedin/entity/client/EntityClient.java @@ -14,8 +14,8 @@ import com.linkedin.entity.EntitiesDoSetWritableRequestBuilder; import com.linkedin.entity.EntitiesRequestBuilders; import com.linkedin.entity.Entity; +import com.linkedin.metadata.browse.BrowseResult; import com.linkedin.metadata.query.AutoCompleteResult; -import com.linkedin.metadata.query.BrowseResult; import com.linkedin.metadata.query.SearchResult; import com.linkedin.r2.RemoteInvocationException; import com.linkedin.restli.client.ActionRequestBuilder; From 07599d8d149609bdf9d116d416743eb2f9455dac Mon Sep 17 00:00:00 2001 From: Dexter Lee Date: Thu, 29 Jul 2021 20:43:18 -0700 Subject: [PATCH 8/9] Fix to Gabe's comment --- .../datahub/graphql/GmsGraphQLEngine.java | 33 +++++++++++-------- .../ElasticSearchServiceTest.java | 2 +- .../linkedin/metadata/utils/BrowseUtil.java | 3 ++ 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index 313846e0ae5b39..297e6bf9c50c79 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -218,6 +218,7 @@ public static String schema() { public static void configureRuntimeWiring(final RuntimeWiring.Builder builder) { configureQueryResolvers(builder); configureMutationResolvers(builder); + configureSearchAndBrowseResolvers(builder); configureDatasetResolvers(builder); configureCorpUserResolvers(builder); configureCorpGroupResolvers(builder); @@ -332,6 +333,24 @@ private static void configureMutationResolvers(final RuntimeWiring.Builder build ); } + private static void configureSearchAndBrowseResolvers(final RuntimeWiring.Builder builder) { + builder + .type("SearchResult", typeWiring -> typeWiring + .dataFetcher("entity", new AuthenticatedResolver<>( + new EntityTypeResolver( + ENTITY_TYPES.stream().collect(Collectors.toList()), + (env) -> ((SearchResult) env.getSource()).getEntity())) + ) + ) + .type("BrowseResults", typeWiring -> typeWiring + .dataFetcher("entities", new AuthenticatedResolver<>( + new EntityTypeBatchResolver( + ENTITY_TYPES.stream().collect(Collectors.toList()), + (env) -> ((BrowseResults) env.getSource()).getEntities())) + ) + ); + } + /** * Configures resolvers responsible for resolving the {@link com.linkedin.datahub.graphql.generated.Dataset} type. */ @@ -386,20 +405,6 @@ private static void configureDatasetResolvers(final RuntimeWiring.Builder builde (env) -> ((EntityRelationship) env.getSource()).getEntity())) ) ) - .type("SearchResult", typeWiring -> typeWiring - .dataFetcher("entity", new AuthenticatedResolver<>( - new EntityTypeResolver( - ENTITY_TYPES.stream().collect(Collectors.toList()), - (env) -> ((SearchResult) env.getSource()).getEntity())) - ) - ) - .type("BrowseResults", typeWiring -> typeWiring - .dataFetcher("entities", new AuthenticatedResolver<>( - new EntityTypeBatchResolver( - ENTITY_TYPES.stream().collect(Collectors.toList()), - (env) -> ((BrowseResults) env.getSource()).getEntities())) - ) - ) .type("InstitutionalMemoryMetadata", typeWiring -> typeWiring .dataFetcher("author", new AuthenticatedResolver<>( new LoadableTypeResolver<>( diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchServiceTest.java index 06cc7efd8a773a..5d09511f1016c2 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchServiceTest.java @@ -7,7 +7,7 @@ import com.linkedin.common.urn.Urn; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.models.registry.SnapshotEntityRegistry; -import com.linkedin.metadata.query.BrowseResult; +import com.linkedin.metadata.browse.BrowseResult; import com.linkedin.metadata.query.SearchResult; import com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilders; import com.linkedin.metadata.search.elasticsearch.query.ESBrowseDAO; diff --git a/metadata-utils/src/main/java/com/linkedin/metadata/utils/BrowseUtil.java b/metadata-utils/src/main/java/com/linkedin/metadata/utils/BrowseUtil.java index 6489f1ed39a78f..3b2116fa651278 100644 --- a/metadata-utils/src/main/java/com/linkedin/metadata/utils/BrowseUtil.java +++ b/metadata-utils/src/main/java/com/linkedin/metadata/utils/BrowseUtil.java @@ -10,6 +10,9 @@ public class BrowseUtil { + private BrowseUtil() { + } + public static com.linkedin.metadata.query.BrowseResult convertToLegacyResult(BrowseResult browseResult) { com.linkedin.metadata.query.BrowseResult legacyResult = new com.linkedin.metadata.query.BrowseResult(); From 7c953ac772389538849e095f38f81ba2284618e0 Mon Sep 17 00:00:00 2001 From: Dexter Lee Date: Fri, 30 Jul 2021 09:54:55 -0700 Subject: [PATCH 9/9] Fixed rest json files --- .../com.linkedin.chart.charts.snapshot.json | 48 ++++++++----------- ...inkedin.dashboard.dashboards.snapshot.json | 48 ++++++++----------- ....linkedin.dataflow.dataFlows.snapshot.json | 48 ++++++++----------- ...om.linkedin.datajob.dataJobs.snapshot.json | 48 ++++++++----------- ...om.linkedin.dataset.datasets.snapshot.json | 48 ++++++++----------- ...kedin.glossary.glossaryTerms.snapshot.json | 48 ++++++++----------- 6 files changed, 120 insertions(+), 168 deletions(-) diff --git a/gms/api/src/main/snapshot/com.linkedin.chart.charts.snapshot.json b/gms/api/src/main/snapshot/com.linkedin.chart.charts.snapshot.json index 34428bea6f108b..c72631c64ecc94 100644 --- a/gms/api/src/main/snapshot/com.linkedin.chart.charts.snapshot.json +++ b/gms/api/src/main/snapshot/com.linkedin.chart.charts.snapshot.json @@ -647,26 +647,6 @@ } }, "doc" : "A list of entities under the queried path" - }, { - "name" : "groups", - "type" : { - "type" : "array", - "items" : { - "type" : "record", - "name" : "BrowseResultGroup", - "fields" : [ { - "name" : "name", - "type" : "string", - "doc" : "Name of the group" - }, { - "name" : "count", - "type" : "long", - "doc" : "Number of entities that can be reached from this path" - } ] - } - }, - "doc" : "A list of groups and total number of entities inside those groups under the queried path", - "default" : [ ] }, { "name" : "metadata", "type" : { @@ -677,6 +657,26 @@ "name" : "path", "type" : "string", "doc" : "Path that is being browsed" + }, { + "name" : "groups", + "type" : { + "type" : "array", + "items" : { + "type" : "record", + "name" : "BrowseResultGroup", + "fields" : [ { + "name" : "name", + "type" : "string", + "doc" : "Name of the group" + }, { + "name" : "count", + "type" : "long", + "doc" : "Number of entities that can be reached from this path" + } ] + } + }, + "doc" : "A list of groups and total number of entities inside those groups under the queried path", + "default" : [ ] }, { "name" : "totalNumEntities", "type" : "long", @@ -696,14 +696,6 @@ "name" : "numEntities", "type" : "int", "doc" : "The total number of entities directly under queried path" - }, { - "name" : "numGroups", - "type" : "int", - "doc" : "The total number of groups directly under queried path" - }, { - "name" : "numElements", - "type" : "int", - "doc" : "The total number of elements (entities + groups) directly under queried path" } ] }, "com.linkedin.metadata.query.BrowseResultEntity", "com.linkedin.metadata.query.BrowseResultGroup", "com.linkedin.metadata.query.BrowseResultMetadata", { "type" : "enum", diff --git a/gms/api/src/main/snapshot/com.linkedin.dashboard.dashboards.snapshot.json b/gms/api/src/main/snapshot/com.linkedin.dashboard.dashboards.snapshot.json index e940efe18a1b77..d1ac44a1309154 100644 --- a/gms/api/src/main/snapshot/com.linkedin.dashboard.dashboards.snapshot.json +++ b/gms/api/src/main/snapshot/com.linkedin.dashboard.dashboards.snapshot.json @@ -586,26 +586,6 @@ } }, "doc" : "A list of entities under the queried path" - }, { - "name" : "groups", - "type" : { - "type" : "array", - "items" : { - "type" : "record", - "name" : "BrowseResultGroup", - "fields" : [ { - "name" : "name", - "type" : "string", - "doc" : "Name of the group" - }, { - "name" : "count", - "type" : "long", - "doc" : "Number of entities that can be reached from this path" - } ] - } - }, - "doc" : "A list of groups and total number of entities inside those groups under the queried path", - "default" : [ ] }, { "name" : "metadata", "type" : { @@ -616,6 +596,26 @@ "name" : "path", "type" : "string", "doc" : "Path that is being browsed" + }, { + "name" : "groups", + "type" : { + "type" : "array", + "items" : { + "type" : "record", + "name" : "BrowseResultGroup", + "fields" : [ { + "name" : "name", + "type" : "string", + "doc" : "Name of the group" + }, { + "name" : "count", + "type" : "long", + "doc" : "Number of entities that can be reached from this path" + } ] + } + }, + "doc" : "A list of groups and total number of entities inside those groups under the queried path", + "default" : [ ] }, { "name" : "totalNumEntities", "type" : "long", @@ -635,14 +635,6 @@ "name" : "numEntities", "type" : "int", "doc" : "The total number of entities directly under queried path" - }, { - "name" : "numGroups", - "type" : "int", - "doc" : "The total number of groups directly under queried path" - }, { - "name" : "numElements", - "type" : "int", - "doc" : "The total number of elements (entities + groups) directly under queried path" } ] }, "com.linkedin.metadata.query.BrowseResultEntity", "com.linkedin.metadata.query.BrowseResultGroup", "com.linkedin.metadata.query.BrowseResultMetadata", { "type" : "enum", diff --git a/gms/api/src/main/snapshot/com.linkedin.dataflow.dataFlows.snapshot.json b/gms/api/src/main/snapshot/com.linkedin.dataflow.dataFlows.snapshot.json index 91ea278359f76d..70c2298a73c0c9 100644 --- a/gms/api/src/main/snapshot/com.linkedin.dataflow.dataFlows.snapshot.json +++ b/gms/api/src/main/snapshot/com.linkedin.dataflow.dataFlows.snapshot.json @@ -539,26 +539,6 @@ } }, "doc" : "A list of entities under the queried path" - }, { - "name" : "groups", - "type" : { - "type" : "array", - "items" : { - "type" : "record", - "name" : "BrowseResultGroup", - "fields" : [ { - "name" : "name", - "type" : "string", - "doc" : "Name of the group" - }, { - "name" : "count", - "type" : "long", - "doc" : "Number of entities that can be reached from this path" - } ] - } - }, - "doc" : "A list of groups and total number of entities inside those groups under the queried path", - "default" : [ ] }, { "name" : "metadata", "type" : { @@ -569,6 +549,26 @@ "name" : "path", "type" : "string", "doc" : "Path that is being browsed" + }, { + "name" : "groups", + "type" : { + "type" : "array", + "items" : { + "type" : "record", + "name" : "BrowseResultGroup", + "fields" : [ { + "name" : "name", + "type" : "string", + "doc" : "Name of the group" + }, { + "name" : "count", + "type" : "long", + "doc" : "Number of entities that can be reached from this path" + } ] + } + }, + "doc" : "A list of groups and total number of entities inside those groups under the queried path", + "default" : [ ] }, { "name" : "totalNumEntities", "type" : "long", @@ -588,14 +588,6 @@ "name" : "numEntities", "type" : "int", "doc" : "The total number of entities directly under queried path" - }, { - "name" : "numGroups", - "type" : "int", - "doc" : "The total number of groups directly under queried path" - }, { - "name" : "numElements", - "type" : "int", - "doc" : "The total number of elements (entities + groups) directly under queried path" } ] }, "com.linkedin.metadata.query.BrowseResultEntity", "com.linkedin.metadata.query.BrowseResultGroup", "com.linkedin.metadata.query.BrowseResultMetadata", { "type" : "enum", diff --git a/gms/api/src/main/snapshot/com.linkedin.datajob.dataJobs.snapshot.json b/gms/api/src/main/snapshot/com.linkedin.datajob.dataJobs.snapshot.json index 50ef052a967d63..0c814cfa9237ac 100644 --- a/gms/api/src/main/snapshot/com.linkedin.datajob.dataJobs.snapshot.json +++ b/gms/api/src/main/snapshot/com.linkedin.datajob.dataJobs.snapshot.json @@ -700,26 +700,6 @@ } }, "doc" : "A list of entities under the queried path" - }, { - "name" : "groups", - "type" : { - "type" : "array", - "items" : { - "type" : "record", - "name" : "BrowseResultGroup", - "fields" : [ { - "name" : "name", - "type" : "string", - "doc" : "Name of the group" - }, { - "name" : "count", - "type" : "long", - "doc" : "Number of entities that can be reached from this path" - } ] - } - }, - "doc" : "A list of groups and total number of entities inside those groups under the queried path", - "default" : [ ] }, { "name" : "metadata", "type" : { @@ -730,6 +710,26 @@ "name" : "path", "type" : "string", "doc" : "Path that is being browsed" + }, { + "name" : "groups", + "type" : { + "type" : "array", + "items" : { + "type" : "record", + "name" : "BrowseResultGroup", + "fields" : [ { + "name" : "name", + "type" : "string", + "doc" : "Name of the group" + }, { + "name" : "count", + "type" : "long", + "doc" : "Number of entities that can be reached from this path" + } ] + } + }, + "doc" : "A list of groups and total number of entities inside those groups under the queried path", + "default" : [ ] }, { "name" : "totalNumEntities", "type" : "long", @@ -749,14 +749,6 @@ "name" : "numEntities", "type" : "int", "doc" : "The total number of entities directly under queried path" - }, { - "name" : "numGroups", - "type" : "int", - "doc" : "The total number of groups directly under queried path" - }, { - "name" : "numElements", - "type" : "int", - "doc" : "The total number of elements (entities + groups) directly under queried path" } ] }, "com.linkedin.metadata.query.BrowseResultEntity", "com.linkedin.metadata.query.BrowseResultGroup", "com.linkedin.metadata.query.BrowseResultMetadata", { "type" : "enum", diff --git a/gms/api/src/main/snapshot/com.linkedin.dataset.datasets.snapshot.json b/gms/api/src/main/snapshot/com.linkedin.dataset.datasets.snapshot.json index ae06a176b09a23..7b278264373718 100644 --- a/gms/api/src/main/snapshot/com.linkedin.dataset.datasets.snapshot.json +++ b/gms/api/src/main/snapshot/com.linkedin.dataset.datasets.snapshot.json @@ -1422,26 +1422,6 @@ } }, "doc" : "A list of entities under the queried path" - }, { - "name" : "groups", - "type" : { - "type" : "array", - "items" : { - "type" : "record", - "name" : "BrowseResultGroup", - "fields" : [ { - "name" : "name", - "type" : "string", - "doc" : "Name of the group" - }, { - "name" : "count", - "type" : "long", - "doc" : "Number of entities that can be reached from this path" - } ] - } - }, - "doc" : "A list of groups and total number of entities inside those groups under the queried path", - "default" : [ ] }, { "name" : "metadata", "type" : { @@ -1452,6 +1432,26 @@ "name" : "path", "type" : "string", "doc" : "Path that is being browsed" + }, { + "name" : "groups", + "type" : { + "type" : "array", + "items" : { + "type" : "record", + "name" : "BrowseResultGroup", + "fields" : [ { + "name" : "name", + "type" : "string", + "doc" : "Name of the group" + }, { + "name" : "count", + "type" : "long", + "doc" : "Number of entities that can be reached from this path" + } ] + } + }, + "doc" : "A list of groups and total number of entities inside those groups under the queried path", + "default" : [ ] }, { "name" : "totalNumEntities", "type" : "long", @@ -1471,14 +1471,6 @@ "name" : "numEntities", "type" : "int", "doc" : "The total number of entities directly under queried path" - }, { - "name" : "numGroups", - "type" : "int", - "doc" : "The total number of groups directly under queried path" - }, { - "name" : "numElements", - "type" : "int", - "doc" : "The total number of elements (entities + groups) directly under queried path" } ] }, "com.linkedin.metadata.query.BrowseResultEntity", "com.linkedin.metadata.query.BrowseResultGroup", "com.linkedin.metadata.query.BrowseResultMetadata", { "type" : "enum", diff --git a/gms/api/src/main/snapshot/com.linkedin.glossary.glossaryTerms.snapshot.json b/gms/api/src/main/snapshot/com.linkedin.glossary.glossaryTerms.snapshot.json index 7ba389639c504f..6b3503f77f9ed9 100644 --- a/gms/api/src/main/snapshot/com.linkedin.glossary.glossaryTerms.snapshot.json +++ b/gms/api/src/main/snapshot/com.linkedin.glossary.glossaryTerms.snapshot.json @@ -388,26 +388,6 @@ } }, "doc" : "A list of entities under the queried path" - }, { - "name" : "groups", - "type" : { - "type" : "array", - "items" : { - "type" : "record", - "name" : "BrowseResultGroup", - "fields" : [ { - "name" : "name", - "type" : "string", - "doc" : "Name of the group" - }, { - "name" : "count", - "type" : "long", - "doc" : "Number of entities that can be reached from this path" - } ] - } - }, - "doc" : "A list of groups and total number of entities inside those groups under the queried path", - "default" : [ ] }, { "name" : "metadata", "type" : { @@ -418,6 +398,26 @@ "name" : "path", "type" : "string", "doc" : "Path that is being browsed" + }, { + "name" : "groups", + "type" : { + "type" : "array", + "items" : { + "type" : "record", + "name" : "BrowseResultGroup", + "fields" : [ { + "name" : "name", + "type" : "string", + "doc" : "Name of the group" + }, { + "name" : "count", + "type" : "long", + "doc" : "Number of entities that can be reached from this path" + } ] + } + }, + "doc" : "A list of groups and total number of entities inside those groups under the queried path", + "default" : [ ] }, { "name" : "totalNumEntities", "type" : "long", @@ -437,14 +437,6 @@ "name" : "numEntities", "type" : "int", "doc" : "The total number of entities directly under queried path" - }, { - "name" : "numGroups", - "type" : "int", - "doc" : "The total number of groups directly under queried path" - }, { - "name" : "numElements", - "type" : "int", - "doc" : "The total number of elements (entities + groups) directly under queried path" } ] }, "com.linkedin.metadata.query.BrowseResultEntity", "com.linkedin.metadata.query.BrowseResultGroup", "com.linkedin.metadata.query.BrowseResultMetadata", { "type" : "enum",