feat(ui): Add rich UI ingestion run summary (#5577)

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java

@@ -2,6 +2,7 @@
 
 import com.datahub.authentication.Authentication;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.ImmutableSet;
 import com.linkedin.datahub.graphql.QueryContext;
 import com.linkedin.datahub.graphql.exception.ValidationException;
 import com.linkedin.datahub.graphql.generated.FacetFilterInput;
@@ -27,6 +28,9 @@
 
 public class ResolverUtils {
 
+    private static final Set<String> KEYWORD_EXCLUDED_FILTERS = ImmutableSet.of(
+        "runId"
+    );
     private static final ObjectMapper MAPPER = new ObjectMapper();
 
     private static final Logger _logger = LoggerFactory.getLogger(ResolverUtils.class.getName());
@@ -89,7 +93,14 @@ public static Filter buildFilter(@Nullable List<FacetFilterInput> facetFilterInp
             return null;
         }
         return new Filter().setOr(new ConjunctiveCriterionArray(new ConjunctiveCriterion().setAnd(new CriterionArray(facetFilterInputs.stream()
-            .map(filter -> new Criterion().setField(filter.getField() + ESUtils.KEYWORD_SUFFIX).setValue(filter.getValue()))
+            .map(filter -> new Criterion().setField(getFilterField(filter.getField())).setValue(filter.getValue()))
             .collect(Collectors.toList())))));
     }
+
+    private static String getFilterField(final String originalField) {
+        if (KEYWORD_EXCLUDED_FILTERS.contains(originalField)) {
+            return originalField;
+        }
+        return originalField + ESUtils.KEYWORD_SUFFIX;
+    }
 }

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/IngestionResolverUtils.java

@@ -41,6 +41,7 @@ public static ExecutionRequest mapExecutionRequest(final EntityResponse entityRe
 
     final ExecutionRequest result = new ExecutionRequest();
     result.setUrn(entityUrn.toString());
+    result.setId(entityUrn.getId());
 
     // Map input aspect. Must be present.
     final EnvelopedAspect envelopedInput = aspects.get(Constants.EXECUTION_REQUEST_INPUT_ASPECT_NAME);

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/execution/CreateIngestionExecutionRequestResolver.java

@@ -20,6 +20,7 @@
 import com.linkedin.metadata.Constants;
 import com.linkedin.metadata.config.IngestionConfiguration;
 import com.linkedin.metadata.key.ExecutionRequestKey;
+import com.linkedin.metadata.utils.EntityKeyUtils;
 import com.linkedin.metadata.utils.GenericRecordUtils;
 import com.linkedin.mxe.MetadataChangeProposal;
 import graphql.schema.DataFetcher;
@@ -28,6 +29,8 @@
 import java.util.Map;
 import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
+import org.json.JSONException;
+import org.json.JSONObject;
 
 import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*;
 
@@ -68,6 +71,7 @@ public CompletableFuture<String> get(final DataFetchingEnvironment environment)
           final UUID uuid = UUID.randomUUID();
           final String uuidStr = uuid.toString();
           key.setId(uuidStr);
+          final Urn executionRequestUrn = EntityKeyUtils.convertEntityKeyToUrn(key, Constants.EXECUTION_REQUEST_ENTITY_NAME);
           proposal.setEntityKeyAspect(GenericRecordUtils.serializeAspect(key));
 
           // Fetch the original ingestion source
@@ -100,7 +104,7 @@ public CompletableFuture<String> get(final DataFetchingEnvironment environment)
           execInput.setRequestedAt(System.currentTimeMillis());
 
           Map<String, String> arguments = new HashMap<>();
-          arguments.put(RECIPE_ARG_NAME, ingestionSourceInfo.getConfig().getRecipe());
+          arguments.put(RECIPE_ARG_NAME, injectRunId(ingestionSourceInfo.getConfig().getRecipe(), executionRequestUrn.toString()));
           arguments.put(VERSION_ARG_NAME, ingestionSourceInfo.getConfig().hasVersion()
               ? ingestionSourceInfo.getConfig().getVersion()
               : _ingestionConfiguration.getDefaultCliVersion()
@@ -123,4 +127,23 @@ public CompletableFuture<String> get(final DataFetchingEnvironment environment)
       throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator.");
     });
   }
+
+  /**
+   * Injects an override run id into a recipe for tracking purposes. Any existing run id will be overwritten.
+   *
+   * TODO: Determine if this should be handled in the executor itself.
+   *
+   * @param runId the run id to place into the recipe
+   * @return a modified recipe JSON string
+   */
+  private String injectRunId(final String originalJson, final String runId) {
+    try {
+      JSONObject obj = new JSONObject(originalJson);
+      obj.put("run_id", runId);
+      return obj.toString();
+    } catch (JSONException e) {
+      // This should ideally never be hit.
+      throw new IllegalArgumentException("Failed to create execution request: Invalid recipe json provided.");
+    }
+  }
 }

datahub-graphql-core/src/main/resources/ingestion.graphql

@@ -167,6 +167,11 @@ type ExecutionRequest {
   """
   urn: String!
 
+  """
+  Unique id for the execution request
+  """
+  id: String!
+
   """
   Input provided when creating the Execution Request
   """

datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/ingest/IngestTestUtils.java

@@ -64,7 +64,7 @@ public static DataHubIngestionSourceInfo getTestIngestionSourceInfo() {
     info.setName("My Test Source");
     info.setType("mysql");
     info.setSchedule(new DataHubIngestionSourceSchedule().setTimezone("UTC").setInterval("* * * * *"));
-    info.setConfig(new DataHubIngestionSourceConfig().setVersion("0.8.18").setRecipe("my recipe").setExecutorId("executor id"));
+    info.setConfig(new DataHubIngestionSourceConfig().setVersion("0.8.18").setRecipe("{}").setExecutorId("executor id"));
     return info;
   }
 

datahub-web-react/src/app/entity/container/ContainerEntitiesTab.tsx

@@ -1,6 +1,6 @@
 import React from 'react';
 import { useEntityData } from '../shared/EntityContext';
-import { EmbeddedListSearch } from '../shared/components/styled/search/EmbeddedListSearch';
+import { EmbeddedListSearchSection } from '../shared/components/styled/search/EmbeddedListSearchSection';
 
 export const ContainerEntitiesTab = () => {
     const { urn } = useEntityData();
@@ -11,7 +11,7 @@ export const ContainerEntitiesTab = () => {
     };
 
     return (
-        <EmbeddedListSearch
+        <EmbeddedListSearchSection
             fixedFilter={fixedFilter}
             emptySearchQuery="*"
             placeholderText="Filter container entities..."

datahub-web-react/src/app/entity/domain/DomainEntitiesTab.tsx

@@ -1,7 +1,7 @@
 import React from 'react';
 import { useEntityData } from '../shared/EntityContext';
 import { EntityType } from '../../../types.generated';
-import { EmbeddedListSearch } from '../shared/components/styled/search/EmbeddedListSearch';
+import { EmbeddedListSearchSection } from '../shared/components/styled/search/EmbeddedListSearchSection';
 
 export const DomainEntitiesTab = () => {
     const { urn, entityType } = useEntityData();
@@ -16,7 +16,7 @@ export const DomainEntitiesTab = () => {
     }
 
     return (
-        <EmbeddedListSearch
+        <EmbeddedListSearchSection
             fixedFilter={fixedFilter}
             emptySearchQuery="*"
             placeholderText="Filter domain entities..."

datahub-web-react/src/app/entity/glossaryTerm/profile/GlossaryRelatedEntity.tsx

@@ -1,7 +1,7 @@
 import { Col, Row } from 'antd';
 import * as React from 'react';
 import styled from 'styled-components';
-import { EmbeddedListSearch } from '../../shared/components/styled/search/EmbeddedListSearch';
+import { EmbeddedListSearchSection } from '../../shared/components/styled/search/EmbeddedListSearchSection';
 
 import { useEntityData } from '../../shared/EntityContext';
 
@@ -21,7 +21,7 @@ export default function GlossaryRelatedEntity() {
     return (
         <GroupAssetsWrapper>
             <Col md={24} lg={24} xl={24}>
-                <EmbeddedListSearch
+                <EmbeddedListSearchSection
                     fixedQuery={fixedQueryString}
                     emptySearchQuery="*"
                     placeholderText="Filter entities..."

datahub-web-react/src/app/entity/group/GroupAssets.tsx

@@ -1,6 +1,6 @@
 import React from 'react';
 import styled from 'styled-components';
-import { EmbeddedListSearch } from '../shared/components/styled/search/EmbeddedListSearch';
+import { EmbeddedListSearchSection } from '../shared/components/styled/search/EmbeddedListSearchSection';
 
 const GroupAssetsWrapper = styled.div`
     height: calc(100vh - 114px);
@@ -13,7 +13,7 @@ type Props = {
 export const GroupAssets = ({ urn }: Props) => {
     return (
         <GroupAssetsWrapper>
-            <EmbeddedListSearch
+            <EmbeddedListSearchSection
                 fixedFilter={{ field: 'owners', value: urn }}
                 emptySearchQuery="*"
                 placeholderText="Filter entities..."

datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearch.tsx

@@ -1,20 +1,14 @@
 import React, { useState, useEffect } from 'react';
-import * as QueryString from 'query-string';
-import { useHistory, useLocation, useParams } from 'react-router';
 import { message } from 'antd';
 import styled from 'styled-components';
 import { ApolloError } from '@apollo/client';
-import { useEntityRegistry } from '../../../../../useEntityRegistry';
 import { EntityType, FacetFilterInput } from '../../../../../../types.generated';
-import useFilters from '../../../../../search/utils/useFilters';
 import { ENTITY_FILTER_NAME } from '../../../../../search/utils/constants';
 import { SearchCfg } from '../../../../../../conf';
-import { navigateToEntitySearchUrl } from './navigateToEntitySearchUrl';
 import { EmbeddedListSearchResults } from './EmbeddedListSearchResults';
 import EmbeddedListSearchHeader from './EmbeddedListSearchHeader';
 import { useGetSearchResultsForMultipleQuery } from '../../../../../../graphql/search.generated';
 import { GetSearchResultsParams, SearchResultsInterface } from './types';
-import { useEntityQueryParams } from '../../../containers/profile/utils';
 import { isListSubset } from '../../../utils';
 import { EntityAndType } from '../../../types';
 
@@ -51,17 +45,21 @@ export const addFixedQuery = (baseQuery: string, fixedQuery: string, emptyQuery:
     return finalQuery;
 };
 
-type SearchPageParams = {
-    type?: string;
-};
-
 type Props = {
+    query: string;
+    page: number;
+    filters: FacetFilterInput[];
+    onChangeQuery: (query) => void;
+    onChangeFilters: (filters) => void;
+    onChangePage: (page) => void;
     emptySearchQuery?: string | null;
     fixedFilter?: FacetFilterInput | null;
     fixedQuery?: string | null;
     placeholderText?: string | null;
     defaultShowFilters?: boolean;
     defaultFilters?: Array<FacetFilterInput>;
+    searchBarStyle?: any;
+    searchBarInputStyle?: any;
     useGetSearchResults?: (params: GetSearchResultsParams) => {
         data: SearchResultsInterface | undefined | null;
         loading: boolean;
@@ -71,24 +69,26 @@ type Props = {
 };
 
 export const EmbeddedListSearch = ({
+    query,
+    filters,
+    page,
+    onChangeQuery,
+    onChangeFilters,
+    onChangePage,
     emptySearchQuery,
     fixedFilter,
     fixedQuery,
     placeholderText,
     defaultShowFilters,
     defaultFilters,
+    searchBarStyle,
+    searchBarInputStyle,
     useGetSearchResults = useWrappedSearchResults,
 }: Props) => {
-    const history = useHistory();
-    const location = useLocation();
-    const entityRegistry = useEntityRegistry();
-    const baseParams = useEntityQueryParams();
-
-    const params = QueryString.parse(location.search, { arrayFormat: 'comma' });
-    const query: string = addFixedQuery(params?.query as string, fixedQuery as string, emptySearchQuery as string);
-    const activeType = entityRegistry.getTypeOrDefaultFromPathName(useParams<SearchPageParams>().type || '', undefined);
-    const page: number = params.page && Number(params.page as string) > 0 ? Number(params.page as string) : 1;
-    const filters: Array<FacetFilterInput> = useFilters(params);
+    // Adjust query based on props
+    const finalQuery: string = addFixedQuery(query as string, fixedQuery as string, emptySearchQuery as string);
+
+    // Adjust filters based on props
     const filtersWithoutEntities: Array<FacetFilterInput> = filters.filter(
         (filter) => filter.field !== ENTITY_FILTER_NAME,
     );
@@ -106,7 +106,7 @@ export const EmbeddedListSearch = ({
         variables: {
             input: {
                 types: entityFilters,
-                query,
+                query: finalQuery,
                 start: (page - 1) * SearchCfg.RESULTS_PER_PAGE,
                 count: SearchCfg.RESULTS_PER_PAGE,
                 filters: finalFilters,
@@ -123,7 +123,7 @@ export const EmbeddedListSearch = ({
         variables: {
             input: {
                 types: entityFilters,
-                query,
+                query: finalQuery,
                 start: (page - 1) * numResultsPerPage,
                 count: numResultsPerPage,
                 filters: finalFilters,
@@ -136,42 +136,6 @@ export const EmbeddedListSearch = ({
     const searchResultUrns = searchResultEntities.map((entity) => entity.urn);
     const selectedEntityUrns = selectedEntities.map((entity) => entity.urn);
 
-    const onSearch = (q: string) => {
-        const finalQuery = addFixedQuery(q as string, fixedQuery as string, emptySearchQuery as string);
-        navigateToEntitySearchUrl({
-            baseUrl: location.pathname,
-            baseParams,
-            type: activeType,
-            query: finalQuery,
-            page: 1,
-            history,
-        });
-    };
-
-    const onChangeFilters = (newFilters: Array<FacetFilterInput>) => {
-        navigateToEntitySearchUrl({
-            baseUrl: location.pathname,
-            baseParams,
-            type: activeType,
-            query,
-            page: 1,
-            filters: newFilters,
-            history,
-        });
-    };
-
-    const onChangePage = (newPage: number) => {
-        navigateToEntitySearchUrl({
-            baseUrl: location.pathname,
-            baseParams,
-            type: activeType,
-            query,
-            page: newPage,
-            filters,
-            history,
-        });
-    };
-
     const onToggleFilters = () => {
         setShowFilters(!showFilters);
     };
@@ -198,6 +162,12 @@ export const EmbeddedListSearch = ({
         }
     };
 
+    useEffect(() => {
+        if (!isSelectMode) {
+            setSelectedEntities([]);
+        }
+    }, [isSelectMode]);
+
     useEffect(() => {
         if (defaultFilters) {
             onChangeFilters(defaultFilters);
@@ -206,32 +176,28 @@ export const EmbeddedListSearch = ({
         // eslint-disable-next-line react-hooks/exhaustive-deps
     }, []);
 
-    useEffect(() => {
-        if (!isSelectMode) {
-            setSelectedEntities([]);
-        }
-    }, [isSelectMode]);
-
     // Filter out the persistent filter values
     const filteredFilters = data?.facets?.filter((facet) => facet.field !== fixedFilter?.field) || [];
 
     return (
         <Container>
             {error && message.error(`Failed to complete search: ${error && error.message}`)}
             <EmbeddedListSearchHeader
-                onSearch={onSearch}
+                onSearch={(q) => onChangeQuery(addFixedQuery(q, fixedQuery as string, emptySearchQuery as string))}
                 placeholderText={placeholderText}
                 onToggleFilters={onToggleFilters}
                 callSearchOnVariables={callSearchOnVariables}
                 entityFilters={entityFilters}
                 filters={finalFilters}
-                query={query}
+                query={finalQuery}
                 isSelectMode={isSelectMode}
                 isSelectAll={selectedEntities.length > 0 && isListSubset(searchResultUrns, selectedEntityUrns)}
                 setIsSelectMode={setIsSelectMode}
                 selectedEntities={selectedEntities}
                 onChangeSelectAll={onChangeSelectAll}
                 refetch={refetch as any}
+                searchBarStyle={searchBarStyle}
+                searchBarInputStyle={searchBarInputStyle}
             />
             <EmbeddedListSearchResults
                 loading={loading}