feat(lineage): hide soft deleted nodes in lineage & adds banner in entity page

datahub-web-react/src/Mocks.tsx

@@ -282,6 +282,7 @@ const dataset2 = {
     downstream: null,
     health: null,
     assertions: null,
+    status: null,
 };
 
 export const dataset3 = {
@@ -484,6 +485,7 @@ export const dataset3 = {
     relationships: null,
     health: null,
     assertions: null,
+    status: null,
 } as Dataset;
 
 export const dataset4 = {
@@ -1082,6 +1084,7 @@ export const dataJob1 = {
         ],
     },
     domain: null,
+    status: null,
 } as DataJob;
 
 export const dataJob2 = {
@@ -1204,6 +1207,7 @@ export const dataJob3 = {
     domain: null,
     upstream: null,
     downstream: null,
+    status: null,
 } as DataJob;
 
 export const mlModel = {
@@ -1278,6 +1282,7 @@ export const mlModel = {
     outgoing: null,
     upstream: null,
     downstream: null,
+    status: null,
 } as MlModel;
 
 export const mlModelGroup = {
@@ -1340,6 +1345,7 @@ export const mlModelGroup = {
     outgoing: null,
     upstream: null,
     downstream: null,
+    status: null,
 } as MlModelGroup;
 
 export const recommendationModules = [

datahub-web-react/src/app/entity/EntityRegistry.tsx

@@ -116,7 +116,13 @@ export default class EntityRegistry {
 
     getLineageVizConfig<T>(type: EntityType, data: T): FetchedEntity | undefined {
         const entity = validatedGet(type, this.entityTypeToEntity);
-        return entity.getLineageVizConfig?.(data) || undefined;
+        const genericEntityProperties = this.getGenericEntityProperties(type, data);
+        return (
+            ({
+                ...entity.getLineageVizConfig?.(data),
+                status: genericEntityProperties?.status,
+            } as FetchedEntity) || undefined
+        );
     }
 
     getDisplayName<T>(type: EntityType, data: T): string {

datahub-web-react/src/app/entity/shared/containers/profile/EntityProfile.tsx

@@ -231,6 +231,12 @@ export const EntityProfile = <T, U>({
         >
             <>
                 {showBrowseBar && <EntityProfileNavBar urn={urn} entityType={entityType} />}
+                {entityData?.status?.removed === true && (
+                    <Alert
+                        message="This entity has been soft deleted and is not discoverable via search or lineage graph"
+                        banner
+                    />
+                )}
                 {loading && <Message type="loading" content="Loading..." style={{ marginTop: '10%' }} />}
                 {!loading && error && (
                     <Alert type="error" message={error?.message || `Entity failed to load for urn ${urn}`} />

datahub-web-react/src/app/entity/shared/types.ts

@@ -23,6 +23,7 @@ import {
     SubTypes,
     Container,
     Health,
+    Status,
 } from '../../../types.generated';
 import { FetchedEntity } from '../../lineage/types';
 
@@ -71,6 +72,7 @@ export type GenericEntityProperties = {
     entityCount?: number;
     container?: Maybe<Container>;
     health?: Maybe<Health>;
+    status?: Maybe<Status>;
 };
 
 export type GenericEntityUpdate = {

datahub-web-react/src/app/lineage/__tests__/constructTree.test.ts

@@ -76,6 +76,7 @@ describe('constructTree', () => {
                     children: [],
                     icon: undefined,
                     platform: 'Kafka',
+                    status: null,
                 },
             ],
         });
@@ -121,6 +122,7 @@ describe('constructTree', () => {
                     children: [],
                     icon: undefined,
                     platform: 'Kafka',
+                    status: null,
                 },
             ],
         });
@@ -166,6 +168,7 @@ describe('constructTree', () => {
                     countercurrentChildrenUrns: ['urn:li:dataset:3'],
                     icon: undefined,
                     platform: 'Kafka',
+                    status: null,
                     children: [
                         {
                             name: 'Sixth Test Dataset',
@@ -175,6 +178,7 @@ describe('constructTree', () => {
                             countercurrentChildrenUrns: ['urn:li:dataset:4'],
                             icon: undefined,
                             platform: 'Kafka',
+                            status: null,
                             children: [
                                 {
                                     name: 'Fifth Test Dataset',
@@ -189,6 +193,7 @@ describe('constructTree', () => {
                                     ],
                                     icon: undefined,
                                     platform: 'Kafka',
+                                    status: null,
                                 },
                             ],
                         },
@@ -201,6 +206,7 @@ describe('constructTree', () => {
                             countercurrentChildrenUrns: ['urn:li:dataset:7', 'urn:li:dataset:6', 'urn:li:dataset:4'],
                             icon: undefined,
                             platform: 'Kafka',
+                            status: null,
                         },
                     ],
                 },
@@ -276,6 +282,7 @@ describe('constructTree', () => {
                     countercurrentChildrenUrns: ['urn:li:dataset:3'],
                     icon: undefined,
                     platform: 'Kafka',
+                    status: null,
                 },
             ],
         });

datahub-web-react/src/app/lineage/types.ts

@@ -9,6 +9,8 @@ import {
     MlFeature,
     MlModel,
     MlModelGroup,
+    Maybe,
+    Status,
 } from '../../types.generated';
 
 export type EntitySelectParams = {
@@ -33,6 +35,7 @@ export type FetchedEntity = {
     downstreamChildren?: Array<EntityAndType>;
     fullyFetched?: boolean;
     platform?: string;
+    status?: Maybe<Status>;
 };
 
 export type NodeData = {
@@ -47,6 +50,7 @@ export type NodeData = {
     // Currently our visualization does not support expanding in two directions
     countercurrentChildrenUrns?: string[];
     platform?: string;
+    status?: Maybe<Status>;
 };
 
 export type VizNode = {

datahub-web-react/src/app/lineage/utils/constructFetchedNode.ts

@@ -35,6 +35,7 @@ export default function constructFetchedNode(
                 ) || [],
             children: [],
             platform: fetchedNode?.platform,
+            status: fetchedNode.status,
         };
 
         // eslint-disable-next-line no-param-reassign

datahub-web-react/src/app/lineage/utils/layoutTree.ts

@@ -44,10 +44,13 @@ export default function layoutTree(
             .filter((urn, pos) => urnsToAddInCurrentLayer.indexOf(urn) === pos)
             .filter((urn) => !nodesByUrn[urn || '']);
 
-        const layerSize = nodesToAddInCurrentLayer.length;
-
-        const layerHeight = nodesInCurrentLayer
+        const filteredNodesInCurrentLayer = nodesInCurrentLayer
             .filter(({ node }) => nodesToAddInCurrentLayer.indexOf(node.urn || '') > -1)
+            .filter(({ node }) => node.status?.removed !== true);
+
+        const layerSize = filteredNodesInCurrentLayer.length;
+
+        const layerHeight = filteredNodesInCurrentLayer
             .map(({ node }) => nodeHeightFromTitleLength(expandTitles ? node.name : undefined))
             .reduce((acc, height) => acc + height, 0);
 
@@ -58,10 +61,14 @@ export default function layoutTree(
             -((nodeHeightFromTitleLength(undefined) + VERTICAL_SPACE_BETWEEN_NODES) * (layerSize - 1)) / 2 +
             canvasHeight / 2 +
             HEADER_HEIGHT;
+
         // eslint-disable-next-line @typescript-eslint/no-loop-func
         nodesInCurrentLayer.forEach(({ node, parent }) => {
             if (!node.urn) return;
 
+            // don't show edges to soft deleted entities
+            if (node.status?.removed) return;
+
             let vizNodeForNode: VizNode;
 
             if (nodesByUrn[node.urn]) {

datahub-web-react/src/graphql/chart.graphql

@@ -62,6 +62,9 @@ query getChart($urn: String!) {
         downstream: lineage(input: { direction: DOWNSTREAM, start: 0, count: 100 }) {
             ...fullLineageResults
         }
+        status {
+            removed
+        }
     }
 }
 

datahub-web-react/src/graphql/dataFlow.graphql

@@ -35,6 +35,9 @@ fragment dataFlowFields on DataFlow {
     domain {
         ...entityDomain
     }
+    status {
+        removed
+    }
 }
 
 query getDataFlow($urn: String!) {

datahub-web-react/src/graphql/dataJob.graphql

@@ -25,6 +25,9 @@ query getDataJob($urn: String!) {
         downstream: lineage(input: { direction: DOWNSTREAM, start: 0, count: 100 }) {
             ...fullLineageResults
         }
+        status {
+            removed
+        }
     }
 }
 

datahub-web-react/src/graphql/dataset.graphql

@@ -137,6 +137,9 @@ query getDataset($urn: String!) {
                 key
             }
         }
+        status {
+            removed
+        }
     }
 }
 

datahub-web-react/src/graphql/fragments.graphql

@@ -333,6 +333,9 @@ fragment dashboardFields on Dashboard {
     container {
         ...entityContainer
     }
+    status {
+        removed
+    }
 }
 
 fragment nonRecursiveMLFeature on MLFeature {
@@ -547,6 +550,9 @@ fragment nonRecursiveMLModel on MLModel {
     globalTags {
         ...globalTagsFields
     }
+    status {
+        removed
+    }
 }
 
 fragment nonRecursiveMLModelGroupFields on MLModelGroup {
@@ -561,6 +567,9 @@ fragment nonRecursiveMLModelGroupFields on MLModelGroup {
     ownership {
         ...ownershipFields
     }
+    status {
+        removed
+    }
 }
 
 fragment platformFields on DataPlatform {

datahub-web-react/src/graphql/lineage.graphql

@@ -6,6 +6,9 @@ fragment relationshipFields on EntityWithRelationships {
         editableProperties {
             description
         }
+        status {
+            removed
+        }
     }
     ... on DataFlow {
         orchestrator
@@ -34,6 +37,9 @@ fragment relationshipFields on EntityWithRelationships {
         domain {
             ...entityDomain
         }
+        status {
+            removed
+        }
     }
     ... on Dashboard {
         ...dashboardFields
@@ -43,6 +49,9 @@ fragment relationshipFields on EntityWithRelationships {
         platform {
             ...platformFields
         }
+        status {
+            removed
+        }
     }
     ... on Chart {
         tool
@@ -63,11 +72,14 @@ fragment relationshipFields on EntityWithRelationships {
         domain {
             ...entityDomain
         }
+        status {
+            removed
+        }
     }
     ... on Dataset {
         name
         properties {
-            name 
+            name
             description
         }
         editableProperties {
@@ -82,6 +94,9 @@ fragment relationshipFields on EntityWithRelationships {
         subTypes {
             typeNames
         }
+        status {
+            removed
+        }
     }
     ... on MLModelGroup {
         urn
@@ -95,6 +110,9 @@ fragment relationshipFields on EntityWithRelationships {
         ownership {
             ...ownershipFields
         }
+        status {
+            removed
+        }
     }
     ... on MLModel {
         urn
@@ -108,6 +126,9 @@ fragment relationshipFields on EntityWithRelationships {
         ownership {
             ...ownershipFields
         }
+        status {
+            removed
+        }
     }
     upstream: lineage(input: { direction: UPSTREAM, start: 0, count: 100 }) {
         ...leafLineageResults