From da187158783d47e5d5b62e75d24b3185fc3e8935 Mon Sep 17 00:00:00 2001 From: Raj Tekal Date: Wed, 3 May 2023 22:26:00 -0400 Subject: [PATCH 1/5] Initial version - Joins Schema --- .../java/com/linkedin/metadata/Constants.java | 6 ++ .../com/linkedin/common/urn/JoinUrn.java | 66 ++++++++++++++++++ .../pegasus/com/linkedin/common/JoinUrn.pdl | 24 +++++++ .../linkedin/join/EditableJoinProperties.pdl | 31 +++++++++ .../pegasus/com/linkedin/join/FieldMap.pdl | 19 +++++ .../com/linkedin/join/JoinFieldMapping.pdl | 18 +++++ .../com/linkedin/join/JoinProperties.pdl | 69 +++++++++++++++++++ .../main/pegasus/com/linkedin/join/Joins.pdl | 15 ++++ .../linkedin/metadata/aspect/JoinAspect.pdl | 30 ++++++++ .../com/linkedin/metadata/key/JoinKey.pdl | 18 +++++ .../metadata/snapshot/JoinSnapshot.pdl | 24 +++++++ .../src/main/resources/entity-registry.yml | 13 ++++ 12 files changed, 333 insertions(+) create mode 100644 li-utils/src/main/javaPegasus/com/linkedin/common/urn/JoinUrn.java create mode 100644 li-utils/src/main/pegasus/com/linkedin/common/JoinUrn.pdl create mode 100644 metadata-models/src/main/pegasus/com/linkedin/join/EditableJoinProperties.pdl create mode 100644 metadata-models/src/main/pegasus/com/linkedin/join/FieldMap.pdl create mode 100644 metadata-models/src/main/pegasus/com/linkedin/join/JoinFieldMapping.pdl create mode 100644 metadata-models/src/main/pegasus/com/linkedin/join/JoinProperties.pdl create mode 100644 metadata-models/src/main/pegasus/com/linkedin/join/Joins.pdl create mode 100644 metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/JoinAspect.pdl create mode 100644 metadata-models/src/main/pegasus/com/linkedin/metadata/key/JoinKey.pdl create mode 100644 metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/JoinSnapshot.pdl diff --git a/li-utils/src/main/java/com/linkedin/metadata/Constants.java b/li-utils/src/main/java/com/linkedin/metadata/Constants.java index d37eedbe4035e..8ff8fb80ce8f0 100644 --- a/li-utils/src/main/java/com/linkedin/metadata/Constants.java +++ b/li-utils/src/main/java/com/linkedin/metadata/Constants.java @@ -45,6 +45,7 @@ public class Constants { public static final String TAG_ENTITY_NAME = "tag"; public static final String CONTAINER_ENTITY_NAME = "container"; public static final String DOMAIN_ENTITY_NAME = "domain"; + public static final String JOIN_ENTITY_NAME = "join"; public static final String ASSERTION_ENTITY_NAME = "assertion"; public static final String INGESTION_SOURCE_ENTITY_NAME = "dataHubIngestionSource"; public static final String SECRETS_ENTITY_NAME = "dataHubSecret"; @@ -216,6 +217,11 @@ public class Constants { public static final String DOMAINS_ASPECT_NAME = "domains"; public static final String DOMAIN_CREATED_TIME_INDEX_FIELD_NAME = "createdTime"; + // Join + public static final String JOIN_KEY_ASPECT_NAME = "joinKey"; + public static final String JOIN_PROPERTIES_ASPECT_NAME = "joinProperties"; + public static final String EDITABLE_JOIN_PROPERTIES_ASPECT_NAME = "editableJoinProperties"; + // Assertion public static final String ASSERTION_KEY_ASPECT_NAME = "assertionKey"; public static final String ASSERTION_INFO_ASPECT_NAME = "assertionInfo"; diff --git a/li-utils/src/main/javaPegasus/com/linkedin/common/urn/JoinUrn.java b/li-utils/src/main/javaPegasus/com/linkedin/common/urn/JoinUrn.java new file mode 100644 index 0000000000000..ed92bca555eea --- /dev/null +++ b/li-utils/src/main/javaPegasus/com/linkedin/common/urn/JoinUrn.java @@ -0,0 +1,66 @@ +package com.linkedin.common.urn; + +import com.linkedin.data.template.Custom; +import com.linkedin.data.template.DirectCoercer; +import com.linkedin.data.template.TemplateOutputCastException; +import java.net.URISyntaxException; + + +public class JoinUrn extends Urn { + public static final String ENTITY_TYPE = "join"; + + private final String _joinId; + + public JoinUrn(String name) { + super(ENTITY_TYPE, TupleKey.create(name)); + this._joinId = name; + } + + public String getName() { + return _joinId; + } + + public static JoinUrn createFromString(String rawUrn) throws URISyntaxException { + return createFromUrn(Urn.createFromString(rawUrn)); + } + + public static JoinUrn createFromUrn(Urn urn) throws URISyntaxException { + if (!"li".equals(urn.getNamespace())) { + throw new URISyntaxException(urn.toString(), "Urn namespace type should be 'li'."); + } else if (!ENTITY_TYPE.equals(urn.getEntityType())) { + throw new URISyntaxException(urn.toString(), "Urn entity type should be 'join'."); + } else { + TupleKey key = urn.getEntityKey(); + if (key.size() != 1) { + throw new URISyntaxException(urn.toString(), "Invalid number of keys."); + } else { + try { + return new JoinUrn((String) key.getAs(0, String.class)); + } catch (Exception var3) { + throw new URISyntaxException(urn.toString(), "Invalid URN Parameter: '" + var3.getMessage()); + } + } + } + } + + public static JoinUrn deserialize(String rawUrn) throws URISyntaxException { + return createFromString(rawUrn); + } + + static { + Custom.initializeCustomClass(JoinUrn.class); + Custom.registerCoercer(new DirectCoercer() { + public Object coerceInput(JoinUrn object) throws ClassCastException { + return object.toString(); + } + + public JoinUrn coerceOutput(Object object) throws TemplateOutputCastException { + try { + return JoinUrn.createFromString((String) object); + } catch (URISyntaxException e) { + throw new TemplateOutputCastException("Invalid URN syntax: " + e.getMessage(), e); + } + } + }, JoinUrn.class); + } +} diff --git a/li-utils/src/main/pegasus/com/linkedin/common/JoinUrn.pdl b/li-utils/src/main/pegasus/com/linkedin/common/JoinUrn.pdl new file mode 100644 index 0000000000000..abfb5ba9a34cb --- /dev/null +++ b/li-utils/src/main/pegasus/com/linkedin/common/JoinUrn.pdl @@ -0,0 +1,24 @@ +namespace com.linkedin.common + +/** + * Standardized join identifier. + */ +@java.class = "com.linkedin.common.urn.JoinUrn" +@validate.`com.linkedin.common.validator.TypedUrnValidator` = { + "accessible" : true, + "owningTeam" : "urn:li:internalTeam:datahub", + "entityType" : "join", + "constructable" : true, + "namespace" : "li", + "name" : "Join", + "doc" : "Standardized Join identifier.", + "owners" : [ "urn:li:corpuser:fbar", "urn:li:corpuser:bfoo" ], + "fields" : [ { + "name" : "joinId", + "doc" : "Join native name e.g. ., /dir/subdir/, or ", + "type" : "string", + "maxLength" : 284 + }], + "maxLength" : 284 +} +typeref JoinUrn = string \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/join/EditableJoinProperties.pdl b/metadata-models/src/main/pegasus/com/linkedin/join/EditableJoinProperties.pdl new file mode 100644 index 0000000000000..beba0d7d13c6f --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/join/EditableJoinProperties.pdl @@ -0,0 +1,31 @@ +namespace com.linkedin.join + +import com.linkedin.common.ChangeAuditStamps + + +/** + * EditableJoinProperties stores editable changes made to join properties. This separates changes made from + * ingestion pipelines and edits in the UI to avoid accidental overwrites of user-provided data by ingestion pipelines + */ +@Aspect = { + "name": "editableJoinProperties" +} +record EditableJoinProperties includes ChangeAuditStamps { + /** + * Documentation of the join + */ + @Searchable = { + "fieldType": "TEXT", + "fieldName": "editedDescription", + } + description: optional string + + /** + * Display name of the Join + */ + @Searchable = { + "fieldType": "TEXT_PARTIAL", + "fieldName": "editedName", + } + name: optional string +} diff --git a/metadata-models/src/main/pegasus/com/linkedin/join/FieldMap.pdl b/metadata-models/src/main/pegasus/com/linkedin/join/FieldMap.pdl new file mode 100644 index 0000000000000..294dbe70e7684 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/join/FieldMap.pdl @@ -0,0 +1,19 @@ +namespace com.linkedin.join + +import com.linkedin.dataset.SchemaFieldPath + +/** + * Field Mapping of 1:1 field + */ +record FieldMap { + /** + * All fields from dataset A that are required for the join, maps to bFields 1:1 + */ + afield: SchemaFieldPath + + /** + * All fields from dataset B that are required for the join, maps to aFields 1:1 + */ + bfield: SchemaFieldPath + +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/join/JoinFieldMapping.pdl b/metadata-models/src/main/pegasus/com/linkedin/join/JoinFieldMapping.pdl new file mode 100644 index 0000000000000..f875f36f42037 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/join/JoinFieldMapping.pdl @@ -0,0 +1,18 @@ +namespace com.linkedin.join + +import com.linkedin.dataset.SchemaFieldPath + +/** + * Field Mapping about a join between two datasets + */ +record JoinFieldMapping { + /** + * All fields from dataset A that are required for the join to dataset B + */ + fieldMapping: array[FieldMap] + + /** + * Any transformation logic or notes pertaining to this specific join + */ + details: string +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/join/JoinProperties.pdl b/metadata-models/src/main/pegasus/com/linkedin/join/JoinProperties.pdl new file mode 100644 index 0000000000000..14831e3428869 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/join/JoinProperties.pdl @@ -0,0 +1,69 @@ +namespace com.linkedin.join + +import com.linkedin.common.TimeStamp +import com.linkedin.common.DatasetUrn +import com.linkedin.common.CustomProperties + +/** + * Properties associated with a Join + */ +@Aspect = { + "name": "joinProperties" +} +record JoinProperties includes CustomProperties { + + /** + * Display name of the Join + */ + @Searchable = { + "fieldType": "TEXT_PARTIAL", + "enableAutocomplete": true, + "boostScore": 10.0 + } + name: optional string + + /** + * First dataset in the join (no directionality) + */ + @Relationship = { + "name": "joinA", + "entityTypes": [ "dataset" ] + } + datasetA: DatasetUrn + + /** + * Second dataset in the join (no directionality) + */ + @Relationship = { + "name": "joinB", + "entityTypes": [ "dataset" ] + } + datasetB: DatasetUrn + + /** + * Array of JoinFieldMapping + */ + joinFieldMappings: array[JoinFieldMapping] + + /** + * A timestamp documenting when the asset was created in the source Data Platform (not on DataHub) + */ + @Searchable = { + "/time": { + "fieldName": "createdAt", + "fieldType": "DATETIME" + } + } + created: optional TimeStamp + + /** + * A timestamp documenting when the asset was last modified in the source Data Platform (not on DataHub) + */ + @Searchable = { + "/time": { + "fieldName": "lastModifiedAt", + "fieldType": "DATETIME" + } + } + lastModified: optional TimeStamp +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/join/Joins.pdl b/metadata-models/src/main/pegasus/com/linkedin/join/Joins.pdl new file mode 100644 index 0000000000000..2e339d15a2a2d --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/join/Joins.pdl @@ -0,0 +1,15 @@ +namespace com.linkedin.join +import com.linkedin.common.JoinUrn + +/** + * Joins information of an entity. + */ +@Aspect = { + "name": "joins" +} +record Joins { + /** + * Join + */ + joins: array[JoinUrn] +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/JoinAspect.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/JoinAspect.pdl new file mode 100644 index 0000000000000..ef8e48ad3252f --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/JoinAspect.pdl @@ -0,0 +1,30 @@ +namespace com.linkedin.metadata.aspect + +import com.linkedin.metadata.key.JoinKey +import com.linkedin.join.JoinProperties +import com.linkedin.join.EditableJoinProperties + +import com.linkedin.common.InstitutionalMemory +import com.linkedin.common.Ownership +import com.linkedin.common.Status +import com.linkedin.container.Container +import com.linkedin.common.GlobalTags +import com.linkedin.common.GlossaryTerms +import com.linkedin.common.BrowsePaths + + +/** + * A union of all supported metadata aspects for a Join + */ +typeref JoinAspect = union[ + JoinKey, + JoinProperties, + EditableJoinProperties, + InstitutionalMemory, + Ownership, + Status, + Container, + GlobalTags, + GlossaryTerms, + BrowsePaths +] diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/key/JoinKey.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/key/JoinKey.pdl new file mode 100644 index 0000000000000..fc209a548aa73 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/metadata/key/JoinKey.pdl @@ -0,0 +1,18 @@ +namespace com.linkedin.metadata.key + +/** + * Key for a Join + */ +@Aspect = { + "name": "joinKey" +} +record JoinKey { + /* + * Unique guid for Join + */ + @Searchable = { + "fieldType": "TEXT", + } + joinId: string + +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/JoinSnapshot.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/JoinSnapshot.pdl new file mode 100644 index 0000000000000..58bc2c59569ab --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/JoinSnapshot.pdl @@ -0,0 +1,24 @@ +namespace com.linkedin.metadata.snapshot + +import com.linkedin.common.JoinUrn +import com.linkedin.metadata.aspect.JoinAspect + +/** + * A metadata snapshot for a specific join entity. + */ +@Entity = { + "name": "join", + "keyAspect": "joinKey" +} +record JoinSnapshot { + + /** + * URN for the entity the metadata snapshot is associated with. + */ + urn: JoinUrn + + /** + * The list of metadata aspects associated with the Join. Depending on the use case, this can either be all, or a selection, of supported aspects. + */ + aspects: array[JoinAspect] +} diff --git a/metadata-models/src/main/resources/entity-registry.yml b/metadata-models/src/main/resources/entity-registry.yml index 041b79cb6cb20..537611de2355a 100644 --- a/metadata-models/src/main/resources/entity-registry.yml +++ b/metadata-models/src/main/resources/entity-registry.yml @@ -419,6 +419,19 @@ entities: keyAspect: dataHubViewKey aspects: - dataHubViewInfo + - name: join + category: core + doc: Join Dataset Fields + keyAspect: joinKey + aspects: + - joinProperties + - editableJoinProperties + - ownership + - status + - container + - globalTags + - glossaryTerms + - browsePaths - name: query category: core keyAspect: queryKey From bdf2bc6ad83ffcbe084f69d4fa18cee5d23ce9a1 Mon Sep 17 00:00:00 2001 From: Raj Tekal Date: Tue, 9 May 2023 15:55:46 -0400 Subject: [PATCH 2/5] Joins slight model changes --- .../src/main/pegasus/com/linkedin/join/JoinProperties.pdl | 2 +- .../main/pegasus/com/linkedin/metadata/snapshot/Snapshot.pdl | 1 + metadata-models/src/main/resources/entity-registry.yml | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/metadata-models/src/main/pegasus/com/linkedin/join/JoinProperties.pdl b/metadata-models/src/main/pegasus/com/linkedin/join/JoinProperties.pdl index 14831e3428869..0f7449d582c2b 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/join/JoinProperties.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/join/JoinProperties.pdl @@ -20,7 +20,7 @@ record JoinProperties includes CustomProperties { "enableAutocomplete": true, "boostScore": 10.0 } - name: optional string + name: string /** * First dataset in the join (no directionality) diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/Snapshot.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/Snapshot.pdl index 91993724afbad..9cf3e325967ed 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/Snapshot.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/Snapshot.pdl @@ -25,4 +25,5 @@ typeref Snapshot = union[ DataHubPolicySnapshot, SchemaFieldSnapshot, DataHubRetentionSnapshot, + JoinSnapshot, ] diff --git a/metadata-models/src/main/resources/entity-registry.yml b/metadata-models/src/main/resources/entity-registry.yml index 537611de2355a..8a535d5f55af9 100644 --- a/metadata-models/src/main/resources/entity-registry.yml +++ b/metadata-models/src/main/resources/entity-registry.yml @@ -420,12 +420,12 @@ entities: aspects: - dataHubViewInfo - name: join - category: core doc: Join Dataset Fields keyAspect: joinKey aspects: - joinProperties - editableJoinProperties + - institutionalMemory - ownership - status - container From eeba4cb3097fe02d9b3771d2701b871a7f11c31c Mon Sep 17 00:00:00 2001 From: Raj Tekal Date: Wed, 31 May 2023 10:56:09 -0400 Subject: [PATCH 3/5] Added resolvers --- .../datahub/graphql/GmsGraphQLEngine.java | 45 +++ .../graphql/resolvers/EntityTypeMapper.java | 1 + .../common/mappers/UrnToEntityMapper.java | 6 + .../datahub/graphql/types/join/JoinType.java | 242 +++++++++++++ .../types/join/mappers/JoinMapper.java | 128 +++++++ .../join/mappers/JoinUpdateInputMapper.java | 119 +++++++ .../src/main/resources/entity.graphql | 325 ++++++++++++++++++ .../src/graphql/fragments.graphql | 22 ++ datahub-web-react/src/graphql/join.graphql | 42 +++ datahub-web-react/src/graphql/lineage.graphql | 11 + datahub-web-react/src/graphql/search.graphql | 55 +++ .../com/linkedin/common/urn/JoinUrn.java | 8 +- .../pegasus/com/linkedin/join/FieldMap.pdl | 10 + .../com/linkedin/join/JoinFieldMapping.pdl | 6 + .../com/linkedin/join/JoinProperties.pdl | 16 +- .../linkedin/metadata/snapshot/Snapshot.pdl | 2 +- 16 files changed, 1030 insertions(+), 8 deletions(-) create mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/JoinType.java create mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/mappers/JoinMapper.java create mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/mappers/JoinUpdateInputMapper.java create mode 100644 datahub-web-react/src/graphql/join.graphql diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index 1289059e8ac31..8c655521b438f 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -58,11 +58,13 @@ import com.linkedin.datahub.graphql.generated.GlossaryTermAssociation; import com.linkedin.datahub.graphql.generated.IngestionSource; import com.linkedin.datahub.graphql.generated.InstitutionalMemoryMetadata; +import com.linkedin.datahub.graphql.generated.Join; import com.linkedin.datahub.graphql.generated.LineageRelationship; import com.linkedin.datahub.graphql.generated.ListAccessTokenResult; import com.linkedin.datahub.graphql.generated.ListDomainsResult; import com.linkedin.datahub.graphql.generated.ListGroupsResult; import com.linkedin.datahub.graphql.generated.ListQueriesResult; +import com.linkedin.datahub.graphql.generated.ListJoinResult; import com.linkedin.datahub.graphql.generated.ListTestsResult; import com.linkedin.datahub.graphql.generated.ListViewsResult; import com.linkedin.datahub.graphql.generated.MLFeature; @@ -88,6 +90,7 @@ import com.linkedin.datahub.graphql.generated.Test; import com.linkedin.datahub.graphql.generated.TestResult; import com.linkedin.datahub.graphql.generated.UserUsageCounts; +import com.linkedin.datahub.graphql.resolvers.AuthenticatedResolver; import com.linkedin.datahub.graphql.resolvers.MeResolver; import com.linkedin.datahub.graphql.resolvers.assertion.AssertionRunEventResolver; import com.linkedin.datahub.graphql.resolvers.assertion.DeleteAssertionResolver; @@ -279,6 +282,7 @@ import com.linkedin.datahub.graphql.types.tag.TagType; import com.linkedin.datahub.graphql.types.test.TestType; import com.linkedin.datahub.graphql.types.view.DataHubViewType; +import com.linkedin.datahub.graphql.types.join.JoinType; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.config.DataHubConfiguration; import com.linkedin.metadata.config.IngestionConfiguration; @@ -400,6 +404,7 @@ public class GmsGraphQLEngine { private final DataHubPolicyType dataHubPolicyType; private final DataHubRoleType dataHubRoleType; private final SchemaFieldType schemaFieldType; + private final JoinType joinType; private final DataHubViewType dataHubViewType; private final QueryType queryType; @@ -493,6 +498,7 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) { this.dataHubPolicyType = new DataHubPolicyType(entityClient); this.dataHubRoleType = new DataHubRoleType(entityClient); this.schemaFieldType = new SchemaFieldType(); + this.joinType = new JoinType(entityClient); this.dataHubViewType = new DataHubViewType(entityClient); this.queryType = new QueryType(entityClient); @@ -525,6 +531,7 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) { dataHubPolicyType, dataHubRoleType, schemaFieldType, + joinType, dataHubViewType, queryType ); @@ -586,6 +593,7 @@ public void configureRuntimeWiring(final RuntimeWiring.Builder builder) { configureTestResultResolvers(builder); configureRoleResolvers(builder); configureSchemaFieldResolvers(builder); + configureJoinResolvers(builder); configureEntityPathResolvers(builder); configureViewResolvers(builder); configureQueryEntityResolvers(builder); @@ -671,6 +679,9 @@ private void configureDataPlatformInstanceResolvers(final RuntimeWiring.Builder private void configureQueryResolvers(final RuntimeWiring.Builder builder) { builder.type("Query", typeWiring -> typeWiring + .dataFetcher("join", new AuthenticatedResolver<>( + new LoadableTypeResolver<>(joinType, + (env) -> env.getArgument(URN_FIELD_NAME)))) .dataFetcher("appConfig", new AppConfigResolver(gitVersion, analyticsService != null, this.ingestionConfiguration, @@ -708,6 +719,7 @@ private void configureQueryResolvers(final RuntimeWiring.Builder builder) { .dataFetcher("glossaryTerm", getResolver(glossaryTermType)) .dataFetcher("glossaryNode", getResolver(glossaryNodeType)) .dataFetcher("domain", getResolver((domainType))) + .dataFetcher("join", getResolver(joinType)) .dataFetcher("dataPlatform", getResolver(dataPlatformType)) .dataFetcher("mlFeatureTable", getResolver(mlFeatureTableType)) .dataFetcher("mlFeature", getResolver(mlFeatureType)) @@ -806,6 +818,8 @@ private void configureMutationResolvers(final RuntimeWiring.Builder builder) { .dataFetcher("updateDataFlow", new MutableTypeResolver<>(dataFlowType)) .dataFetcher("updateCorpUserProperties", new MutableTypeResolver<>(corpUserType)) .dataFetcher("updateCorpGroupProperties", new MutableTypeResolver<>(corpGroupType)) + .dataFetcher("updateJoin", new AuthenticatedResolver<>(new MutableTypeResolver<>(joinType))) + .dataFetcher("createJoin", new MutableTypeResolver<>(joinType)) .dataFetcher("addTag", new AddTagResolver(entityService)) .dataFetcher("addTags", new AddTagsResolver(entityService)) .dataFetcher("batchAddTags", new BatchAddTagsResolver(entityService)) @@ -940,6 +954,12 @@ private void configureGenericEntityResolvers(final RuntimeWiring.Builder builder .map(Domain::getUrn) .collect(Collectors.toList()))) ) + .type("ListJoinResult", typeWiring -> typeWiring + .dataFetcher("join", new LoadableTypeBatchResolver<>(joinType, + (env) -> ((ListJoinResult) env.getSource()).getJoin().stream() + .map(Join::getUrn) + .collect(Collectors.toList()))) + ) .type("GetRootGlossaryTermsResult", typeWiring -> typeWiring .dataFetcher("terms", new LoadableTypeBatchResolver<>(glossaryTermType, (env) -> ((GetRootGlossaryTermsResult) env.getSource()).getTerms().stream() @@ -1341,6 +1361,31 @@ private void configureTypeExtensions(final RuntimeWiring.Builder builder) { builder.scalar(GraphQLLong); } + /** + * Configures resolvers responsible for resolving the {@link Join} type. + */ + private void configureJoinResolvers(final RuntimeWiring.Builder builder) { + builder + .type("Join", typeWiring -> typeWiring + .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) + .dataFetcher("browsePaths", new EntityBrowsePathsResolver(this.joinType)) + .dataFetcher("container", + new LoadableTypeResolver<>(containerType, + (env) -> { + final Join join = env.getSource(); + return join.getContainer() != null ? join.getContainer().getUrn() : null; + }) + )) + .type("Owner", typeWiring -> typeWiring + .dataFetcher("owner", new OwnerTypeResolver<>(ownerTypes, + (env) -> ((Owner) env.getSource()).getOwner())) + ) + .type("InstitutionalMemoryMetadata", typeWiring -> typeWiring + .dataFetcher("author", new LoadableTypeResolver<>(corpUserType, + (env) -> ((InstitutionalMemoryMetadata) env.getSource()).getAuthor().getUrn())) + ); + } + /** * Configures resolvers responsible for resolving the {@link com.linkedin.datahub.graphql.generated.DataJob} type. */ diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/EntityTypeMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/EntityTypeMapper.java index 958f78a98b7fd..5009f5b202c8a 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/EntityTypeMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/EntityTypeMapper.java @@ -36,6 +36,7 @@ public class EntityTypeMapper { .put(EntityType.NOTEBOOK, "notebook") .put(EntityType.DATA_PLATFORM_INSTANCE, "dataPlatformInstance") .put(EntityType.TEST, "test") + .put(EntityType.JOIN, Constants.JOIN_ENTITY_NAME) .put(EntityType.DATAHUB_VIEW, Constants.DATAHUB_VIEW_ENTITY_NAME) .build(); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/UrnToEntityMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/UrnToEntityMapper.java index 23385130fcc7e..f38e316d48061 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/UrnToEntityMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/UrnToEntityMapper.java @@ -20,6 +20,7 @@ import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.GlossaryNode; import com.linkedin.datahub.graphql.generated.GlossaryTerm; +import com.linkedin.datahub.graphql.generated.Join; import com.linkedin.datahub.graphql.generated.MLFeature; import com.linkedin.datahub.graphql.generated.MLFeatureTable; import com.linkedin.datahub.graphql.generated.MLModel; @@ -145,6 +146,11 @@ public Entity apply(Urn input) { ((Domain) partialEntity).setUrn(input.toString()); ((Domain) partialEntity).setType(EntityType.DOMAIN); } + if (input.getEntityType().equals("join")) { + partialEntity = new Join(); + ((Join) partialEntity).setUrn(input.toString()); + ((Join) partialEntity).setType(EntityType.JOIN); + } if (input.getEntityType().equals("assertion")) { partialEntity = new Assertion(); ((Assertion) partialEntity).setUrn(input.toString()); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/JoinType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/JoinType.java new file mode 100644 index 0000000000000..1164354966ee8 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/JoinType.java @@ -0,0 +1,242 @@ +package com.linkedin.datahub.graphql.types.join; + +import com.google.common.collect.ImmutableSet; +import com.linkedin.common.urn.CorpuserUrn; +import com.linkedin.common.urn.JoinUrn; +import com.linkedin.common.urn.Urn; +import com.linkedin.data.template.StringArray; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.AutoCompleteResults; +import com.linkedin.datahub.graphql.generated.BrowsePath; +import com.linkedin.datahub.graphql.generated.BrowseResults; +import com.linkedin.datahub.graphql.generated.Entity; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.generated.FacetFilterInput; +import com.linkedin.datahub.graphql.generated.FieldSortInput; +import com.linkedin.datahub.graphql.generated.Join; +import com.linkedin.datahub.graphql.generated.JoinUpdateInput; +import com.linkedin.datahub.graphql.generated.SearchResults; +import com.linkedin.datahub.graphql.generated.Sort; +import com.linkedin.datahub.graphql.resolvers.ResolverUtils; +import com.linkedin.datahub.graphql.types.BrowsableEntityType; +import com.linkedin.datahub.graphql.types.MutableType; +import com.linkedin.datahub.graphql.types.SearchableEntityType; +import com.linkedin.datahub.graphql.types.join.mappers.JoinMapper; +import com.linkedin.datahub.graphql.types.join.mappers.JoinUpdateInputMapper; +import com.linkedin.datahub.graphql.types.mappers.AutoCompleteResultsMapper; +import com.linkedin.datahub.graphql.types.mappers.BrowsePathsMapper; +import com.linkedin.datahub.graphql.types.mappers.BrowseResultMapper; +import com.linkedin.datahub.graphql.types.mappers.UrnSearchResultsMapper; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.browse.BrowseResult; +import com.linkedin.metadata.query.AutoCompleteResult; +import com.linkedin.metadata.query.filter.SortCriterion; +import com.linkedin.metadata.query.filter.SortOrder; +import com.linkedin.metadata.search.SearchResult; +import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.r2.RemoteInvocationException; +import graphql.execution.DataFetcherResult; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.function.Function; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import static com.linkedin.datahub.graphql.Constants.*; +import static com.linkedin.metadata.Constants.*; + + +public class JoinType implements com.linkedin.datahub.graphql.types.EntityType, + BrowsableEntityType, SearchableEntityType, + MutableType { + + + static final Set ASPECTS_TO_RESOLVE = ImmutableSet.of( + JOIN_KEY_ASPECT_NAME, + JOIN_PROPERTIES_ASPECT_NAME, + EDITABLE_JOIN_PROPERTIES_ASPECT_NAME, + INSTITUTIONAL_MEMORY_ASPECT_NAME, + OWNERSHIP_ASPECT_NAME, + STATUS_ASPECT_NAME, + CONTAINER_ASPECT_NAME, + GLOBAL_TAGS_ASPECT_NAME, + GLOSSARY_TERMS_ASPECT_NAME, + BROWSE_PATHS_ASPECT_NAME + ); + + private static final Set FACET_FIELDS = ImmutableSet.of("name"); + private static final String ENTITY_NAME = "join"; + + private final EntityClient _entityClient; + + + public JoinType(final EntityClient entityClient) { + _entityClient = entityClient; + } + + @Override + public Class objectClass() { + return Join.class; + } + + + @Override + public Class inputClass() { + return JoinUpdateInput.class; + } + + @Override + public EntityType type() { + return EntityType.JOIN; + } + + @Override + public Function getKeyProvider() { + return Entity::getUrn; + } + + @Override + public List> batchLoad(@Nonnull final List urns, @Nonnull final QueryContext context) + throws Exception { + final List joinUrns = urns.stream() + .map(this::getUrn) + .collect(Collectors.toList()); + + try { + final Map entities = _entityClient.batchGetV2( + JOIN_ENTITY_NAME, + new HashSet<>(joinUrns), + ASPECTS_TO_RESOLVE, + context.getAuthentication()); + + final List gmsResults = new ArrayList<>(); + for (Urn urn : joinUrns) { + gmsResults.add(entities.getOrDefault(urn, null)); + } + return gmsResults.stream() + .map(gmsResult -> + gmsResult == null ? null : DataFetcherResult.newResult() + .data(JoinMapper.map(gmsResult)) + .build() + ) + .collect(Collectors.toList()); + } catch (Exception e) { + throw new RuntimeException("Failed to load join entity", e); + } + } + + @Nonnull + @Override + public BrowseResults browse(@Nonnull List path, @Nullable List filters, int start, + int count, @Nonnull QueryContext context) throws Exception { + final Map facetFilters = ResolverUtils.buildFacetFilters(filters, FACET_FIELDS); + final String pathStr = path.size() > 0 ? BROWSE_PATH_DELIMITER + String.join(BROWSE_PATH_DELIMITER, path) : ""; + final BrowseResult result = _entityClient.browse( + "join", + pathStr, + facetFilters, + start, + count, + context.getAuthentication()); + return BrowseResultMapper.map(result); + } + + private Urn getUrn(final String urnStr) { + try { + return Urn.createFromString(urnStr); + } catch (URISyntaxException e) { + throw new RuntimeException(String.format("Failed to convert urn string %s into Urn", urnStr)); + } + } + + + + @Nonnull + @Override + public List browsePaths(@Nonnull String urn, @Nonnull QueryContext context) throws Exception { + final StringArray result = _entityClient.getBrowsePaths(getJoinUrn(urn), context.getAuthentication()); + return BrowsePathsMapper.map(result); + } + + private com.linkedin.common.urn.JoinUrn getJoinUrn(String urnStr) { + try { + return JoinUrn.createFromString(urnStr); + } catch (URISyntaxException e) { + throw new RuntimeException(String.format("Failed to retrieve data product with urn %s, invalid urn", urnStr)); + } + } + + @Override + public SearchResults search(@Nonnull String query, @Nullable List filters, + @Nullable FieldSortInput sort, int start, int count, @Nonnull QueryContext context) throws Exception { + final Map facetFilters = ResolverUtils.buildFacetFilters(filters, FACET_FIELDS); + String sortField = sort != null ? sort.getField() : null; + SortOrder sortOrder = sort != null ? (sort.getSortOrder().equals(Sort.asc) ? SortOrder.ASCENDING : SortOrder.DESCENDING) : null; + SortCriterion sortCriterion = null; + if (sortField != null && sortOrder != null) { + sortCriterion = + new SortCriterion().setField(sortField).setOrder(sortOrder); + } + final SearchResult searchResult = _entityClient.search(ENTITY_NAME, query, facetFilters, sortCriterion, start, + count, context.getAuthentication()); + return UrnSearchResultsMapper.map(searchResult); + + } + + @Override + public AutoCompleteResults autoComplete(@Nonnull String query, @Nullable String field, + @Nullable List filters, int limit, @Nonnull QueryContext context) throws Exception { + final Map facetFilters = ResolverUtils.buildFacetFilters(filters, FACET_FIELDS); + final AutoCompleteResult result = _entityClient.autoComplete(ENTITY_NAME, query, facetFilters, limit, context.getAuthentication()); + return AutoCompleteResultsMapper.map(result); + } + + + + + @Override + public Join update(String urn, @Nonnull JoinUpdateInput input, @Nonnull QueryContext context) + throws Exception { + if (isAuthorized(urn, input, context)) { + final CorpuserUrn actor = CorpuserUrn.createFromString(context.getAuthentication().getActor().toUrnStr()); + + // Same routine used by create - hence this check + JoinUrn inputUrn = new JoinUrn(UUID.randomUUID().toString()); + if (urn != null) { + inputUrn = JoinUrn.createFromString(urn); + if ("new".equals(inputUrn.getJoinIdEntity())) { + inputUrn = new JoinUrn(UUID.randomUUID().toString()); + } + } else { + urn = inputUrn.toString(); + } + + final JoinUrn updatedUrn = inputUrn; + + final Collection proposals = JoinUpdateInputMapper.map(input, actor); + proposals.forEach(proposal -> proposal.setEntityUrn(updatedUrn)); + + try { + _entityClient.batchIngestProposals(proposals, context.getAuthentication(), false); + } catch (RemoteInvocationException e) { + throw new RuntimeException(String.format("Failed to write entity with urn %s", urn), e); + } + + return load(urn, context).getData(); + } + throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); + } + + private boolean isAuthorized(String urn, JoinUpdateInput input, QueryContext context) { + return true; + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/mappers/JoinMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/mappers/JoinMapper.java new file mode 100644 index 0000000000000..e0d4b2a54dbf0 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/mappers/JoinMapper.java @@ -0,0 +1,128 @@ +package com.linkedin.datahub.graphql.types.join.mappers; + +import com.linkedin.common.GlobalTags; +import com.linkedin.common.GlossaryTerms; +import com.linkedin.common.InstitutionalMemory; +import com.linkedin.common.Ownership; +import com.linkedin.common.Status; +import com.linkedin.common.urn.Urn; +import com.linkedin.data.DataMap; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.datahub.graphql.generated.Container; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.generated.Join; +import com.linkedin.datahub.graphql.types.common.mappers.InstitutionalMemoryMapper; +import com.linkedin.datahub.graphql.types.common.mappers.OwnershipMapper; +import com.linkedin.datahub.graphql.types.common.mappers.StatusMapper; +import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper; +import com.linkedin.datahub.graphql.types.glossary.mappers.GlossaryTermsMapper; +import com.linkedin.datahub.graphql.types.mappers.ModelMapper; +import com.linkedin.datahub.graphql.types.tag.mappers.GlobalTagsMapper; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.join.EditableJoinProperties; +import com.linkedin.join.JoinProperties; +import com.linkedin.metadata.key.JoinKey; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; + +import static com.linkedin.metadata.Constants.*; + +/** + * Maps Pegasus {@link RecordTemplate} objects to objects conforming to the GQL schema. + * + * To be replaced by auto-generated mappers implementations + */ +public class JoinMapper implements ModelMapper { + + public static final JoinMapper INSTANCE = new JoinMapper(); + + public static Join map(@Nonnull final EntityResponse entityResponse) { + return INSTANCE.apply(entityResponse); + } + + public Join apply(final EntityResponse entityResponse) { + final Join result = new Join(); + final Urn entityUrn = entityResponse.getUrn(); + + result.setUrn(entityUrn.toString()); + result.setType(EntityType.JOIN); + + final EnvelopedAspectMap aspectMap = entityResponse.getAspects(); + MappingHelper mappingHelper = new MappingHelper<>(aspectMap, result); + mappingHelper.mapToResult(JOIN_KEY_ASPECT_NAME, this::mapJoinKey); + mappingHelper.mapToResult(JOIN_PROPERTIES_ASPECT_NAME, this::mapProperties); + mappingHelper.mapToResult(EDITABLE_JOIN_PROPERTIES_ASPECT_NAME, this::mapEditableProperties); + mappingHelper.mapToResult(INSTITUTIONAL_MEMORY_ASPECT_NAME, (join, dataMap) -> + join.setInstitutionalMemory(InstitutionalMemoryMapper.map(new InstitutionalMemory(dataMap)))); + mappingHelper.mapToResult(OWNERSHIP_ASPECT_NAME, (join, dataMap) -> + join.setOwnership(OwnershipMapper.map(new Ownership(dataMap), entityUrn))); + mappingHelper.mapToResult(STATUS_ASPECT_NAME, (join, dataMap) -> + join.setStatus(StatusMapper.map(new Status(dataMap)))); + mappingHelper.mapToResult(GLOBAL_TAGS_ASPECT_NAME, (join, dataMap) -> this.mapGlobalTags(join, dataMap, entityUrn)); + mappingHelper.mapToResult(GLOSSARY_TERMS_ASPECT_NAME, (join, dataMap) -> + join.setGlossaryTerms(GlossaryTermsMapper.map(new GlossaryTerms(dataMap), entityUrn))); + mappingHelper.mapToResult(CONTAINER_ASPECT_NAME, this::mapContainers); + return mappingHelper.getResult(); + } + + private void mapEditableProperties(@Nonnull Join join, @Nonnull DataMap dataMap) { + final EditableJoinProperties editableJoinProperties = new EditableJoinProperties(dataMap); + join.setEditableProperties( + com.linkedin.datahub.graphql.generated.JoinEditableProperties.builder() + .setDescription(editableJoinProperties.getDescription()) + .setName(editableJoinProperties.getName()) + .build() + ); + } + + + private void mapJoinKey(@Nonnull Join join, @Nonnull DataMap datamap) { + JoinKey joinKey = new JoinKey(datamap); + join.setJoinId(joinKey.getJoinId()); + } + + private void mapProperties(@Nonnull Join join, @Nonnull DataMap dataMap) { + final JoinProperties joinProperties = new JoinProperties(dataMap); + join.setProperties( + com.linkedin.datahub.graphql.generated.JoinProperties.builder() + .setName(joinProperties.getName()) + .setDatasetA(joinProperties.getDatasetA().toString()) + .setDatasetB(joinProperties.getDatasetB().toString()) + .setJoinFieldMappings(mapJoinFieldMappings(joinProperties)) + .build()); + } + + private com.linkedin.datahub.graphql.generated.JoinFieldMapping mapJoinFieldMappings(JoinProperties joinProperties) { + return com.linkedin.datahub.graphql.generated.JoinFieldMapping.builder() + .setDetails(joinProperties.getJoinFieldMappings().getDetails()) + .setFieldMapping(joinProperties.getJoinFieldMappings() + .getFieldMapping() + .stream() + .map(this::mapFieldMap) + .collect(Collectors.toList())) + .build(); + } + + private com.linkedin.datahub.graphql.generated.FieldMap mapFieldMap(com.linkedin.join.FieldMap fieldMap) { + return com.linkedin.datahub.graphql.generated.FieldMap.builder() + .setAfield(fieldMap.getAfield()) + .setBfield(fieldMap.getBfield()) + .build(); + } + + private void mapGlobalTags(@Nonnull Join join, @Nonnull DataMap dataMap, @Nonnull final Urn entityUrn) { + com.linkedin.datahub.graphql.generated.GlobalTags globalTags = GlobalTagsMapper.map(new GlobalTags(dataMap), entityUrn); + join.setTags(globalTags); + } + + private void mapContainers(@Nonnull Join join, @Nonnull DataMap dataMap) { + final com.linkedin.container.Container gmsContainer = new com.linkedin.container.Container(dataMap); + join.setContainer(Container + .builder() + .setType(EntityType.CONTAINER) + .setUrn(gmsContainer.getContainer().toString()) + .build()); + } + +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/mappers/JoinUpdateInputMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/mappers/JoinUpdateInputMapper.java new file mode 100644 index 0000000000000..11b27849aa32e --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/mappers/JoinUpdateInputMapper.java @@ -0,0 +1,119 @@ +package com.linkedin.datahub.graphql.types.join.mappers; + +import com.linkedin.common.AuditStamp; +import com.linkedin.common.GlobalTags; +import com.linkedin.common.TagAssociationArray; +import com.linkedin.common.urn.DatasetUrn; +import com.linkedin.common.urn.Urn; +import com.linkedin.data.template.SetMode; +import com.linkedin.datahub.graphql.generated.JoinFieldMappingInput; +import com.linkedin.datahub.graphql.generated.JoinUpdateInput; +import com.linkedin.datahub.graphql.types.common.mappers.InstitutionalMemoryUpdateMapper; +import com.linkedin.datahub.graphql.types.common.mappers.OwnershipUpdateMapper; +import com.linkedin.datahub.graphql.types.common.mappers.util.UpdateMappingHelper; +import com.linkedin.datahub.graphql.types.mappers.InputModelMapper; +import com.linkedin.datahub.graphql.types.tag.mappers.TagAssociationUpdateMapper; +import com.linkedin.join.EditableJoinProperties; +import com.linkedin.join.FieldMap; +import com.linkedin.join.FieldMapArray; +import com.linkedin.join.JoinFieldMapping; +import com.linkedin.mxe.MetadataChangeProposal; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; + +import static com.linkedin.metadata.Constants.*; + + +public class JoinUpdateInputMapper + implements InputModelMapper, Urn> { + public static final JoinUpdateInputMapper INSTANCE = new JoinUpdateInputMapper(); + + public static Collection map(@Nonnull final JoinUpdateInput joinUpdateInput, + @Nonnull final Urn actor) { + return INSTANCE.apply(joinUpdateInput, actor); + } + + @Override + public Collection apply(JoinUpdateInput input, Urn actor) { + final Collection proposals = new ArrayList<>(8); + final UpdateMappingHelper updateMappingHelper = new UpdateMappingHelper(JOIN_ENTITY_NAME); + final AuditStamp auditStamp = new AuditStamp(); + auditStamp.setActor(actor, SetMode.IGNORE_NULL); + auditStamp.setTime(System.currentTimeMillis()); + + if (input.getProperties() != null) { + com.linkedin.join.JoinProperties joinProperties = new com.linkedin.join.JoinProperties(); + if (input.getProperties().getName() != null) { + joinProperties.setName(input.getProperties().getName()); + } + try { + if (input.getProperties().getDataSetA() != null) { + joinProperties.setDatasetA(DatasetUrn.createFromString(input.getProperties().getDataSetA())); + } + if (input.getProperties().getDatasetB() != null) { + joinProperties.setDatasetB(DatasetUrn.createFromString(input.getProperties().getDatasetB())); + } + } catch (URISyntaxException e) { + e.printStackTrace(); + } + + if (input.getProperties().getJoinFieldmappings() != null) { + JoinFieldMappingInput joinFieldMapping = input.getProperties().getJoinFieldmappings(); + if (joinFieldMapping.getDetails() != null || (joinFieldMapping.getFieldMapping() != null && joinFieldMapping.getFieldMapping().size() > 0)) { + JoinFieldMapping joinFieldMapping1 = new JoinFieldMapping(); + if (joinFieldMapping.getDetails() != null) { + joinFieldMapping1.setDetails(joinFieldMapping.getDetails()); + } + + if (joinFieldMapping.getFieldMapping() != null && joinFieldMapping.getFieldMapping().size() > 0) { + com.linkedin.join.FieldMapArray fieldMapArray = new FieldMapArray(); + joinFieldMapping.getFieldMapping().forEach(fieldMappingInput -> { + FieldMap fieldMap = new FieldMap(); + if (fieldMappingInput.getAfield() != null) { + fieldMap.setAfield(fieldMappingInput.getAfield()); + } + if (fieldMappingInput.getBfield() != null) { + fieldMap.setBfield(fieldMappingInput.getBfield()); + } + fieldMapArray.add(fieldMap); + }); + joinFieldMapping1.setFieldMapping(fieldMapArray); + } + joinProperties.setJoinFieldMappings(joinFieldMapping1); + } + proposals.add(updateMappingHelper.aspectToProposal(joinProperties, JOIN_PROPERTIES_ASPECT_NAME)); + } + + if (input.getOwnership() != null) { + proposals.add(updateMappingHelper.aspectToProposal(OwnershipUpdateMapper.map(input.getOwnership(), actor), + OWNERSHIP_ASPECT_NAME)); + } + + if (input.getInstitutionalMemory() != null) { + proposals.add(updateMappingHelper.aspectToProposal(InstitutionalMemoryUpdateMapper.map(input.getInstitutionalMemory()), + INSTITUTIONAL_MEMORY_ASPECT_NAME)); + } + + if (input.getTags() != null) { + final GlobalTags globalTags = new GlobalTags(); + if (input.getTags() != null) { + globalTags.setTags(new TagAssociationArray( + input.getTags().getTags().stream().map(TagAssociationUpdateMapper::map).collect(Collectors.toList()))); + } + proposals.add(updateMappingHelper.aspectToProposal(globalTags, GLOBAL_TAGS_ASPECT_NAME)); + } + + if (input.getEditableProperties() != null) { + final EditableJoinProperties editableJoinProperties = new EditableJoinProperties(); + editableJoinProperties.setName(input.getEditableProperties().getName()); + editableJoinProperties.setDescription(input.getEditableProperties().getDescription()); + proposals.add(updateMappingHelper.aspectToProposal(editableJoinProperties, EDITABLE_JOIN_PROPERTIES_ASPECT_NAME)); + } + } + return proposals; + } +} + diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index ee5d6e5e278e3..279d015d363c6 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -69,6 +69,11 @@ type Query { """ tag(urn: String!): Tag + """ + Fetch a Join by primary key (urn) + """ + join(urn: String!): Join + """ Fetch a Glossary Term by primary key (urn) """ @@ -184,6 +189,11 @@ type Query { """ getInviteToken(input: GetInviteTokenInput!): InviteToken + """ + List all Join + """ + listJoin(input: ListJoinInput!): ListJoinResult + """ List all Posts """ @@ -212,6 +222,200 @@ type Query { getQuickFilters(input: GetQuickFiltersInput!): GetQuickFiltersResult } +""" +Input provided when listing existing join +""" +input ListJoinInput { + """ + The starting offset of the result set returned + """ + start: Int + + """ + The maximum number of Roles to be returned in the result set + """ + count: Int + + """ + Optional search query + """ + query: String +} +""" +The result obtained when listing Join +""" +type ListJoinResult { + """ + The starting offset of the result set returned + """ + start: Int! + + """ + The number of Join in the returned result set + """ + count: Int! + + """ + The total number of Join in the result set + """ + total: Int! + + """ + The Join themselves + """ + join: [Join!]! +} + + +""" +A Join is a high-level abstraction that dictates what datasets fields are joined. +""" +type Join implements EntityWithRelationships & Entity & BrowsableEntity { + """ + The primary key of the role + """ + urn: String! + + """ + The standard Entity Type + """ + type: EntityType! + + """ + Unique id for the join + """ + joinId: String! + + """ + An additional set of read only properties + """ + properties: JoinProperties + + """ + An additional set of of read write properties + """ + editableProperties: JoinEditableProperties + + """ + References to internal resources related to the dataset + """ + institutionalMemory: InstitutionalMemory + + """ + Ownership metadata of the dataset + """ + ownership: Ownership + + """ + Status of the Dataset + """ + status: Status + + """ + The parent container in which the entity resides + """ + container: Container + + """ + Recursively get the lineage of containers for this entity + """ + parentContainers: ParentContainersResult + + """ + Tags used for searching dataset + """ + tags: GlobalTags + + """ + The structured glossary terms associated with the dataset + """ + glossaryTerms: GlossaryTerms + + """ + List of relationships between the source Entity and some destination entities with a given types + """ + relationships(input: RelationshipsInput!): EntityRelationshipsResult + """ + Edges extending from this entity grouped by direction in the lineage graph + """ + lineage(input: LineageInput!): EntityLineageResult + + """ + The browse paths corresponding to the dataset. If no Browse Paths have been generated before, this will be null. + """ + browsePaths: [BrowsePath!] +} + +""" +Additional properties about a Join +""" +type JoinEditableProperties { + + """ + Documentation of the Join + """ + description: String + """ + Display name of the Join + """ + name: String +} + +""" +Additional properties about a Join +""" +type JoinProperties { + + """ + The name of the Join used in display + """ + name: String! + """ + The urn of datasetA + """ + datasetA: String! + + """ + The urn of datasetB + """ + datasetB: String! + + """ + The joinFieldMappings + """ + joinFieldMappings: [JoinFieldMapping!] + +} + +""" +Join Field Mapping +""" +type JoinFieldMapping { + """ + The Array of FieldMap + """ + fieldMapping: [FieldMap!] + """ + Any transformation logic or notes pertaining to this specific join + """ + details: String! +} + +""" +Join FieldMap +""" +type FieldMap { + """ + afield + """ + afield: String! + """ + bfield + """ + bfield: String! +} + + """ Root type used for updating DataHub Metadata Coming soon createEntity, addOwner, removeOwner mutations @@ -441,6 +645,31 @@ type Mutation { """ unsetDomain(entityUrn: String!): Boolean + """ + Create a new DataHub View (Saved Filter) + """ + createJoin( + "Input required to create a new Join" + input: JoinUpdateInput!): Join + + """ + Update a Join + """ + updateJoin( + "The urn of the Join to update" + urn: String!, + "Input required to updat an existing DataHub View" + input: JoinUpdateInput!): Join + + """ + Delete a Join + """ + deleteJoin( + "The urn of the Join to delete" + urn: String!): Boolean + + + """ Sets the Deprecation status for a Metadata Entity. Requires the Edit Deprecation status privilege for an entity. """ @@ -677,6 +906,11 @@ enum EntityType { """ DATA_PLATFORM + """ + The Join Entity + """ + JOIN + """ The Dashboard Entity """ @@ -4097,6 +4331,21 @@ input DatasetEditablePropertiesUpdate { description: String! } +""" +Update to writable Dataset fields +""" +input JoinEditablePropertiesUpdate { + """ + Display name of the Join + """ + name: String + + """ + Writable description for Join + """ + description: String! +} + """ Update to writable Chart fields """ @@ -4215,6 +4464,82 @@ input CreateTagInput { description: String } +""" +Input required to create/update a new Join +""" +input JoinUpdateInput { + """ + Details about the Join + """ + properties: JoinPropertiesInput + """ + Update to editable properties + """ + editableProperties: JoinEditablePropertiesUpdate + """ + Update to institutional memory, ie documentation + """ + institutionalMemory: InstitutionalMemoryUpdate + """ + Update to ownership + """ + ownership: OwnershipUpdate + """ + Update to tags + """ + tags: GlobalTagsUpdate +} + +""" +Details about the Join +""" +input JoinPropertiesInput { + """ + Details about the Join + """ + name: String! + """ + Details about the Join + """ + dataSetA: String! + """ + Details about the Join + """ + datasetB: String! + """ + Details about the Join + """ + joinFieldmappings: JoinFieldMappingInput! + +} + +""" +Details about the Join +""" +input JoinFieldMappingInput { + """ + Details about the Join + """ + fieldMapping: [FieldMappingInput] + """ + Details about the Join + """ + details: String +} +""" +Details about the Join +""" +input FieldMappingInput { + """ + Details about the Join + """ + afield: String + """ + Details about the Join + """ + bfield: String +} + """ An update for the ownership information for a Metadata Entity """ diff --git a/datahub-web-react/src/graphql/fragments.graphql b/datahub-web-react/src/graphql/fragments.graphql index 0df857510fdb3..a0799c473d35d 100644 --- a/datahub-web-react/src/graphql/fragments.graphql +++ b/datahub-web-react/src/graphql/fragments.graphql @@ -907,3 +907,25 @@ fragment inputFieldsFields on InputFields { } } } + +fragment joinPropertiesFields on JoinProperties { + name + datasetA + datasetB + joinFieldMappings { + ...joinFieldMappingFields + } +} + +fragment joinFieldMappingFields on JoinFieldMapping { + fieldMapping { + afield + bfield + } + details +} + +fragment joinEditablePropertiesFields on JoinEditableProperties { + description + name +} \ No newline at end of file diff --git a/datahub-web-react/src/graphql/join.graphql b/datahub-web-react/src/graphql/join.graphql new file mode 100644 index 0000000000000..5b36778cb3d17 --- /dev/null +++ b/datahub-web-react/src/graphql/join.graphql @@ -0,0 +1,42 @@ +query getJoin($urn: String!) { + join(urn: $urn) { + urn + type + joinId + properties { + ...joinPropertiesFields + } + editableProperties { + ...joinEditablePropertiesFields + } + institutionalMemory { + ...institutionalMemoryFields + } + ownership { + ...ownershipFields + } + status { + removed + } + container { + ...entityContainer + } + parentContainers { + ...parentContainersFields + } + tags { + ...globalTagsFields + } + glossaryTerms { + ...glossaryTerms + } + outgoing: relationships( + input: { types: ["JoinA", "JoinB"], direction: OUTGOING, start: 0, count: 100 } + ) { + ...fullRelationshipResults + } + browsePaths { + path + } + } +} diff --git a/datahub-web-react/src/graphql/lineage.graphql b/datahub-web-react/src/graphql/lineage.graphql index fb8d6daa02824..5416cd77fdf51 100644 --- a/datahub-web-react/src/graphql/lineage.graphql +++ b/datahub-web-react/src/graphql/lineage.graphql @@ -224,6 +224,17 @@ fragment lineageNodeProperties on EntityWithRelationships { ... on MLPrimaryKey { ...nonRecursiveMLPrimaryKey } + ... on Join { + urn + type + joinId + properties { + ...joinPropertiesFields + } + editableProperties { + ...joinEditablePropertiesFields + } + } } fragment lineageFields on EntityWithRelationships { diff --git a/datahub-web-react/src/graphql/search.graphql b/datahub-web-react/src/graphql/search.graphql index 3f94417985961..8cc09a1ffc536 100644 --- a/datahub-web-react/src/graphql/search.graphql +++ b/datahub-web-react/src/graphql/search.graphql @@ -631,6 +631,17 @@ fragment searchResultFields on Entity { ...parentContainersFields } } + ... on Join { + urn + type + joinId + properties { + ...joinPropertiesFields + } + editableProperties { + ...joinEditablePropertiesFields + } + } ... on MLFeatureTable { name description @@ -707,6 +718,17 @@ fragment searchResultFields on Entity { ... on DataPlatform { ...nonConflictingPlatformFields } + ... on Join { + urn + type + joinId + properties { + ...joinPropertiesFields + } + editableProperties { + ...joinEditablePropertiesFields + } + } } fragment facetFields on FacetMetadata { @@ -802,6 +824,28 @@ fragment schemaFieldEntityFields on SchemaFieldEntity { } } +fragment searchResultsWithRelationships on SearchResults { + start + count + total + searchResults { + entity { + ...searchResultFieldsWithRelationships + } + matchedFields { + name + value + } + insights { + text + icon + } + } + facets { + ...facetFields + } +} + fragment searchAcrossRelationshipResults on SearchAcrossLineageResults { start count @@ -850,12 +894,23 @@ query getSearchResults($input: SearchInput!) { } } +query getSearchResultsWithRelationships($input: SearchInput!) { + search(input: $input) { + ...searchResultsWithRelationships + } +} + query getSearchResultsForMultiple($input: SearchAcrossEntitiesInput!) { searchAcrossEntities(input: $input) { ...searchResults } } +query getSearchResultsForMultipleWithRelationships($input: SearchAcrossEntitiesInput!) { + searchAcrossEntities(input: $input) { + ...searchResultsWithRelationships + } +} query searchAcrossLineage($input: SearchAcrossLineageInput!, $includeAssertions: Boolean = false) { searchAcrossLineage(input: $input) { ...searchAcrossRelationshipResults diff --git a/li-utils/src/main/javaPegasus/com/linkedin/common/urn/JoinUrn.java b/li-utils/src/main/javaPegasus/com/linkedin/common/urn/JoinUrn.java index ed92bca555eea..6b7833ca99032 100644 --- a/li-utils/src/main/javaPegasus/com/linkedin/common/urn/JoinUrn.java +++ b/li-utils/src/main/javaPegasus/com/linkedin/common/urn/JoinUrn.java @@ -11,12 +11,12 @@ public class JoinUrn extends Urn { private final String _joinId; - public JoinUrn(String name) { - super(ENTITY_TYPE, TupleKey.create(name)); - this._joinId = name; + public JoinUrn(String joinId) { + super(ENTITY_TYPE, TupleKey.create(joinId)); + this._joinId = joinId; } - public String getName() { + public String getJoinIdEntity() { return _joinId; } diff --git a/metadata-models/src/main/pegasus/com/linkedin/join/FieldMap.pdl b/metadata-models/src/main/pegasus/com/linkedin/join/FieldMap.pdl index 294dbe70e7684..c979aade06e95 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/join/FieldMap.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/join/FieldMap.pdl @@ -9,11 +9,21 @@ record FieldMap { /** * All fields from dataset A that are required for the join, maps to bFields 1:1 */ + @Searchable = { + "fieldType": "TEXT_PARTIAL", + "enableAutocomplete": true, + "boostScore": 10.0 + } afield: SchemaFieldPath /** * All fields from dataset B that are required for the join, maps to aFields 1:1 */ + @Searchable = { + "fieldType": "TEXT_PARTIAL", + "enableAutocomplete": true, + "boostScore": 10.0 + } bfield: SchemaFieldPath } \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/join/JoinFieldMapping.pdl b/metadata-models/src/main/pegasus/com/linkedin/join/JoinFieldMapping.pdl index f875f36f42037..01517dcebebc2 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/join/JoinFieldMapping.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/join/JoinFieldMapping.pdl @@ -6,6 +6,7 @@ import com.linkedin.dataset.SchemaFieldPath * Field Mapping about a join between two datasets */ record JoinFieldMapping { + /** * All fields from dataset A that are required for the join to dataset B */ @@ -14,5 +15,10 @@ record JoinFieldMapping { /** * Any transformation logic or notes pertaining to this specific join */ + @Searchable = { + "fieldType": "TEXT_PARTIAL", + "enableAutocomplete": true, + "boostScore": 10.0 + } details: string } \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/join/JoinProperties.pdl b/metadata-models/src/main/pegasus/com/linkedin/join/JoinProperties.pdl index 0f7449d582c2b..41bd7c843ce2e 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/join/JoinProperties.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/join/JoinProperties.pdl @@ -13,7 +13,7 @@ import com.linkedin.common.CustomProperties record JoinProperties includes CustomProperties { /** - * Display name of the Join + * Name of the Join */ @Searchable = { "fieldType": "TEXT_PARTIAL", @@ -29,6 +29,11 @@ record JoinProperties includes CustomProperties { "name": "joinA", "entityTypes": [ "dataset" ] } + @Searchable = { + "fieldType": "TEXT_PARTIAL", + "enableAutocomplete": true, + "boostScore": 10.0 + } datasetA: DatasetUrn /** @@ -38,12 +43,17 @@ record JoinProperties includes CustomProperties { "name": "joinB", "entityTypes": [ "dataset" ] } + @Searchable = { + "fieldType": "TEXT_PARTIAL", + "enableAutocomplete": true, + "boostScore": 10.0 + } datasetB: DatasetUrn /** - * Array of JoinFieldMapping + * JoinFieldMapping (in future we can make it an array) */ - joinFieldMappings: array[JoinFieldMapping] + joinFieldMappings: JoinFieldMapping /** * A timestamp documenting when the asset was created in the source Data Platform (not on DataHub) diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/Snapshot.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/Snapshot.pdl index 9cf3e325967ed..374bdb6cd9906 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/Snapshot.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/Snapshot.pdl @@ -25,5 +25,5 @@ typeref Snapshot = union[ DataHubPolicySnapshot, SchemaFieldSnapshot, DataHubRetentionSnapshot, - JoinSnapshot, + JoinSnapshot, ] From 89e878292ea5086910c29253fb7d82057b301d0e Mon Sep 17 00:00:00 2001 From: Raj Tekal Date: Wed, 31 May 2023 11:38:39 -0400 Subject: [PATCH 4/5] Graphql change and extraneous code removed --- .../datahub/graphql/types/join/JoinType.java | 27 +- .../src/main/resources/entity.graphql | 3 +- .../com.linkedin.entity.aspects.snapshot.json | 5 +- ...com.linkedin.entity.entities.snapshot.json | 247 +++++++++++++++++- .../com.linkedin.entity.runs.snapshot.json | 5 +- ...m.linkedin.platform.platform.snapshot.json | 243 ++++++++++++++++- 6 files changed, 496 insertions(+), 34 deletions(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/JoinType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/JoinType.java index 1164354966ee8..1659630f31a28 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/JoinType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/JoinType.java @@ -13,11 +13,9 @@ import com.linkedin.datahub.graphql.generated.Entity; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.FacetFilterInput; -import com.linkedin.datahub.graphql.generated.FieldSortInput; import com.linkedin.datahub.graphql.generated.Join; import com.linkedin.datahub.graphql.generated.JoinUpdateInput; import com.linkedin.datahub.graphql.generated.SearchResults; -import com.linkedin.datahub.graphql.generated.Sort; import com.linkedin.datahub.graphql.resolvers.ResolverUtils; import com.linkedin.datahub.graphql.types.BrowsableEntityType; import com.linkedin.datahub.graphql.types.MutableType; @@ -32,8 +30,8 @@ import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.browse.BrowseResult; import com.linkedin.metadata.query.AutoCompleteResult; -import com.linkedin.metadata.query.filter.SortCriterion; -import com.linkedin.metadata.query.filter.SortOrder; +import com.linkedin.metadata.query.SearchFlags; +import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.search.SearchResult; import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.r2.RemoteInvocationException; @@ -177,32 +175,21 @@ private com.linkedin.common.urn.JoinUrn getJoinUrn(String urnStr) { @Override public SearchResults search(@Nonnull String query, @Nullable List filters, - @Nullable FieldSortInput sort, int start, int count, @Nonnull QueryContext context) throws Exception { + int start, int count, @Nonnull QueryContext context) throws Exception { final Map facetFilters = ResolverUtils.buildFacetFilters(filters, FACET_FIELDS); - String sortField = sort != null ? sort.getField() : null; - SortOrder sortOrder = sort != null ? (sort.getSortOrder().equals(Sort.asc) ? SortOrder.ASCENDING : SortOrder.DESCENDING) : null; - SortCriterion sortCriterion = null; - if (sortField != null && sortOrder != null) { - sortCriterion = - new SortCriterion().setField(sortField).setOrder(sortOrder); - } - final SearchResult searchResult = _entityClient.search(ENTITY_NAME, query, facetFilters, sortCriterion, start, - count, context.getAuthentication()); + final SearchResult searchResult = _entityClient.search(ENTITY_NAME, query, facetFilters, start, + count, context.getAuthentication(), new SearchFlags().setFulltext(true)); return UrnSearchResultsMapper.map(searchResult); } @Override public AutoCompleteResults autoComplete(@Nonnull String query, @Nullable String field, - @Nullable List filters, int limit, @Nonnull QueryContext context) throws Exception { - final Map facetFilters = ResolverUtils.buildFacetFilters(filters, FACET_FIELDS); - final AutoCompleteResult result = _entityClient.autoComplete(ENTITY_NAME, query, facetFilters, limit, context.getAuthentication()); + @Nullable Filter filters, int limit, @Nonnull QueryContext context) throws Exception { + final AutoCompleteResult result = _entityClient.autoComplete(ENTITY_NAME, query, filters, limit, context.getAuthentication()); return AutoCompleteResultsMapper.map(result); } - - - @Override public Join update(String urn, @Nonnull JoinUpdateInput input, @Nonnull QueryContext context) throws Exception { diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index 279d015d363c6..7afb3b9c2283a 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -383,7 +383,7 @@ type JoinProperties { """ The joinFieldMappings """ - joinFieldMappings: [JoinFieldMapping!] + joinFieldMappings: JoinFieldMapping! } @@ -415,7 +415,6 @@ type FieldMap { bfield: String! } - """ Root type used for updating DataHub Metadata Coming soon createEntity, addOwner, removeOwner mutations diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json index 67de18770786b..f6f84ebe94035 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json @@ -1777,7 +1777,7 @@ } ] } }, - "doc" : "Fine-grained column-level lineages", + "doc" : "Fine-grained column-level lineages\nNot currently supported in the UI\nUse UpstreamLineage aspect for datasets to express Column Level Lineage for the UI", "optional" : true } ], "Aspect" : { @@ -2743,7 +2743,8 @@ "Searchable" : { "boostScore" : 5.0, "fieldName" : "fieldPaths", - "fieldType" : "TEXT" + "fieldType" : "TEXT", + "queryByDefault" : "true" } }, { "name" : "jsonPath", diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json index 4918985a200e1..e0f51a5bf1de3 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json @@ -1006,6 +1006,34 @@ "name" : "institutionalMemory" } }, "com.linkedin.common.InstitutionalMemoryMetadata", { + "type" : "typeref", + "name" : "JoinUrn", + "namespace" : "com.linkedin.common", + "doc" : "Standardized join identifier.", + "ref" : "string", + "java" : { + "class" : "com.linkedin.common.urn.JoinUrn" + }, + "validate" : { + "com.linkedin.common.validator.TypedUrnValidator" : { + "accessible" : true, + "constructable" : true, + "doc" : "Standardized Join identifier.", + "entityType" : "join", + "fields" : [ { + "doc" : "Join native name e.g. .
, /dir/subdir/, or ", + "maxLength" : 284, + "name" : "joinId", + "type" : "string" + } ], + "maxLength" : 284, + "name" : "Join", + "namespace" : "li", + "owners" : [ "urn:li:corpuser:fbar", "urn:li:corpuser:bfoo" ], + "owningTeam" : "urn:li:internalTeam:datahub" + } + } + }, { "type" : "enum", "name" : "MLFeatureDataType", "namespace" : "com.linkedin.common", @@ -1257,6 +1285,30 @@ "optional" : true } ] }, "com.linkedin.common.fieldtransformer.TransformationType", "com.linkedin.common.fieldtransformer.UDFTransformer", { + "type" : "record", + "name" : "Container", + "namespace" : "com.linkedin.container", + "doc" : "Link from an asset to its parent container", + "fields" : [ { + "name" : "container", + "type" : "com.linkedin.common.Urn", + "doc" : "The parent container of an asset", + "Relationship" : { + "entityTypes" : [ "container" ], + "name" : "IsPartOf" + }, + "Searchable" : { + "addToFilters" : true, + "fieldName" : "container", + "fieldType" : "URN", + "filterNameOverride" : "Container", + "hasValuesFieldName" : "hasContainer" + } + } ], + "Aspect" : { + "name" : "container" + } + }, { "type" : "record", "name" : "DashboardInfo", "namespace" : "com.linkedin.dashboard", @@ -1806,7 +1858,7 @@ } ] } }, - "doc" : "Fine-grained column-level lineages", + "doc" : "Fine-grained column-level lineages\nNot currently supported in the UI\nUse UpstreamLineage aspect for datasets to express Column Level Lineage for the UI", "optional" : true } ], "Aspect" : { @@ -3103,7 +3155,8 @@ "Searchable" : { "boostScore" : 5.0, "fieldName" : "fieldPaths", - "fieldType" : "TEXT" + "fieldType" : "TEXT", + "queryByDefault" : "true" } }, { "name" : "jsonPath", @@ -5401,10 +5454,194 @@ "keyAspect" : "dataHubRetentionKey", "name" : "dataHubRetention" } + }, { + "type" : "record", + "name" : "JoinSnapshot", + "doc" : "A metadata snapshot for a specific join entity.", + "fields" : [ { + "name" : "urn", + "type" : "com.linkedin.common.JoinUrn", + "doc" : "URN for the entity the metadata snapshot is associated with." + }, { + "name" : "aspects", + "type" : { + "type" : "array", + "items" : { + "type" : "typeref", + "name" : "JoinAspect", + "namespace" : "com.linkedin.metadata.aspect", + "doc" : "A union of all supported metadata aspects for a Join", + "ref" : [ { + "type" : "record", + "name" : "JoinKey", + "namespace" : "com.linkedin.metadata.key", + "doc" : "Key for a Join", + "fields" : [ { + "name" : "joinId", + "type" : "string", + "Searchable" : { + "fieldType" : "TEXT" + } + } ], + "Aspect" : { + "name" : "joinKey" + } + }, { + "type" : "record", + "name" : "JoinProperties", + "namespace" : "com.linkedin.join", + "doc" : "Properties associated with a Join", + "include" : [ "com.linkedin.common.CustomProperties" ], + "fields" : [ { + "name" : "name", + "type" : "string", + "doc" : "Name of the Join", + "Searchable" : { + "boostScore" : 10.0, + "enableAutocomplete" : true, + "fieldType" : "TEXT_PARTIAL" + } + }, { + "name" : "datasetA", + "type" : "com.linkedin.common.DatasetUrn", + "doc" : "First dataset in the join (no directionality)", + "Relationship" : { + "entityTypes" : [ "dataset" ], + "name" : "joinA" + }, + "Searchable" : { + "boostScore" : 10.0, + "enableAutocomplete" : true, + "fieldType" : "TEXT_PARTIAL" + } + }, { + "name" : "datasetB", + "type" : "com.linkedin.common.DatasetUrn", + "doc" : "Second dataset in the join (no directionality)", + "Relationship" : { + "entityTypes" : [ "dataset" ], + "name" : "joinB" + }, + "Searchable" : { + "boostScore" : 10.0, + "enableAutocomplete" : true, + "fieldType" : "TEXT_PARTIAL" + } + }, { + "name" : "joinFieldMappings", + "type" : { + "type" : "record", + "name" : "JoinFieldMapping", + "doc" : "Field Mapping about a join between two datasets", + "fields" : [ { + "name" : "fieldMapping", + "type" : { + "type" : "array", + "items" : { + "type" : "record", + "name" : "FieldMap", + "doc" : "Field Mapping of 1:1 field", + "fields" : [ { + "name" : "afield", + "type" : "com.linkedin.dataset.SchemaFieldPath", + "doc" : "All fields from dataset A that are required for the join, maps to bFields 1:1", + "Searchable" : { + "boostScore" : 10.0, + "enableAutocomplete" : true, + "fieldType" : "TEXT_PARTIAL" + } + }, { + "name" : "bfield", + "type" : "com.linkedin.dataset.SchemaFieldPath", + "doc" : "All fields from dataset B that are required for the join, maps to aFields 1:1", + "Searchable" : { + "boostScore" : 10.0, + "enableAutocomplete" : true, + "fieldType" : "TEXT_PARTIAL" + } + } ] + } + }, + "doc" : "All fields from dataset A that are required for the join to dataset B" + }, { + "name" : "details", + "type" : "string", + "doc" : " Any transformation logic or notes pertaining to this specific join", + "Searchable" : { + "boostScore" : 10.0, + "enableAutocomplete" : true, + "fieldType" : "TEXT_PARTIAL" + } + } ] + }, + "doc" : "JoinFieldMapping (in future we can make it an array)" + }, { + "name" : "created", + "type" : "com.linkedin.common.TimeStamp", + "doc" : "A timestamp documenting when the asset was created in the source Data Platform (not on DataHub)", + "optional" : true, + "Searchable" : { + "/time" : { + "fieldName" : "createdAt", + "fieldType" : "DATETIME" + } + } + }, { + "name" : "lastModified", + "type" : "com.linkedin.common.TimeStamp", + "doc" : "A timestamp documenting when the asset was last modified in the source Data Platform (not on DataHub)", + "optional" : true, + "Searchable" : { + "/time" : { + "fieldName" : "lastModifiedAt", + "fieldType" : "DATETIME" + } + } + } ], + "Aspect" : { + "name" : "joinProperties" + } + }, { + "type" : "record", + "name" : "EditableJoinProperties", + "namespace" : "com.linkedin.join", + "doc" : "EditableJoinProperties stores editable changes made to join properties. This separates changes made from\ningestion pipelines and edits in the UI to avoid accidental overwrites of user-provided data by ingestion pipelines", + "include" : [ "com.linkedin.common.ChangeAuditStamps" ], + "fields" : [ { + "name" : "description", + "type" : "string", + "doc" : "Documentation of the join", + "optional" : true, + "Searchable" : { + "fieldName" : "editedDescription", + "fieldType" : "TEXT" + } + }, { + "name" : "name", + "type" : "string", + "doc" : "Display name of the Join", + "optional" : true, + "Searchable" : { + "fieldName" : "editedName", + "fieldType" : "TEXT_PARTIAL" + } + } ], + "Aspect" : { + "name" : "editableJoinProperties" + } + }, "com.linkedin.common.InstitutionalMemory", "com.linkedin.common.Ownership", "com.linkedin.common.Status", "com.linkedin.container.Container", "com.linkedin.common.GlobalTags", "com.linkedin.common.GlossaryTerms", "com.linkedin.common.BrowsePaths" ] + } + }, + "doc" : "The list of metadata aspects associated with the Join. Depending on the use case, this can either be all, or a selection, of supported aspects." + } ], + "Entity" : { + "keyAspect" : "joinKey", + "name" : "join" + } } ] } } ] - }, "com.linkedin.glossary.GlossaryNodeInfo", "com.linkedin.glossary.GlossaryRelatedTerms", "com.linkedin.glossary.GlossaryTermInfo", "com.linkedin.identity.CorpGroupInfo", "com.linkedin.identity.CorpUserEditableInfo", "com.linkedin.identity.CorpUserInfo", "com.linkedin.identity.CorpUserStatus", "com.linkedin.identity.GroupMembership", "com.linkedin.metadata.aspect.ChartAspect", "com.linkedin.metadata.aspect.CorpGroupAspect", "com.linkedin.metadata.aspect.CorpUserAspect", "com.linkedin.metadata.aspect.DashboardAspect", "com.linkedin.metadata.aspect.DataFlowAspect", "com.linkedin.metadata.aspect.DataHubPolicyAspect", "com.linkedin.metadata.aspect.DataHubRetentionAspect", "com.linkedin.metadata.aspect.DataJobAspect", "com.linkedin.metadata.aspect.DataPlatformAspect", "com.linkedin.metadata.aspect.DataProcessAspect", "com.linkedin.metadata.aspect.DatasetAspect", "com.linkedin.metadata.aspect.GlossaryNodeAspect", "com.linkedin.metadata.aspect.GlossaryTermAspect", "com.linkedin.metadata.aspect.MLFeatureAspect", "com.linkedin.metadata.aspect.MLFeatureTableAspect", "com.linkedin.metadata.aspect.MLModelAspect", "com.linkedin.metadata.aspect.MLModelDeploymentAspect", "com.linkedin.metadata.aspect.MLModelGroupAspect", "com.linkedin.metadata.aspect.MLPrimaryKeyAspect", "com.linkedin.metadata.aspect.SchemaFieldAspect", "com.linkedin.metadata.aspect.TagAspect", { + }, "com.linkedin.glossary.GlossaryNodeInfo", "com.linkedin.glossary.GlossaryRelatedTerms", "com.linkedin.glossary.GlossaryTermInfo", "com.linkedin.identity.CorpGroupInfo", "com.linkedin.identity.CorpUserEditableInfo", "com.linkedin.identity.CorpUserInfo", "com.linkedin.identity.CorpUserStatus", "com.linkedin.identity.GroupMembership", "com.linkedin.join.EditableJoinProperties", "com.linkedin.join.FieldMap", "com.linkedin.join.JoinFieldMapping", "com.linkedin.join.JoinProperties", "com.linkedin.metadata.aspect.ChartAspect", "com.linkedin.metadata.aspect.CorpGroupAspect", "com.linkedin.metadata.aspect.CorpUserAspect", "com.linkedin.metadata.aspect.DashboardAspect", "com.linkedin.metadata.aspect.DataFlowAspect", "com.linkedin.metadata.aspect.DataHubPolicyAspect", "com.linkedin.metadata.aspect.DataHubRetentionAspect", "com.linkedin.metadata.aspect.DataJobAspect", "com.linkedin.metadata.aspect.DataPlatformAspect", "com.linkedin.metadata.aspect.DataProcessAspect", "com.linkedin.metadata.aspect.DatasetAspect", "com.linkedin.metadata.aspect.GlossaryNodeAspect", "com.linkedin.metadata.aspect.GlossaryTermAspect", "com.linkedin.metadata.aspect.JoinAspect", "com.linkedin.metadata.aspect.MLFeatureAspect", "com.linkedin.metadata.aspect.MLFeatureTableAspect", "com.linkedin.metadata.aspect.MLModelAspect", "com.linkedin.metadata.aspect.MLModelDeploymentAspect", "com.linkedin.metadata.aspect.MLModelGroupAspect", "com.linkedin.metadata.aspect.MLPrimaryKeyAspect", "com.linkedin.metadata.aspect.SchemaFieldAspect", "com.linkedin.metadata.aspect.TagAspect", { "type" : "record", "name" : "BrowseResult", "namespace" : "com.linkedin.metadata.browse", @@ -5488,7 +5725,7 @@ "type" : "int", "doc" : "The total number of elements (entities + groups) directly under queried path" } ] - }, "com.linkedin.metadata.browse.BrowseResultEntity", "com.linkedin.metadata.browse.BrowseResultGroup", "com.linkedin.metadata.browse.BrowseResultMetadata", "com.linkedin.metadata.key.ChartKey", "com.linkedin.metadata.key.CorpGroupKey", "com.linkedin.metadata.key.CorpUserKey", "com.linkedin.metadata.key.DashboardKey", "com.linkedin.metadata.key.DataFlowKey", "com.linkedin.metadata.key.DataHubPolicyKey", "com.linkedin.metadata.key.DataHubRetentionKey", "com.linkedin.metadata.key.DataJobKey", "com.linkedin.metadata.key.DataPlatformKey", "com.linkedin.metadata.key.DataProcessKey", "com.linkedin.metadata.key.DatasetKey", "com.linkedin.metadata.key.GlossaryNodeKey", "com.linkedin.metadata.key.GlossaryTermKey", "com.linkedin.metadata.key.MLFeatureKey", "com.linkedin.metadata.key.MLFeatureTableKey", "com.linkedin.metadata.key.MLModelDeploymentKey", "com.linkedin.metadata.key.MLModelGroupKey", "com.linkedin.metadata.key.MLModelKey", "com.linkedin.metadata.key.MLPrimaryKeyKey", "com.linkedin.metadata.key.SchemaFieldKey", "com.linkedin.metadata.key.TagKey", { + }, "com.linkedin.metadata.browse.BrowseResultEntity", "com.linkedin.metadata.browse.BrowseResultGroup", "com.linkedin.metadata.browse.BrowseResultMetadata", "com.linkedin.metadata.key.ChartKey", "com.linkedin.metadata.key.CorpGroupKey", "com.linkedin.metadata.key.CorpUserKey", "com.linkedin.metadata.key.DashboardKey", "com.linkedin.metadata.key.DataFlowKey", "com.linkedin.metadata.key.DataHubPolicyKey", "com.linkedin.metadata.key.DataHubRetentionKey", "com.linkedin.metadata.key.DataJobKey", "com.linkedin.metadata.key.DataPlatformKey", "com.linkedin.metadata.key.DataProcessKey", "com.linkedin.metadata.key.DatasetKey", "com.linkedin.metadata.key.GlossaryNodeKey", "com.linkedin.metadata.key.GlossaryTermKey", "com.linkedin.metadata.key.JoinKey", "com.linkedin.metadata.key.MLFeatureKey", "com.linkedin.metadata.key.MLFeatureTableKey", "com.linkedin.metadata.key.MLModelDeploymentKey", "com.linkedin.metadata.key.MLModelGroupKey", "com.linkedin.metadata.key.MLModelKey", "com.linkedin.metadata.key.MLPrimaryKeyKey", "com.linkedin.metadata.key.SchemaFieldKey", "com.linkedin.metadata.key.TagKey", { "type" : "record", "name" : "AnyResult", "namespace" : "com.linkedin.metadata.query", @@ -6108,7 +6345,7 @@ "type" : "int", "doc" : "The total number of entities directly under searched path" } ] - }, "com.linkedin.metadata.search.SearchResultMetadata", "com.linkedin.metadata.snapshot.ChartSnapshot", "com.linkedin.metadata.snapshot.CorpGroupSnapshot", "com.linkedin.metadata.snapshot.CorpUserSnapshot", "com.linkedin.metadata.snapshot.DashboardSnapshot", "com.linkedin.metadata.snapshot.DataFlowSnapshot", "com.linkedin.metadata.snapshot.DataHubPolicySnapshot", "com.linkedin.metadata.snapshot.DataHubRetentionSnapshot", "com.linkedin.metadata.snapshot.DataJobSnapshot", "com.linkedin.metadata.snapshot.DataPlatformSnapshot", "com.linkedin.metadata.snapshot.DataProcessSnapshot", "com.linkedin.metadata.snapshot.DatasetSnapshot", "com.linkedin.metadata.snapshot.GlossaryNodeSnapshot", "com.linkedin.metadata.snapshot.GlossaryTermSnapshot", "com.linkedin.metadata.snapshot.MLFeatureSnapshot", "com.linkedin.metadata.snapshot.MLFeatureTableSnapshot", "com.linkedin.metadata.snapshot.MLModelDeploymentSnapshot", "com.linkedin.metadata.snapshot.MLModelGroupSnapshot", "com.linkedin.metadata.snapshot.MLModelSnapshot", "com.linkedin.metadata.snapshot.MLPrimaryKeySnapshot", "com.linkedin.metadata.snapshot.SchemaFieldSnapshot", "com.linkedin.metadata.snapshot.Snapshot", "com.linkedin.metadata.snapshot.TagSnapshot", "com.linkedin.ml.metadata.BaseData", "com.linkedin.ml.metadata.CaveatDetails", "com.linkedin.ml.metadata.CaveatsAndRecommendations", "com.linkedin.ml.metadata.DeploymentStatus", "com.linkedin.ml.metadata.EthicalConsiderations", "com.linkedin.ml.metadata.EvaluationData", "com.linkedin.ml.metadata.HyperParameterValueType", "com.linkedin.ml.metadata.IntendedUse", "com.linkedin.ml.metadata.IntendedUserType", "com.linkedin.ml.metadata.MLFeatureProperties", "com.linkedin.ml.metadata.MLFeatureTableProperties", "com.linkedin.ml.metadata.MLHyperParam", "com.linkedin.ml.metadata.MLMetric", "com.linkedin.ml.metadata.MLModelDeploymentProperties", "com.linkedin.ml.metadata.MLModelFactorPrompts", "com.linkedin.ml.metadata.MLModelFactors", "com.linkedin.ml.metadata.MLModelGroupProperties", "com.linkedin.ml.metadata.MLModelProperties", "com.linkedin.ml.metadata.MLPrimaryKeyProperties", "com.linkedin.ml.metadata.Metrics", "com.linkedin.ml.metadata.QuantitativeAnalyses", "com.linkedin.ml.metadata.ResultsType", "com.linkedin.ml.metadata.SourceCode", "com.linkedin.ml.metadata.SourceCodeUrl", "com.linkedin.ml.metadata.SourceCodeUrlType", "com.linkedin.ml.metadata.TrainingData", { + }, "com.linkedin.metadata.search.SearchResultMetadata", "com.linkedin.metadata.snapshot.ChartSnapshot", "com.linkedin.metadata.snapshot.CorpGroupSnapshot", "com.linkedin.metadata.snapshot.CorpUserSnapshot", "com.linkedin.metadata.snapshot.DashboardSnapshot", "com.linkedin.metadata.snapshot.DataFlowSnapshot", "com.linkedin.metadata.snapshot.DataHubPolicySnapshot", "com.linkedin.metadata.snapshot.DataHubRetentionSnapshot", "com.linkedin.metadata.snapshot.DataJobSnapshot", "com.linkedin.metadata.snapshot.DataPlatformSnapshot", "com.linkedin.metadata.snapshot.DataProcessSnapshot", "com.linkedin.metadata.snapshot.DatasetSnapshot", "com.linkedin.metadata.snapshot.GlossaryNodeSnapshot", "com.linkedin.metadata.snapshot.GlossaryTermSnapshot", "com.linkedin.metadata.snapshot.JoinSnapshot", "com.linkedin.metadata.snapshot.MLFeatureSnapshot", "com.linkedin.metadata.snapshot.MLFeatureTableSnapshot", "com.linkedin.metadata.snapshot.MLModelDeploymentSnapshot", "com.linkedin.metadata.snapshot.MLModelGroupSnapshot", "com.linkedin.metadata.snapshot.MLModelSnapshot", "com.linkedin.metadata.snapshot.MLPrimaryKeySnapshot", "com.linkedin.metadata.snapshot.SchemaFieldSnapshot", "com.linkedin.metadata.snapshot.Snapshot", "com.linkedin.metadata.snapshot.TagSnapshot", "com.linkedin.ml.metadata.BaseData", "com.linkedin.ml.metadata.CaveatDetails", "com.linkedin.ml.metadata.CaveatsAndRecommendations", "com.linkedin.ml.metadata.DeploymentStatus", "com.linkedin.ml.metadata.EthicalConsiderations", "com.linkedin.ml.metadata.EvaluationData", "com.linkedin.ml.metadata.HyperParameterValueType", "com.linkedin.ml.metadata.IntendedUse", "com.linkedin.ml.metadata.IntendedUserType", "com.linkedin.ml.metadata.MLFeatureProperties", "com.linkedin.ml.metadata.MLFeatureTableProperties", "com.linkedin.ml.metadata.MLHyperParam", "com.linkedin.ml.metadata.MLMetric", "com.linkedin.ml.metadata.MLModelDeploymentProperties", "com.linkedin.ml.metadata.MLModelFactorPrompts", "com.linkedin.ml.metadata.MLModelFactors", "com.linkedin.ml.metadata.MLModelGroupProperties", "com.linkedin.ml.metadata.MLModelProperties", "com.linkedin.ml.metadata.MLPrimaryKeyProperties", "com.linkedin.ml.metadata.Metrics", "com.linkedin.ml.metadata.QuantitativeAnalyses", "com.linkedin.ml.metadata.ResultsType", "com.linkedin.ml.metadata.SourceCode", "com.linkedin.ml.metadata.SourceCodeUrl", "com.linkedin.ml.metadata.SourceCodeUrlType", "com.linkedin.ml.metadata.TrainingData", { "type" : "record", "name" : "SystemMetadata", "namespace" : "com.linkedin.mxe", diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.runs.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.runs.snapshot.json index 252ddde6081f8..915b7cd64bfde 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.runs.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.runs.snapshot.json @@ -1530,7 +1530,7 @@ } ] } }, - "doc" : "Fine-grained column-level lineages", + "doc" : "Fine-grained column-level lineages\nNot currently supported in the UI\nUse UpstreamLineage aspect for datasets to express Column Level Lineage for the UI", "optional" : true } ], "Aspect" : { @@ -2488,7 +2488,8 @@ "Searchable" : { "boostScore" : 5.0, "fieldName" : "fieldPaths", - "fieldType" : "TEXT" + "fieldType" : "TEXT", + "queryByDefault" : "true" } }, { "name" : "jsonPath", diff --git a/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json b/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json index c216317131112..c16ea02d007c5 100644 --- a/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json +++ b/metadata-service/restli-api/src/main/snapshot/com.linkedin.platform.platform.snapshot.json @@ -1006,6 +1006,34 @@ "name" : "institutionalMemory" } }, "com.linkedin.common.InstitutionalMemoryMetadata", { + "type" : "typeref", + "name" : "JoinUrn", + "namespace" : "com.linkedin.common", + "doc" : "Standardized join identifier.", + "ref" : "string", + "java" : { + "class" : "com.linkedin.common.urn.JoinUrn" + }, + "validate" : { + "com.linkedin.common.validator.TypedUrnValidator" : { + "accessible" : true, + "constructable" : true, + "doc" : "Standardized Join identifier.", + "entityType" : "join", + "fields" : [ { + "doc" : "Join native name e.g. .
, /dir/subdir/, or ", + "maxLength" : 284, + "name" : "joinId", + "type" : "string" + } ], + "maxLength" : 284, + "name" : "Join", + "namespace" : "li", + "owners" : [ "urn:li:corpuser:fbar", "urn:li:corpuser:bfoo" ], + "owningTeam" : "urn:li:internalTeam:datahub" + } + } + }, { "type" : "enum", "name" : "MLFeatureDataType", "namespace" : "com.linkedin.common", @@ -1257,6 +1285,30 @@ "optional" : true } ] }, "com.linkedin.common.fieldtransformer.TransformationType", "com.linkedin.common.fieldtransformer.UDFTransformer", { + "type" : "record", + "name" : "Container", + "namespace" : "com.linkedin.container", + "doc" : "Link from an asset to its parent container", + "fields" : [ { + "name" : "container", + "type" : "com.linkedin.common.Urn", + "doc" : "The parent container of an asset", + "Relationship" : { + "entityTypes" : [ "container" ], + "name" : "IsPartOf" + }, + "Searchable" : { + "addToFilters" : true, + "fieldName" : "container", + "fieldType" : "URN", + "filterNameOverride" : "Container", + "hasValuesFieldName" : "hasContainer" + } + } ], + "Aspect" : { + "name" : "container" + } + }, { "type" : "record", "name" : "DashboardInfo", "namespace" : "com.linkedin.dashboard", @@ -1806,7 +1858,7 @@ } ] } }, - "doc" : "Fine-grained column-level lineages", + "doc" : "Fine-grained column-level lineages\nNot currently supported in the UI\nUse UpstreamLineage aspect for datasets to express Column Level Lineage for the UI", "optional" : true } ], "Aspect" : { @@ -3097,7 +3149,8 @@ "Searchable" : { "boostScore" : 5.0, "fieldName" : "fieldPaths", - "fieldType" : "TEXT" + "fieldType" : "TEXT", + "queryByDefault" : "true" } }, { "name" : "jsonPath", @@ -5395,10 +5448,194 @@ "keyAspect" : "dataHubRetentionKey", "name" : "dataHubRetention" } + }, { + "type" : "record", + "name" : "JoinSnapshot", + "doc" : "A metadata snapshot for a specific join entity.", + "fields" : [ { + "name" : "urn", + "type" : "com.linkedin.common.JoinUrn", + "doc" : "URN for the entity the metadata snapshot is associated with." + }, { + "name" : "aspects", + "type" : { + "type" : "array", + "items" : { + "type" : "typeref", + "name" : "JoinAspect", + "namespace" : "com.linkedin.metadata.aspect", + "doc" : "A union of all supported metadata aspects for a Join", + "ref" : [ { + "type" : "record", + "name" : "JoinKey", + "namespace" : "com.linkedin.metadata.key", + "doc" : "Key for a Join", + "fields" : [ { + "name" : "joinId", + "type" : "string", + "Searchable" : { + "fieldType" : "TEXT" + } + } ], + "Aspect" : { + "name" : "joinKey" + } + }, { + "type" : "record", + "name" : "JoinProperties", + "namespace" : "com.linkedin.join", + "doc" : "Properties associated with a Join", + "include" : [ "com.linkedin.common.CustomProperties" ], + "fields" : [ { + "name" : "name", + "type" : "string", + "doc" : "Name of the Join", + "Searchable" : { + "boostScore" : 10.0, + "enableAutocomplete" : true, + "fieldType" : "TEXT_PARTIAL" + } + }, { + "name" : "datasetA", + "type" : "com.linkedin.common.DatasetUrn", + "doc" : "First dataset in the join (no directionality)", + "Relationship" : { + "entityTypes" : [ "dataset" ], + "name" : "joinA" + }, + "Searchable" : { + "boostScore" : 10.0, + "enableAutocomplete" : true, + "fieldType" : "TEXT_PARTIAL" + } + }, { + "name" : "datasetB", + "type" : "com.linkedin.common.DatasetUrn", + "doc" : "Second dataset in the join (no directionality)", + "Relationship" : { + "entityTypes" : [ "dataset" ], + "name" : "joinB" + }, + "Searchable" : { + "boostScore" : 10.0, + "enableAutocomplete" : true, + "fieldType" : "TEXT_PARTIAL" + } + }, { + "name" : "joinFieldMappings", + "type" : { + "type" : "record", + "name" : "JoinFieldMapping", + "doc" : "Field Mapping about a join between two datasets", + "fields" : [ { + "name" : "fieldMapping", + "type" : { + "type" : "array", + "items" : { + "type" : "record", + "name" : "FieldMap", + "doc" : "Field Mapping of 1:1 field", + "fields" : [ { + "name" : "afield", + "type" : "com.linkedin.dataset.SchemaFieldPath", + "doc" : "All fields from dataset A that are required for the join, maps to bFields 1:1", + "Searchable" : { + "boostScore" : 10.0, + "enableAutocomplete" : true, + "fieldType" : "TEXT_PARTIAL" + } + }, { + "name" : "bfield", + "type" : "com.linkedin.dataset.SchemaFieldPath", + "doc" : "All fields from dataset B that are required for the join, maps to aFields 1:1", + "Searchable" : { + "boostScore" : 10.0, + "enableAutocomplete" : true, + "fieldType" : "TEXT_PARTIAL" + } + } ] + } + }, + "doc" : "All fields from dataset A that are required for the join to dataset B" + }, { + "name" : "details", + "type" : "string", + "doc" : " Any transformation logic or notes pertaining to this specific join", + "Searchable" : { + "boostScore" : 10.0, + "enableAutocomplete" : true, + "fieldType" : "TEXT_PARTIAL" + } + } ] + }, + "doc" : "JoinFieldMapping (in future we can make it an array)" + }, { + "name" : "created", + "type" : "com.linkedin.common.TimeStamp", + "doc" : "A timestamp documenting when the asset was created in the source Data Platform (not on DataHub)", + "optional" : true, + "Searchable" : { + "/time" : { + "fieldName" : "createdAt", + "fieldType" : "DATETIME" + } + } + }, { + "name" : "lastModified", + "type" : "com.linkedin.common.TimeStamp", + "doc" : "A timestamp documenting when the asset was last modified in the source Data Platform (not on DataHub)", + "optional" : true, + "Searchable" : { + "/time" : { + "fieldName" : "lastModifiedAt", + "fieldType" : "DATETIME" + } + } + } ], + "Aspect" : { + "name" : "joinProperties" + } + }, { + "type" : "record", + "name" : "EditableJoinProperties", + "namespace" : "com.linkedin.join", + "doc" : "EditableJoinProperties stores editable changes made to join properties. This separates changes made from\ningestion pipelines and edits in the UI to avoid accidental overwrites of user-provided data by ingestion pipelines", + "include" : [ "com.linkedin.common.ChangeAuditStamps" ], + "fields" : [ { + "name" : "description", + "type" : "string", + "doc" : "Documentation of the join", + "optional" : true, + "Searchable" : { + "fieldName" : "editedDescription", + "fieldType" : "TEXT" + } + }, { + "name" : "name", + "type" : "string", + "doc" : "Display name of the Join", + "optional" : true, + "Searchable" : { + "fieldName" : "editedName", + "fieldType" : "TEXT_PARTIAL" + } + } ], + "Aspect" : { + "name" : "editableJoinProperties" + } + }, "com.linkedin.common.InstitutionalMemory", "com.linkedin.common.Ownership", "com.linkedin.common.Status", "com.linkedin.container.Container", "com.linkedin.common.GlobalTags", "com.linkedin.common.GlossaryTerms", "com.linkedin.common.BrowsePaths" ] + } + }, + "doc" : "The list of metadata aspects associated with the Join. Depending on the use case, this can either be all, or a selection, of supported aspects." + } ], + "Entity" : { + "keyAspect" : "joinKey", + "name" : "join" + } } ] } } ] - }, "com.linkedin.glossary.GlossaryNodeInfo", "com.linkedin.glossary.GlossaryRelatedTerms", "com.linkedin.glossary.GlossaryTermInfo", "com.linkedin.identity.CorpGroupInfo", "com.linkedin.identity.CorpUserEditableInfo", "com.linkedin.identity.CorpUserInfo", "com.linkedin.identity.CorpUserStatus", "com.linkedin.identity.GroupMembership", "com.linkedin.metadata.aspect.ChartAspect", "com.linkedin.metadata.aspect.CorpGroupAspect", "com.linkedin.metadata.aspect.CorpUserAspect", "com.linkedin.metadata.aspect.DashboardAspect", "com.linkedin.metadata.aspect.DataFlowAspect", "com.linkedin.metadata.aspect.DataHubPolicyAspect", "com.linkedin.metadata.aspect.DataHubRetentionAspect", "com.linkedin.metadata.aspect.DataJobAspect", "com.linkedin.metadata.aspect.DataPlatformAspect", "com.linkedin.metadata.aspect.DataProcessAspect", "com.linkedin.metadata.aspect.DatasetAspect", "com.linkedin.metadata.aspect.GlossaryNodeAspect", "com.linkedin.metadata.aspect.GlossaryTermAspect", "com.linkedin.metadata.aspect.MLFeatureAspect", "com.linkedin.metadata.aspect.MLFeatureTableAspect", "com.linkedin.metadata.aspect.MLModelAspect", "com.linkedin.metadata.aspect.MLModelDeploymentAspect", "com.linkedin.metadata.aspect.MLModelGroupAspect", "com.linkedin.metadata.aspect.MLPrimaryKeyAspect", "com.linkedin.metadata.aspect.SchemaFieldAspect", "com.linkedin.metadata.aspect.TagAspect", "com.linkedin.metadata.key.ChartKey", "com.linkedin.metadata.key.CorpGroupKey", "com.linkedin.metadata.key.CorpUserKey", "com.linkedin.metadata.key.DashboardKey", "com.linkedin.metadata.key.DataFlowKey", "com.linkedin.metadata.key.DataHubPolicyKey", "com.linkedin.metadata.key.DataHubRetentionKey", "com.linkedin.metadata.key.DataJobKey", "com.linkedin.metadata.key.DataPlatformKey", "com.linkedin.metadata.key.DataProcessKey", "com.linkedin.metadata.key.DatasetKey", "com.linkedin.metadata.key.GlossaryNodeKey", "com.linkedin.metadata.key.GlossaryTermKey", "com.linkedin.metadata.key.MLFeatureKey", "com.linkedin.metadata.key.MLFeatureTableKey", "com.linkedin.metadata.key.MLModelDeploymentKey", "com.linkedin.metadata.key.MLModelGroupKey", "com.linkedin.metadata.key.MLModelKey", "com.linkedin.metadata.key.MLPrimaryKeyKey", "com.linkedin.metadata.key.SchemaFieldKey", "com.linkedin.metadata.key.TagKey", "com.linkedin.metadata.snapshot.ChartSnapshot", "com.linkedin.metadata.snapshot.CorpGroupSnapshot", "com.linkedin.metadata.snapshot.CorpUserSnapshot", "com.linkedin.metadata.snapshot.DashboardSnapshot", "com.linkedin.metadata.snapshot.DataFlowSnapshot", "com.linkedin.metadata.snapshot.DataHubPolicySnapshot", "com.linkedin.metadata.snapshot.DataHubRetentionSnapshot", "com.linkedin.metadata.snapshot.DataJobSnapshot", "com.linkedin.metadata.snapshot.DataPlatformSnapshot", "com.linkedin.metadata.snapshot.DataProcessSnapshot", "com.linkedin.metadata.snapshot.DatasetSnapshot", "com.linkedin.metadata.snapshot.GlossaryNodeSnapshot", "com.linkedin.metadata.snapshot.GlossaryTermSnapshot", "com.linkedin.metadata.snapshot.MLFeatureSnapshot", "com.linkedin.metadata.snapshot.MLFeatureTableSnapshot", "com.linkedin.metadata.snapshot.MLModelDeploymentSnapshot", "com.linkedin.metadata.snapshot.MLModelGroupSnapshot", "com.linkedin.metadata.snapshot.MLModelSnapshot", "com.linkedin.metadata.snapshot.MLPrimaryKeySnapshot", "com.linkedin.metadata.snapshot.SchemaFieldSnapshot", "com.linkedin.metadata.snapshot.Snapshot", "com.linkedin.metadata.snapshot.TagSnapshot", "com.linkedin.ml.metadata.BaseData", "com.linkedin.ml.metadata.CaveatDetails", "com.linkedin.ml.metadata.CaveatsAndRecommendations", "com.linkedin.ml.metadata.DeploymentStatus", "com.linkedin.ml.metadata.EthicalConsiderations", "com.linkedin.ml.metadata.EvaluationData", "com.linkedin.ml.metadata.HyperParameterValueType", "com.linkedin.ml.metadata.IntendedUse", "com.linkedin.ml.metadata.IntendedUserType", "com.linkedin.ml.metadata.MLFeatureProperties", "com.linkedin.ml.metadata.MLFeatureTableProperties", "com.linkedin.ml.metadata.MLHyperParam", "com.linkedin.ml.metadata.MLMetric", "com.linkedin.ml.metadata.MLModelDeploymentProperties", "com.linkedin.ml.metadata.MLModelFactorPrompts", "com.linkedin.ml.metadata.MLModelFactors", "com.linkedin.ml.metadata.MLModelGroupProperties", "com.linkedin.ml.metadata.MLModelProperties", "com.linkedin.ml.metadata.MLPrimaryKeyProperties", "com.linkedin.ml.metadata.Metrics", "com.linkedin.ml.metadata.QuantitativeAnalyses", "com.linkedin.ml.metadata.ResultsType", "com.linkedin.ml.metadata.SourceCode", "com.linkedin.ml.metadata.SourceCodeUrl", "com.linkedin.ml.metadata.SourceCodeUrlType", "com.linkedin.ml.metadata.TrainingData", { + }, "com.linkedin.glossary.GlossaryNodeInfo", "com.linkedin.glossary.GlossaryRelatedTerms", "com.linkedin.glossary.GlossaryTermInfo", "com.linkedin.identity.CorpGroupInfo", "com.linkedin.identity.CorpUserEditableInfo", "com.linkedin.identity.CorpUserInfo", "com.linkedin.identity.CorpUserStatus", "com.linkedin.identity.GroupMembership", "com.linkedin.join.EditableJoinProperties", "com.linkedin.join.FieldMap", "com.linkedin.join.JoinFieldMapping", "com.linkedin.join.JoinProperties", "com.linkedin.metadata.aspect.ChartAspect", "com.linkedin.metadata.aspect.CorpGroupAspect", "com.linkedin.metadata.aspect.CorpUserAspect", "com.linkedin.metadata.aspect.DashboardAspect", "com.linkedin.metadata.aspect.DataFlowAspect", "com.linkedin.metadata.aspect.DataHubPolicyAspect", "com.linkedin.metadata.aspect.DataHubRetentionAspect", "com.linkedin.metadata.aspect.DataJobAspect", "com.linkedin.metadata.aspect.DataPlatformAspect", "com.linkedin.metadata.aspect.DataProcessAspect", "com.linkedin.metadata.aspect.DatasetAspect", "com.linkedin.metadata.aspect.GlossaryNodeAspect", "com.linkedin.metadata.aspect.GlossaryTermAspect", "com.linkedin.metadata.aspect.JoinAspect", "com.linkedin.metadata.aspect.MLFeatureAspect", "com.linkedin.metadata.aspect.MLFeatureTableAspect", "com.linkedin.metadata.aspect.MLModelAspect", "com.linkedin.metadata.aspect.MLModelDeploymentAspect", "com.linkedin.metadata.aspect.MLModelGroupAspect", "com.linkedin.metadata.aspect.MLPrimaryKeyAspect", "com.linkedin.metadata.aspect.SchemaFieldAspect", "com.linkedin.metadata.aspect.TagAspect", "com.linkedin.metadata.key.ChartKey", "com.linkedin.metadata.key.CorpGroupKey", "com.linkedin.metadata.key.CorpUserKey", "com.linkedin.metadata.key.DashboardKey", "com.linkedin.metadata.key.DataFlowKey", "com.linkedin.metadata.key.DataHubPolicyKey", "com.linkedin.metadata.key.DataHubRetentionKey", "com.linkedin.metadata.key.DataJobKey", "com.linkedin.metadata.key.DataPlatformKey", "com.linkedin.metadata.key.DataProcessKey", "com.linkedin.metadata.key.DatasetKey", "com.linkedin.metadata.key.GlossaryNodeKey", "com.linkedin.metadata.key.GlossaryTermKey", "com.linkedin.metadata.key.JoinKey", "com.linkedin.metadata.key.MLFeatureKey", "com.linkedin.metadata.key.MLFeatureTableKey", "com.linkedin.metadata.key.MLModelDeploymentKey", "com.linkedin.metadata.key.MLModelGroupKey", "com.linkedin.metadata.key.MLModelKey", "com.linkedin.metadata.key.MLPrimaryKeyKey", "com.linkedin.metadata.key.SchemaFieldKey", "com.linkedin.metadata.key.TagKey", "com.linkedin.metadata.snapshot.ChartSnapshot", "com.linkedin.metadata.snapshot.CorpGroupSnapshot", "com.linkedin.metadata.snapshot.CorpUserSnapshot", "com.linkedin.metadata.snapshot.DashboardSnapshot", "com.linkedin.metadata.snapshot.DataFlowSnapshot", "com.linkedin.metadata.snapshot.DataHubPolicySnapshot", "com.linkedin.metadata.snapshot.DataHubRetentionSnapshot", "com.linkedin.metadata.snapshot.DataJobSnapshot", "com.linkedin.metadata.snapshot.DataPlatformSnapshot", "com.linkedin.metadata.snapshot.DataProcessSnapshot", "com.linkedin.metadata.snapshot.DatasetSnapshot", "com.linkedin.metadata.snapshot.GlossaryNodeSnapshot", "com.linkedin.metadata.snapshot.GlossaryTermSnapshot", "com.linkedin.metadata.snapshot.JoinSnapshot", "com.linkedin.metadata.snapshot.MLFeatureSnapshot", "com.linkedin.metadata.snapshot.MLFeatureTableSnapshot", "com.linkedin.metadata.snapshot.MLModelDeploymentSnapshot", "com.linkedin.metadata.snapshot.MLModelGroupSnapshot", "com.linkedin.metadata.snapshot.MLModelSnapshot", "com.linkedin.metadata.snapshot.MLPrimaryKeySnapshot", "com.linkedin.metadata.snapshot.SchemaFieldSnapshot", "com.linkedin.metadata.snapshot.Snapshot", "com.linkedin.metadata.snapshot.TagSnapshot", "com.linkedin.ml.metadata.BaseData", "com.linkedin.ml.metadata.CaveatDetails", "com.linkedin.ml.metadata.CaveatsAndRecommendations", "com.linkedin.ml.metadata.DeploymentStatus", "com.linkedin.ml.metadata.EthicalConsiderations", "com.linkedin.ml.metadata.EvaluationData", "com.linkedin.ml.metadata.HyperParameterValueType", "com.linkedin.ml.metadata.IntendedUse", "com.linkedin.ml.metadata.IntendedUserType", "com.linkedin.ml.metadata.MLFeatureProperties", "com.linkedin.ml.metadata.MLFeatureTableProperties", "com.linkedin.ml.metadata.MLHyperParam", "com.linkedin.ml.metadata.MLMetric", "com.linkedin.ml.metadata.MLModelDeploymentProperties", "com.linkedin.ml.metadata.MLModelFactorPrompts", "com.linkedin.ml.metadata.MLModelFactors", "com.linkedin.ml.metadata.MLModelGroupProperties", "com.linkedin.ml.metadata.MLModelProperties", "com.linkedin.ml.metadata.MLPrimaryKeyProperties", "com.linkedin.ml.metadata.Metrics", "com.linkedin.ml.metadata.QuantitativeAnalyses", "com.linkedin.ml.metadata.ResultsType", "com.linkedin.ml.metadata.SourceCode", "com.linkedin.ml.metadata.SourceCodeUrl", "com.linkedin.ml.metadata.SourceCodeUrlType", "com.linkedin.ml.metadata.TrainingData", { "type" : "record", "name" : "GenericPayload", "namespace" : "com.linkedin.mxe", From 35ea6ca39105987f5acd09357a93f74c16423f2e Mon Sep 17 00:00:00 2001 From: Raj Tekal Date: Thu, 15 Jun 2023 17:11:12 -0400 Subject: [PATCH 5/5] Remove Snapshot and Aspect --- .../datahub/graphql/GmsGraphQLEngine.java | 16 ++++++++++ .../join/mappers/JoinUpdateInputMapper.java | 25 +++++++++------- .../src/main/resources/entity.graphql | 4 +-- .../main/pegasus/com/linkedin/join/Joins.pdl | 15 ---------- .../linkedin/metadata/aspect/JoinAspect.pdl | 30 ------------------- .../metadata/snapshot/JoinSnapshot.pdl | 24 --------------- .../linkedin/metadata/snapshot/Snapshot.pdl | 1 - 7 files changed, 32 insertions(+), 83 deletions(-) delete mode 100644 metadata-models/src/main/pegasus/com/linkedin/join/Joins.pdl delete mode 100644 metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/JoinAspect.pdl delete mode 100644 metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/JoinSnapshot.pdl diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index d5252473b0652..f16a6ecc74025 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -60,6 +60,7 @@ import com.linkedin.datahub.graphql.generated.IngestionSource; import com.linkedin.datahub.graphql.generated.InstitutionalMemoryMetadata; import com.linkedin.datahub.graphql.generated.Join; +import com.linkedin.datahub.graphql.generated.JoinProperties; import com.linkedin.datahub.graphql.generated.LineageRelationship; import com.linkedin.datahub.graphql.generated.ListAccessTokenResult; import com.linkedin.datahub.graphql.generated.ListOwnershipTypesResult; @@ -1410,6 +1411,7 @@ private void configureTypeExtensions(final RuntimeWiring.Builder builder) { private void configureJoinResolvers(final RuntimeWiring.Builder builder) { builder .type("Join", typeWiring -> typeWiring + .dataFetcher("privileges", new EntityPrivilegesResolver(entityClient)) .dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) .dataFetcher("browsePaths", new EntityBrowsePathsResolver(this.joinType)) .dataFetcher("container", @@ -1419,6 +1421,20 @@ private void configureJoinResolvers(final RuntimeWiring.Builder builder) { return join.getContainer() != null ? join.getContainer().getUrn() : null; }) )) + .type("JoinProperties", typeWiring -> typeWiring + .dataFetcher("datasetA", + new LoadableTypeResolver<>(datasetType, + (env) -> { + final JoinProperties joinProperties = env.getSource(); + return joinProperties.getDatasetA() != null ? joinProperties.getDatasetA().getUrn() : null; + })) + .dataFetcher("datasetB", + new LoadableTypeResolver<>(datasetType, + (env) -> { + final JoinProperties joinProperties = env.getSource(); + return joinProperties.getDatasetB() != null ? joinProperties.getDatasetB().getUrn() : null; + })) + ) .type("Owner", typeWiring -> typeWiring .dataFetcher("owner", new OwnerTypeResolver<>(ownerTypes, (env) -> ((Owner) env.getSource()).getOwner())) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/mappers/JoinUpdateInputMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/mappers/JoinUpdateInputMapper.java index 11b27849aa32e..d074447e894b2 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/mappers/JoinUpdateInputMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/join/mappers/JoinUpdateInputMapper.java @@ -1,8 +1,8 @@ package com.linkedin.datahub.graphql.types.join.mappers; -import com.linkedin.common.AuditStamp; -import com.linkedin.common.GlobalTags; import com.linkedin.common.TagAssociationArray; +import com.linkedin.common.TimeStamp; +import com.linkedin.common.GlobalTags; import com.linkedin.common.urn.DatasetUrn; import com.linkedin.common.urn.Urn; import com.linkedin.data.template.SetMode; @@ -40,10 +40,10 @@ public static Collection map(@Nonnull final JoinUpdateIn public Collection apply(JoinUpdateInput input, Urn actor) { final Collection proposals = new ArrayList<>(8); final UpdateMappingHelper updateMappingHelper = new UpdateMappingHelper(JOIN_ENTITY_NAME); - final AuditStamp auditStamp = new AuditStamp(); - auditStamp.setActor(actor, SetMode.IGNORE_NULL); - auditStamp.setTime(System.currentTimeMillis()); - + final long currentTime = System.currentTimeMillis(); + final TimeStamp timestamp = new TimeStamp(); + timestamp.setActor(actor, SetMode.IGNORE_NULL); + timestamp.setTime(currentTime); if (input.getProperties() != null) { com.linkedin.join.JoinProperties joinProperties = new com.linkedin.join.JoinProperties(); if (input.getProperties().getName() != null) { @@ -84,9 +84,10 @@ public Collection apply(JoinUpdateInput input, Urn actor } joinProperties.setJoinFieldMappings(joinFieldMapping1); } + joinProperties.setLastModified(timestamp); proposals.add(updateMappingHelper.aspectToProposal(joinProperties, JOIN_PROPERTIES_ASPECT_NAME)); } - + } if (input.getOwnership() != null) { proposals.add(updateMappingHelper.aspectToProposal(OwnershipUpdateMapper.map(input.getOwnership(), actor), OWNERSHIP_ASPECT_NAME)); @@ -105,14 +106,16 @@ public Collection apply(JoinUpdateInput input, Urn actor } proposals.add(updateMappingHelper.aspectToProposal(globalTags, GLOBAL_TAGS_ASPECT_NAME)); } - if (input.getEditableProperties() != null) { final EditableJoinProperties editableJoinProperties = new EditableJoinProperties(); - editableJoinProperties.setName(input.getEditableProperties().getName()); - editableJoinProperties.setDescription(input.getEditableProperties().getDescription()); + if (input.getEditableProperties().getName().trim().length() > 0) { + editableJoinProperties.setName(input.getEditableProperties().getName()); + } + if (input.getEditableProperties().getDescription().trim().length() > 0) { + editableJoinProperties.setDescription(input.getEditableProperties().getDescription()); + } proposals.add(updateMappingHelper.aspectToProposal(editableJoinProperties, EDITABLE_JOIN_PROPERTIES_ASPECT_NAME)); } - } return proposals; } } diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index 54c63fd5e546d..f14a603b231da 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -385,12 +385,12 @@ type JoinProperties { """ The urn of datasetA """ - datasetA: String! + datasetA: Dataset! """ The urn of datasetB """ - datasetB: String! + datasetB: Dataset! """ The joinFieldMappings diff --git a/metadata-models/src/main/pegasus/com/linkedin/join/Joins.pdl b/metadata-models/src/main/pegasus/com/linkedin/join/Joins.pdl deleted file mode 100644 index 2e339d15a2a2d..0000000000000 --- a/metadata-models/src/main/pegasus/com/linkedin/join/Joins.pdl +++ /dev/null @@ -1,15 +0,0 @@ -namespace com.linkedin.join -import com.linkedin.common.JoinUrn - -/** - * Joins information of an entity. - */ -@Aspect = { - "name": "joins" -} -record Joins { - /** - * Join - */ - joins: array[JoinUrn] -} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/JoinAspect.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/JoinAspect.pdl deleted file mode 100644 index ef8e48ad3252f..0000000000000 --- a/metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/JoinAspect.pdl +++ /dev/null @@ -1,30 +0,0 @@ -namespace com.linkedin.metadata.aspect - -import com.linkedin.metadata.key.JoinKey -import com.linkedin.join.JoinProperties -import com.linkedin.join.EditableJoinProperties - -import com.linkedin.common.InstitutionalMemory -import com.linkedin.common.Ownership -import com.linkedin.common.Status -import com.linkedin.container.Container -import com.linkedin.common.GlobalTags -import com.linkedin.common.GlossaryTerms -import com.linkedin.common.BrowsePaths - - -/** - * A union of all supported metadata aspects for a Join - */ -typeref JoinAspect = union[ - JoinKey, - JoinProperties, - EditableJoinProperties, - InstitutionalMemory, - Ownership, - Status, - Container, - GlobalTags, - GlossaryTerms, - BrowsePaths -] diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/JoinSnapshot.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/JoinSnapshot.pdl deleted file mode 100644 index 58bc2c59569ab..0000000000000 --- a/metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/JoinSnapshot.pdl +++ /dev/null @@ -1,24 +0,0 @@ -namespace com.linkedin.metadata.snapshot - -import com.linkedin.common.JoinUrn -import com.linkedin.metadata.aspect.JoinAspect - -/** - * A metadata snapshot for a specific join entity. - */ -@Entity = { - "name": "join", - "keyAspect": "joinKey" -} -record JoinSnapshot { - - /** - * URN for the entity the metadata snapshot is associated with. - */ - urn: JoinUrn - - /** - * The list of metadata aspects associated with the Join. Depending on the use case, this can either be all, or a selection, of supported aspects. - */ - aspects: array[JoinAspect] -} diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/Snapshot.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/Snapshot.pdl index 374bdb6cd9906..91993724afbad 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/Snapshot.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/Snapshot.pdl @@ -25,5 +25,4 @@ typeref Snapshot = union[ DataHubPolicySnapshot, SchemaFieldSnapshot, DataHubRetentionSnapshot, - JoinSnapshot, ]