Merge pull request #5 from linkedin/master

Upstream

README.md

@@ -67,7 +67,7 @@ HOSTED_DOCS_ONLY-->
 ## Introduction
 
 DataHub is an open-source metadata platform for the modern data stack. Read about the architectures of different metadata systems and why DataHub excels [here](https://engineering.linkedin.com/blog/2020/datahub-popular-metadata-architectures-explained). Also read our
-[LinkedIn Engineering blog post](https://engineering.linkedin.com/blog/2019/data-hub), check out our [Strata presentation](https://speakerdeck.com/shirshanka/the-evolution-of-metadata-linkedins-journey-strata-nyc-2019) and watch our [Crunch Conference Talk](https://www.youtube.com/watch?v=OB-O0Y6OYDE). You should also visit [DataHub Architecture](docs/architecture/architecture.md) to get a better understanding of how DataHub is implemented and [DataHub Onboarding Guide](docs/modeling/extending-the-metadata-model.md) to understand how to extend DataHub for your own use cases.
+[LinkedIn Engineering blog post](https://engineering.linkedin.com/blog/2019/data-hub), check out our [Strata presentation](https://speakerdeck.com/shirshanka/the-evolution-of-metadata-linkedins-journey-strata-nyc-2019) and watch our [Crunch Conference Talk](https://www.youtube.com/watch?v=OB-O0Y6OYDE). You should also visit [DataHub Architecture](docs/architecture/architecture.md) to get a better understanding of how DataHub is implemented.
 
 ## Quickstart
 
@@ -99,6 +99,10 @@ Check out DataHub's [Features](docs/features.md) & [Roadmap](https://feature-req
 
 We welcome contributions from the community. Please refer to our [Contributing Guidelines](docs/CONTRIBUTING.md) for more details. We also have a [contrib](contrib) directory for incubating experimental features.
 
+### Extending
+
+If you need to understand how to extend our model with custom types, please see [Extending the Metadata Model](docs/modeling/extending-the-metadata-model.md)
+
 ## Community
 
 Join our [slack workspace](https://slack.datahubproject.io) for discussions and important announcements. You can also find out more about our upcoming [town hall meetings](docs/townhalls.md) and view past recordings.

build.gradle

@@ -78,6 +78,7 @@ project.ext.externalDependency = [
     'jerseyCore': 'org.glassfish.jersey.core:jersey-client:2.25.1',
     'jerseyGuava': 'org.glassfish.jersey.bundles.repackaged:jersey-guava:2.25.1',
     'jettyJaas': 'org.eclipse.jetty:jetty-jaas:9.4.32.v20200930',
+    'jgrapht': 'org.jgrapht:jgrapht-core:1.5.1',
     'jsonSimple': 'com.googlecode.json-simple:json-simple:1.1.1',
     'junitJupiterApi': "org.junit.jupiter:junit-jupiter-api:$junitJupiterVersion",
     'junitJupiterParams': "org.junit.jupiter:junit-jupiter-params:$junitJupiterVersion",
@@ -110,6 +111,7 @@ project.ext.externalDependency = [
     'pac4j': 'org.pac4j:pac4j-oidc:3.6.0',
     'playPac4j': 'org.pac4j:play-pac4j_2.11:7.0.1',
     'postgresql': 'org.postgresql:postgresql:42.3.3',
+    'protobuf': 'com.google.protobuf:protobuf-java:3.19.3',
     'reflections': 'org.reflections:reflections:0.9.9',
     'resilience4j': 'io.github.resilience4j:resilience4j-retry:1.7.1',
     'rythmEngine': 'org.rythmengine:rythm-engine:1.3.0',
@@ -194,14 +196,27 @@ subprojects {
     }
   }
 
-  tasks.withType(JavaCompile).configureEach {
-    javaCompiler = javaToolchains.compilerFor {
-      languageVersion = JavaLanguageVersion.of(8)
+  if (project.name != 'datahub-protobuf') {
+    tasks.withType(JavaCompile).configureEach {
+      javaCompiler = javaToolchains.compilerFor {
+        languageVersion = JavaLanguageVersion.of(8)
+      }
     }
-  }
-  tasks.withType(Test).configureEach {
-    javaLauncher = javaToolchains.launcherFor {
-      languageVersion = JavaLanguageVersion.of(8)
+    tasks.withType(Test).configureEach {
+      javaLauncher = javaToolchains.launcherFor {
+        languageVersion = JavaLanguageVersion.of(8)
+      }
+    }
+  } else {
+    tasks.withType(JavaCompile).configureEach {
+      javaCompiler = javaToolchains.compilerFor {
+        languageVersion = JavaLanguageVersion.of(11)
+      }
+    }
+    tasks.withType(Test).configureEach {
+      javaLauncher = javaToolchains.launcherFor {
+        languageVersion = JavaLanguageVersion.of(11)
+      }
     }
   }
 

datahub-frontend/app/auth/sso/oidc/OidcCallbackLogic.java

@@ -55,6 +55,7 @@
 import play.mvc.Result;
 import auth.sso.SsoManager;
 
+import static com.linkedin.metadata.Constants.*;
 import static play.mvc.Results.*;
 import static auth.AuthUtils.*;
 
@@ -125,16 +126,14 @@ private Result handleOidcCallback(
         if (oidcConfigs.isJitProvisioningEnabled()) {
           log.debug("Just-in-time provisioning is enabled. Beginning provisioning process...");
           CorpUserSnapshot extractedUser = extractUser(corpUserUrn, profile);
+          tryProvisionUser(extractedUser);
           if (oidcConfigs.isExtractGroupsEnabled()) {
             // Extract groups & provision them.
             List<CorpGroupSnapshot> extractedGroups = extractGroups(profile);
             tryProvisionGroups(extractedGroups);
-            if (extractedGroups.size() > 0) {
-              // Associate group with the user logging in.
-              extractedUser.getAspects().add(CorpUserAspect.create(createGroupMembership(extractedGroups)));
-            }
+            // Add users to groups on DataHub. Note that this clears existing group membership for a user if it already exists.
+            updateGroupMembership(corpUserUrn, createGroupMembership(extractedGroups));
           }
-          tryProvisionUser(extractedUser);
         } else if (oidcConfigs.isPreProvisioningRequired()) {
           // We should only allow logins for user accounts that have been pre-provisioned
           log.debug("Pre Provisioning is required. Beginning validation of extracted user...");
@@ -372,6 +371,21 @@ private void tryProvisionGroups(List<CorpGroupSnapshot> corpGroups) {
     }
   }
 
+  private void updateGroupMembership(Urn urn, GroupMembership groupMembership) {
+    log.debug(String.format("Updating group membership for user %s", urn));
+    final MetadataChangeProposal proposal = new MetadataChangeProposal();
+    proposal.setEntityUrn(urn);
+    proposal.setEntityType(CORP_USER_ENTITY_NAME);
+    proposal.setAspectName(GROUP_MEMBERSHIP_ASPECT_NAME);
+    proposal.setAspect(GenericAspectUtils.serializeAspect(groupMembership));
+    proposal.setChangeType(ChangeType.UPSERT);
+    try {
+      _entityClient.ingestProposal(proposal, _systemAuthentication);
+    } catch (RemoteInvocationException e) {
+      throw new RuntimeException(String.format("Failed to update group membership for user with urn %s", urn), e);
+    }
+  }
+
   private void verifyPreProvisionedUser(CorpuserUrn urn) {
     // Validate that the user exists in the system (there is more than just a key aspect for them, as of today).
     try {

datahub-frontend/app/auth/sso/oidc/OidcConfigs.java

@@ -47,7 +47,7 @@ public class OidcConfigs extends SsoConfigs {
     private static final String DEFAULT_OIDC_CLIENT_AUTHENTICATION_METHOD = "client_secret_basic";
     private static final String DEFAULT_OIDC_JIT_PROVISIONING_ENABLED = "true";
     private static final String DEFAULT_OIDC_PRE_PROVISIONING_REQUIRED = "false";
-    private static final String DEFAULT_OIDC_EXTRACT_GROUPS_ENABLED = "true";
+    private static final String DEFAULT_OIDC_EXTRACT_GROUPS_ENABLED = "false"; // False since extraction of groups can overwrite existing group membership.
     private static final String DEFAULT_OIDC_GROUPS_CLAIM = "groups";
 
     private String clientId;

datahub-web-react/package.json

@@ -78,7 +78,6 @@
         "react-syntax-highlighter": "^15.4.4",
         "react-timezone-select": "^1.1.15",
         "react-visibility-sensor": "^5.1.1",
-        "rgb-hex": "^4.0.0",
         "sinon": "^11.1.1",
         "start-server-and-test": "1.12.2",
         "styled-components": "^5.2.1",

datahub-web-react/src/app/browse/BrowseResultsPage.tsx

@@ -61,7 +61,7 @@ export const BrowseResultsPage = () => {
                 <LegacyBrowsePath type={entityType} path={path} isBrowsable />
             </Affix>
             {loading && <Message type="loading" content="Loading..." style={{ marginTop: '10%' }} />}
-            {data && data.browse && (
+            {data && data.browse && !loading && (
                 <BrowseResults
                     type={entityType}
                     rootPath={rootPath}

datahub-web-react/src/app/shared/TagStyleEntity.tsx

@@ -6,8 +6,6 @@ import { ApolloError } from '@apollo/client';
 import styled from 'styled-components';
 import { ChromePicker } from 'react-color';
 import ColorHash from 'color-hash';
-import rgbHex from 'rgb-hex';
-
 import { PlusOutlined } from '@ant-design/icons';
 import { useGetTagQuery } from '../../graphql/tag.generated';
 import { EntityType, FacetMetadata, Maybe, Scalars } from '../../types.generated';
@@ -258,6 +256,10 @@ export default function TagStyleEntity({ urn, useGetSearchResults = useWrappedSe
         saveColor();
     };
 
+    const handleColorChange = (color: any) => {
+        setColorValue(color?.hex);
+    };
+
     // Update Description
     const updateDescriptionValue = (desc: string) => {
         setUpdatedDescription(desc);
@@ -312,10 +314,7 @@ export default function TagStyleEntity({ urn, useGetSearchResults = useWrappedSe
                 </TagName>
                 {displayColorPicker && (
                     <ColorPickerPopOver ref={colorPickerRef}>
-                        <ChromePicker
-                            color={colorValue}
-                            onChange={(c) => setColorValue(`#${rgbHex(c.rgb.r, c.rgb.g, c.rgb.b, c.rgb.a)}`)}
-                        />
+                        <ChromePicker color={colorValue} onChange={handleColorChange} />
                     </ColorPickerPopOver>
                 )}
             </div>

datahub-web-react/yarn.lock

@@ -14722,11 +14722,6 @@ rework@1.0.1:
     convert-source-map "^0.3.3"
     css "^2.0.0"
 
-rgb-hex@^4.0.0:
-  version "4.0.0"
-  resolved "https://registry.yarnpkg.com/rgb-hex/-/rgb-hex-4.0.0.tgz#02fb1e215e3fe5d070501579a3d2e4fa3c0acec8"
-  integrity sha512-Eg2ev5CiMBnQ9Gpflmqbwbso0CCdISqtVIow7OpYSLN1ULUv2jTB9YieS1DSSn/17AD7KkPWDPzSFzI4GSuu/Q==
-
 rgb-regex@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/rgb-regex/-/rgb-regex-1.0.1.tgz#c0e0d6882df0e23be254a475e8edd41915feaeb1"

docker/datahub-frontend/env/docker.env

@@ -28,7 +28,7 @@ JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf
 # Optional Provisioning Configs
 # AUTH_OIDC_JIT_PROVISIONING_ENABLED=true
 # AUTH_OIDC_PRE_PROVISIONING_REQUIRED=false
-# AUTH_OIDC_EXTRACT_GROUPS_ENABLED=true
+# AUTH_OIDC_EXTRACT_GROUPS_ENABLED=false
 # AUTH_OIDC_GROUPS_CLAIM=groups
 
 # Uncomment to disable JAAS username / password authentication (enabled by default)

docs-website/sidebars.js

@@ -82,6 +82,7 @@ module.exports = {
           "metadata-ingestion/as-a-library",
           "metadata-integration/java/as-a-library",
           "metadata-ingestion/integration_docs/great-expectations",
+          "metadata-integration/java/datahub-protobuf/README",
         ],
       },
       {