feat(config): configurable bootstrap policies file (#8812)

Co-authored-by: John Joyce <john@acryl.io>
datahub-project · Oct 11, 2023 · dfcea24 · dfcea24
1 parent a17db67
commit dfcea24
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 4 deletions.
diff --git a/metadata-service/configuration/src/main/resources/application.yml b/metadata-service/configuration/src/main/resources/application.yml
@@ -276,6 +276,10 @@ bootstrap:
     enabled: ${UPGRADE_DEFAULT_BROWSE_PATHS_ENABLED:false} # enable to run the upgrade to migrate legacy default browse paths to new ones
   backfillBrowsePathsV2:
     enabled: ${BACKFILL_BROWSE_PATHS_V2:false} # Enables running the backfill of browsePathsV2 upgrade step. There are concerns about the load of this step so hiding it behind a flag. Deprecating in favor of running through SystemUpdate
+  policies:
+    file: ${BOOTSTRAP_POLICIES_FILE:classpath:boot/policies.json}
+    # eg for local file
+    # file: "file:///datahub/datahub-gms/resources/custom-policies.json"
   servlets:
     waitTimeout: ${BOOTSTRAP_SERVLETS_WAITTIMEOUT:60} # Total waiting time in seconds for servlets to initialize
 

diff --git a/...factories/src/main/java/com/linkedin/metadata/boot/factories/BootstrapManagerFactory.java b/...factories/src/main/java/com/linkedin/metadata/boot/factories/BootstrapManagerFactory.java
@@ -31,6 +31,7 @@
 import com.linkedin.metadata.search.EntitySearchService;
 import com.linkedin.metadata.search.SearchService;
 import com.linkedin.metadata.search.transformer.SearchDocumentTransformer;
+
 import java.util.ArrayList;
 import java.util.List;
 import javax.annotation.Nonnull;
@@ -41,6 +42,7 @@
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Scope;
+import org.springframework.core.io.Resource;
 
 
 @Configuration
@@ -89,13 +91,16 @@ public class BootstrapManagerFactory {
   @Value("${bootstrap.backfillBrowsePathsV2.enabled}")
   private Boolean _backfillBrowsePathsV2Enabled;
 
+  @Value("${bootstrap.policies.file}")
+  private Resource _policiesResource;
+
   @Bean(name = "bootstrapManager")
   @Scope("singleton")
   @Nonnull
   protected BootstrapManager createInstance() {
     final IngestRootUserStep ingestRootUserStep = new IngestRootUserStep(_entityService);
     final IngestPoliciesStep ingestPoliciesStep =
-        new IngestPoliciesStep(_entityRegistry, _entityService, _entitySearchService, _searchDocumentTransformer);
+        new IngestPoliciesStep(_entityRegistry, _entityService, _entitySearchService, _searchDocumentTransformer, _policiesResource);
     final IngestRolesStep ingestRolesStep = new IngestRolesStep(_entityService, _entityRegistry);
     final IngestDataPlatformsStep ingestDataPlatformsStep = new IngestDataPlatformsStep(_entityService);
     final IngestDataPlatformInstancesStep ingestDataPlatformInstancesStep =

diff --git a/...-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestPoliciesStep.java b/...-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestPoliciesStep.java
@@ -25,6 +25,7 @@
 import com.linkedin.mxe.GenericAspect;
 import com.linkedin.mxe.MetadataChangeProposal;
 import com.linkedin.policy.DataHubPolicyInfo;
+
 import java.io.IOException;
 import java.net.URISyntaxException;
 import java.util.Collections;
@@ -35,7 +36,8 @@
 
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
-import org.springframework.core.io.ClassPathResource;
+import org.springframework.core.io.Resource;
+
 
 import static com.linkedin.metadata.Constants.*;
 
@@ -52,6 +54,8 @@ public class IngestPoliciesStep implements BootstrapStep {
   private final EntitySearchService _entitySearchService;
   private final SearchDocumentTransformer _searchDocumentTransformer;
 
+  private final Resource _policiesResource;
+
   @Override
   public String name() {
     return "IngestPoliciesStep";
@@ -66,10 +70,10 @@ public void execute() throws IOException, URISyntaxException {
         .maxStringLength(maxSize).build());
 
     // 0. Execute preflight check to see whether we need to ingest policies
-    log.info("Ingesting default access policies...");
+    log.info("Ingesting default access policies from: {}...", _policiesResource);
 
     // 1. Read from the file into JSON.
-    final JsonNode policiesObj = mapper.readTree(new ClassPathResource("./boot/policies.json").getFile());
+    final JsonNode policiesObj = mapper.readTree(_policiesResource.getFile());
 
     if (!policiesObj.isArray()) {
       throw new RuntimeException(