From 545553cde18db7fd916863af69a9a8d6060b1899 Mon Sep 17 00:00:00 2001 From: aditya-radhakrishnan Date: Tue, 12 Jul 2022 16:16:35 -0700 Subject: [PATCH] fix(bootstrap): skip ingesting data platforms that already exist --- .../boot/steps/IngestDataPlatformsStep.java | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDataPlatformsStep.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDataPlatformsStep.java index 56910cf24baea0..7d460419adf7b2 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDataPlatformsStep.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDataPlatformsStep.java @@ -1,5 +1,6 @@ package com.linkedin.metadata.boot.steps; +import com.datahub.util.RecordUtils; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.linkedin.common.AuditStamp; @@ -7,12 +8,9 @@ import com.linkedin.dataplatform.DataPlatformInfo; import com.linkedin.metadata.Constants; import com.linkedin.metadata.boot.BootstrapStep; -import com.datahub.util.RecordUtils; import com.linkedin.metadata.entity.EntityService; - import java.io.IOException; import java.net.URISyntaxException; - import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.core.io.ClassPathResource; @@ -46,13 +44,24 @@ public void execute() throws IOException, URISyntaxException { // 2. For each JSON object, cast into a DataPlatformSnapshot object. for (final JsonNode dataPlatform : dataPlatforms) { + final String urnString; final Urn urn; try { - urn = Urn.createFromString(dataPlatform.get("urn").asText()); + urnString = dataPlatform.get("urn").asText(); + urn = Urn.createFromString(urnString); } catch (URISyntaxException e) { log.error("Malformed urn: {}", dataPlatform.get("urn").asText()); throw new RuntimeException("Malformed urn", e); } + + final DataPlatformInfo existingInfo = + (DataPlatformInfo) _entityService.getLatestAspect(urn, PLATFORM_ASPECT_NAME); + // Skip ingesting for this JSON object if info already exists. + if (existingInfo != null) { + log.debug(String.format("%s already exists for %s. Skipping...", PLATFORM_ASPECT_NAME, urnString)); + continue; + } + final DataPlatformInfo info = RecordUtils.toRecordTemplate(DataPlatformInfo.class, dataPlatform.get("aspect").toString());