Merge branch 'master' into add-oracle-lib-ingestion

datahub-project · Jul 31, 2024 · bb6879c · bb6879c
2 parents 0fcbcd4 + e14dc91
commit bb6879c
Show file tree

Hide file tree

Showing 10 changed files with 115 additions and 105 deletions.
diff --git a/docker/datahub-ingestion-base/base-requirements.txt b/docker/datahub-ingestion-base/base-requirements.txt
diff --git a/docker/datahub-ingestion-base/build.gradle b/docker/datahub-ingestion-base/build.gradle
@@ -12,7 +12,7 @@ ext {
     docker_target = project.getProperties().getOrDefault("dockerTarget", "slim")
     docker_version = "${version}${docker_target == 'slim' ? '-slim' : ''}"
 
-    revision = 3 // increment to trigger rebuild
+    revision = 4 // increment to trigger rebuild
 }
 
 docker {

diff --git a/docker/datahub-ingestion/build.gradle b/docker/datahub-ingestion/build.gradle
@@ -12,7 +12,7 @@ ext {
     docker_target = project.getProperties().getOrDefault("dockerTarget", "slim")
     docker_version = "${version}${docker_target == 'slim' ? '-slim' : ''}"
 
-    revision = 3 // increment to trigger rebuild
+    revision = 4 // increment to trigger rebuild
 }
 
 dependencies {

diff --git a/docker/docker-compose-with-cassandra.yml b/docker/docker-compose-with-cassandra.yml
@@ -144,7 +144,7 @@ services:
       - neo4jdata:/data
   schema-registry:
     hostname: schema-registry
-    image: confluentinc/cp-schema-registry:7.4.0
+    image: ${DATAHUB_CONFLUENT_SCHEMA_REGISTRY_IMAGE:-confluentinc/cp-schema-registry}:${DATAHUB_CONFLUENT_VERSION:-7.4.0}
     ports:
       - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081
     env_file: schema-registry/env/docker.env
@@ -159,7 +159,7 @@ services:
         condition: service_healthy
   broker:
     hostname: broker
-    image: confluentinc/cp-kafka:7.4.0
+    image: ${DATAHUB_CONFLUENT_KAFKA_IMAGE:-confluentinc/cp-kafka}:${DATAHUB_CONFLUENT_VERSION:-7.4.0}
     ports:
       - 29092:29092
       - 9092:9092
@@ -177,7 +177,7 @@ services:
       - broker:/var/lib/kafka/data/
   zookeeper:
     hostname: zookeeper
-    image: confluentinc/cp-zookeeper:7.4.0
+    image: ${DATAHUB_CONFLUENT_ZOOKEEPER_IMAGE:-confluentinc/cp-zookeeper}:${DATAHUB_CONFLUENT_VERSION:-7.4.0}
     ports:
       - 2181:2181
     env_file: zookeeper/env/docker.env

diff --git a/docker/docker-compose-without-neo4j.yml b/docker/docker-compose-without-neo4j.yml
@@ -123,7 +123,7 @@ services:
     - esdata:/usr/share/elasticsearch/data
   schema-registry:
     hostname: schema-registry
-    image: confluentinc/cp-schema-registry:7.4.0
+    image: ${DATAHUB_CONFLUENT_SCHEMA_REGISTRY_IMAGE:-confluentinc/cp-schema-registry}:${DATAHUB_CONFLUENT_VERSION:-7.4.0}
     ports:
     - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081
     env_file: schema-registry/env/docker.env
@@ -138,7 +138,7 @@ services:
         condition: service_healthy
   broker:
     hostname: broker
-    image: confluentinc/cp-kafka:7.4.0
+    image: ${DATAHUB_CONFLUENT_KAFKA_IMAGE:-confluentinc/cp-kafka}:${DATAHUB_CONFLUENT_VERSION:-7.4.0}
     ports:
     - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092
     env_file: broker/env/docker.env
@@ -155,7 +155,7 @@ services:
     - broker:/var/lib/kafka/data/
   zookeeper:
     hostname: zookeeper
-    image: confluentinc/cp-zookeeper:7.4.0
+    image: ${DATAHUB_CONFLUENT_ZOOKEEPER_IMAGE:-confluentinc/cp-zookeeper}:${DATAHUB_CONFLUENT_VERSION:-7.4.0}
     ports:
     - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181
     env_file: zookeeper/env/docker.env

diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
@@ -143,7 +143,7 @@ services:
     - neo4jdata:/data
   schema-registry:
     hostname: schema-registry
-    image: confluentinc/cp-schema-registry:7.4.0
+    image: ${DATAHUB_CONFLUENT_SCHEMA_REGISTRY_IMAGE:-confluentinc/cp-schema-registry}:${DATAHUB_CONFLUENT_VERSION:-7.4.0}
     ports:
     - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081
     env_file: schema-registry/env/docker.env
@@ -158,7 +158,7 @@ services:
         condition: service_healthy
   broker:
     hostname: broker
-    image: confluentinc/cp-kafka:7.4.0
+    image: ${DATAHUB_CONFLUENT_KAFKA_IMAGE:-confluentinc/cp-kafka}:${DATAHUB_CONFLUENT_VERSION:-7.4.0}
     ports:
     - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092
     env_file: broker/env/docker.env
@@ -175,7 +175,7 @@ services:
     - broker:/var/lib/kafka/data/
   zookeeper:
     hostname: zookeeper
-    image: confluentinc/cp-zookeeper:7.4.0
+    image: ${DATAHUB_CONFLUENT_ZOOKEEPER_IMAGE:-confluentinc/cp-zookeeper}:${DATAHUB_CONFLUENT_VERSION:-7.4.0}
     ports:
     - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181
     env_file: zookeeper/env/docker.env

diff --git a/docker/profiles/docker-compose.prerequisites.yml b/docker/profiles/docker-compose.prerequisites.yml
@@ -210,7 +210,7 @@ services:
       - neo4jdata:/data
   kafka-broker:
     hostname: broker
-    image: confluentinc/cp-kafka:7.4.0
+    image: ${DATAHUB_CONFLUENT_KAFKA_IMAGE:-confluentinc/cp-kafka}:${DATAHUB_CONFLUENT_VERSION:-7.4.0}
     command:
       - /bin/bash
       - -c

diff --git a/docs/deploy/aws.md b/docs/deploy/aws.md
@@ -76,7 +76,7 @@ First, if you did not use eksctl to setup the kubernetes cluster, make sure to g
 Download the IAM policy document for allowing the controller to make calls to AWS APIs on your behalf.
 
 ```
-curl -o iam_policy.json https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.2.0/docs/install/iam_policy.json
+curl -o iam_policy.json https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/main/docs/install/iam_policy.json
 ```
 
 Create an IAM policy based on the policy document by running the following.
@@ -148,13 +148,9 @@ datahub-frontend:
       alb.ingress.kubernetes.io/certificate-arn: <<certificate-arn>>
       alb.ingress.kubernetes.io/inbound-cidrs: 0.0.0.0/0
       alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]'
-      alb.ingress.kubernetes.io/actions.ssl-redirect: '{"Type": "redirect", "RedirectConfig": { "Protocol": "HTTPS", "Port": "443", "StatusCode": "HTTP_301"}}'
+      alb.ingress.kubernetes.io/ssl-redirect: '443'
     hosts:
       - host: <<host-name>>
-        redirectPaths:
-          - path: /*
-            name: ssl-redirect
-            port: use-annotation
         paths:
           - /*
 ```

diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md
@@ -81,7 +81,8 @@ profiling:
 - #10498 - Tableau ingestion can now be configured to ingest multiple sites at once and add the sites as containers. The feature is currently only available for Tableau Server.
 - #10466 - Extends configuration in `~/.datahubenv` to match `DatahubClientConfig` object definition. See full configuration in https://datahubproject.io/docs/python-sdk/clients/. The CLI should now respect the updated configurations specified in `~/.datahubenv` across its functions and utilities. This means that for systems where ssl certification is disabled, setting `disable_ssl_verification: true` in `~./datahubenv` will apply to all CLI calls.
 - #11002 - We will not auto-generate a `~/.datahubenv` file. You must either run `datahub init` to create that file, or set environment variables so that the config is loaded.
-
+- #11023 - Added a new parameter to datahub's `put` cli command: `--run-id`. This parameter is useful to associate a given write to an ingestion process. A use-case can be mimick transformers when a transformer for aspect being written does not exist.
+- #11051 - Ingestion reports will now trim the summary text to a maximum of 800k characters to avoid generating `dataHubExecutionRequestResult` that are too large for GMS to handle.
 ## 0.13.3
 
 ### Breaking Changes

diff --git a/metadata-ingestion/src/datahub/ingestion/reporting/datahub_ingestion_run_summary_provider.py b/metadata-ingestion/src/datahub/ingestion/reporting/datahub_ingestion_run_summary_provider.py
@@ -31,6 +31,7 @@
 from datahub.utilities.logging_manager import get_log_buffer
 from datahub.utilities.urns.urn import Urn
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -43,6 +44,7 @@ class DatahubIngestionRunSummaryProvider(PipelineRunListener):
     _EXECUTOR_ID: str = "__datahub_cli_"
     _EXECUTION_REQUEST_SOURCE_TYPE: str = "CLI_INGESTION_SOURCE"
     _INGESTION_TASK_NAME: str = "CLI Ingestion"
+    _MAX_SUMMARY_SIZE: int = 800000
 
     @staticmethod
     def get_cur_time_in_ms() -> int:
@@ -209,7 +211,9 @@ def on_completion(
             status=status,
             startTimeMs=self.start_time_ms,
             durationMs=self.get_cur_time_in_ms() - self.start_time_ms,
-            report=summary,
+            # Truncate summary such that the generated MCP will not exceed GMS's payload limit.
+            # Hardcoding the overall size of dataHubExecutionRequestResult to >1MB by trimming summary to 800,000 chars
+            report=summary[-self._MAX_SUMMARY_SIZE:],
             structuredReport=structured_report,
         )