diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt.py b/metadata-ingestion/src/datahub/ingestion/source/dbt.py index ef286cb130f7b..5cc239c508df4 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dbt.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dbt.py @@ -83,6 +83,8 @@ class DBTNode: columns: List[DBTColumn] = field(default_factory=list) upstream_urns: List[str] = field(default_factory=list) + meta: Dict[str, Any] = field(default_factory=dict) + def __repr__(self): fields = tuple("{}={}".format(k, v) for k, v in self.__dict__.items()) return self.__class__.__name__ + str(tuple(sorted(fields))).replace("'", "") @@ -177,6 +179,7 @@ def extract_dbt_entities( target_platform, environment, ), + meta=node.get("meta", {}), ) # overwrite columns from catalog @@ -259,8 +262,11 @@ def get_urn_from_dbtNode( def get_custom_properties(node: DBTNode) -> Dict[str, str]: - custom_properties = {} + # initialize custom properties to node's meta props + # (dbt-native node properties) + custom_properties = node.meta + # additional node attributes to extract to custom properties node_attributes = ["node_type", "materialization", "dbt_file_path", "catalog_type"] for attribute in node_attributes: diff --git a/metadata-ingestion/tests/integration/dbt/dbt_catalog.json b/metadata-ingestion/tests/integration/dbt/dbt_catalog.json index c8fde3b144445..231e7e3b20e7a 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_catalog.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_catalog.json @@ -4,8 +4,8 @@ "dbt_schema_version": "https://schemas.getdbt.com/dbt/catalog/v1.json", "dbt_version": "0.19.1", "env": {}, - "generated_at": "2021-06-21T17:28:04.980076Z", - "invocation_id": "a878f3a5-ecf3-44b1-94ac-4334c92ca3f6" + "generated_at": "2021-07-13T18:38:17.650799Z", + "invocation_id": "a0290787-2b80-46c6-a3b7-bf882f654ff3" }, "nodes": { "model.sample_dbt.monthly_billing_with_cust": { diff --git a/metadata-ingestion/tests/integration/dbt/dbt_manifest.json b/metadata-ingestion/tests/integration/dbt/dbt_manifest.json index 4fedf1f668573..9a66e83460464 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_manifest.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_manifest.json @@ -3359,7 +3359,11 @@ "identifier": "actor", "loaded_at_field": "last_update", "loader": "", - "meta": {}, + "meta": { + "model_maturity": "in dev", + "owner": "@alice", + "some_other_property": "test 1" + }, "name": "actor", "original_file_path": "models/base.yml", "package_name": "sample_dbt", @@ -3559,7 +3563,11 @@ "identifier": "country", "loaded_at_field": "last_update", "loader": "", - "meta": {}, + "meta": { + "model_maturity": "in prod", + "owner": "@bob", + "some_other_property": "test 2" + }, "name": "country", "original_file_path": "models/base.yml", "package_name": "sample_dbt", @@ -3709,7 +3717,11 @@ "identifier": "payment_p2020_02", "loaded_at_field": "payment_date", "loader": "", - "meta": {}, + "meta": { + "model_maturity": "in prod", + "owner": "@charles", + "some_other_property": "test 3" + }, "name": "payment_p2020_02", "original_file_path": "models/base.yml", "package_name": "sample_dbt", diff --git a/metadata-ingestion/tests/integration/dbt/dbt_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_mces_golden.json index 2ad3756659114..fb8f13fce42e1 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_mces_golden.json @@ -562,6 +562,9 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { + "model_maturity": "in dev", + "owner": "@alice", + "some_other_property": "test 1", "node_type": "source", "dbt_file_path": "models/base.yml", "catalog_type": "BASE TABLE" @@ -1083,6 +1086,9 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { + "model_maturity": "in prod", + "owner": "@bob", + "some_other_property": "test 2", "node_type": "source", "dbt_file_path": "models/base.yml", "catalog_type": "BASE TABLE" @@ -1545,6 +1551,9 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { + "model_maturity": "in prod", + "owner": "@charles", + "some_other_property": "test 3", "node_type": "source", "dbt_file_path": "models/base.yml", "catalog_type": "BASE TABLE"