From e8c85bfe6cafae8e8f5916e9335a219080b0fd9a Mon Sep 17 00:00:00 2001 From: Allison Suarez Miranda Date: Sun, 21 Feb 2021 15:12:38 -0800 Subject: [PATCH 1/6] added lineage item and lineage entities Signed-off-by: Allison Suarez Miranda --- amundsen_common/models/lineage.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 amundsen_common/models/lineage.py diff --git a/amundsen_common/models/lineage.py b/amundsen_common/models/lineage.py new file mode 100644 index 0000000..fd27640 --- /dev/null +++ b/amundsen_common/models/lineage.py @@ -0,0 +1,30 @@ +# Copyright Contributors to the Amundsen project. +# SPDX-License-Identifier: Apache-2.0 + +from typing import Optional, List + +import attr +from marshmallow_annotations.ext.attrs import AttrsSchema + +@attr.s(auto_attribs=True, kw_only=True) +class LineageItem: + key: str # down/upstream table/col/task key + level: int # upstream/downstream distance from current resource + source: str # database this resource is from + badges: Optional[List[str]] + usage: Optional[int] # statistic to sort lineage items by + +class LineageItemSchema(AttrsSchema): + target = LineageItem + register_as_scheme = True + +@attr.s(auto_attribs=True, kw_only=True) +class Lineage: + key: str # current table/col/task key + direction: str # upstream/downstream + depth: int # how many levels up/down 0 == all + lineage_entities: List[LineageItem] # list of up/downstream entities + +class LineageSchema(AttrsSchema): + target = Lineage + register_as_scheme = True From d7cf7d6c66fa9b02e1b8c6292e1bc784b2188c4f Mon Sep 17 00:00:00 2001 From: Allison Suarez Miranda Date: Sun, 21 Feb 2021 15:16:33 -0800 Subject: [PATCH 2/6] lint fixes Signed-off-by: Allison Suarez Miranda --- amundsen_common/models/lineage.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/amundsen_common/models/lineage.py b/amundsen_common/models/lineage.py index fd27640..e7a8c64 100644 --- a/amundsen_common/models/lineage.py +++ b/amundsen_common/models/lineage.py @@ -6,25 +6,29 @@ import attr from marshmallow_annotations.ext.attrs import AttrsSchema + @attr.s(auto_attribs=True, kw_only=True) class LineageItem: key: str # down/upstream table/col/task key level: int # upstream/downstream distance from current resource source: str # database this resource is from badges: Optional[List[str]] - usage: Optional[int] # statistic to sort lineage items by + usage: Optional[int] # statistic to sort lineage items by + class LineageItemSchema(AttrsSchema): target = LineageItem register_as_scheme = True + @attr.s(auto_attribs=True, kw_only=True) class Lineage: - key: str # current table/col/task key + key: str # current table/col/task key direction: str # upstream/downstream - depth: int # how many levels up/down 0 == all + depth: int # how many levels up/down 0 == all lineage_entities: List[LineageItem] # list of up/downstream entities + class LineageSchema(AttrsSchema): target = Lineage register_as_scheme = True From 3c18437d104626e495aea39482bdb498886f74c0 Mon Sep 17 00:00:00 2001 From: Allison Suarez Miranda Date: Mon, 22 Feb 2021 10:25:13 -0800 Subject: [PATCH 3/6] added BAdge for badge list Signed-off-by: Allison Suarez Miranda --- amundsen_common/models/lineage.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/amundsen_common/models/lineage.py b/amundsen_common/models/lineage.py index e7a8c64..6646112 100644 --- a/amundsen_common/models/lineage.py +++ b/amundsen_common/models/lineage.py @@ -3,6 +3,8 @@ from typing import Optional, List +from amundsen_common.models.table import Badge + import attr from marshmallow_annotations.ext.attrs import AttrsSchema @@ -12,7 +14,7 @@ class LineageItem: key: str # down/upstream table/col/task key level: int # upstream/downstream distance from current resource source: str # database this resource is from - badges: Optional[List[str]] + badges: Optional[List[Badge]] usage: Optional[int] # statistic to sort lineage items by From aeaa79d0f99bc5b9eea84edf40efc20960f1dfd5 Mon Sep 17 00:00:00 2001 From: Allison Suarez Miranda Date: Tue, 23 Feb 2021 10:28:07 -0800 Subject: [PATCH 4/6] small change to lineage schema Signed-off-by: Allison Suarez Miranda --- amundsen_common/models/lineage.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/amundsen_common/models/lineage.py b/amundsen_common/models/lineage.py index 6646112..823eab0 100644 --- a/amundsen_common/models/lineage.py +++ b/amundsen_common/models/lineage.py @@ -14,8 +14,8 @@ class LineageItem: key: str # down/upstream table/col/task key level: int # upstream/downstream distance from current resource source: str # database this resource is from - badges: Optional[List[Badge]] - usage: Optional[int] # statistic to sort lineage items by + badges: Optional[List[Badge]] = None + usage: Optional[int] = None # statistic to sort lineage items by class LineageItemSchema(AttrsSchema): @@ -26,9 +26,11 @@ class LineageItemSchema(AttrsSchema): @attr.s(auto_attribs=True, kw_only=True) class Lineage: key: str # current table/col/task key - direction: str # upstream/downstream + direction: str # upstream/downstream/both depth: int # how many levels up/down 0 == all - lineage_entities: List[LineageItem] # list of up/downstream entities + lineage_entities_upstream: List[LineageItem] # list of upstream entities + lineage_entities_downstream: List[LineageItem] # list of downstream entities + class LineageSchema(AttrsSchema): From 43b61e353bb0b940573e1686b3946f82249ff6f2 Mon Sep 17 00:00:00 2001 From: Allison Suarez Miranda Date: Tue, 23 Feb 2021 10:30:08 -0800 Subject: [PATCH 5/6] whitespace Signed-off-by: Allison Suarez Miranda --- amundsen_common/models/lineage.py | 1 - 1 file changed, 1 deletion(-) diff --git a/amundsen_common/models/lineage.py b/amundsen_common/models/lineage.py index 823eab0..d855068 100644 --- a/amundsen_common/models/lineage.py +++ b/amundsen_common/models/lineage.py @@ -32,7 +32,6 @@ class Lineage: lineage_entities_downstream: List[LineageItem] # list of downstream entities - class LineageSchema(AttrsSchema): target = Lineage register_as_scheme = True From c21b731a1dacfe6df0fc45a6ae26004ad515db94 Mon Sep 17 00:00:00 2001 From: Allison Suarez Miranda Date: Tue, 23 Feb 2021 14:37:37 -0800 Subject: [PATCH 6/6] rename Signed-off-by: Allison Suarez Miranda --- amundsen_common/models/lineage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/amundsen_common/models/lineage.py b/amundsen_common/models/lineage.py index d855068..1f0e24d 100644 --- a/amundsen_common/models/lineage.py +++ b/amundsen_common/models/lineage.py @@ -28,8 +28,8 @@ class Lineage: key: str # current table/col/task key direction: str # upstream/downstream/both depth: int # how many levels up/down 0 == all - lineage_entities_upstream: List[LineageItem] # list of upstream entities - lineage_entities_downstream: List[LineageItem] # list of downstream entities + upstream_entities: List[LineageItem] # list of upstream entities + downstream_entities: List[LineageItem] # list of downstream entities class LineageSchema(AttrsSchema):