1.8.0

CHANGELOG.md

@@ -1,5 +1,9 @@
-# 1.7.0
-* Rule miner that proposes data quality rule configuration
-* Notifications can be filtered by various parameters, allowing to send notifications to the correct teams
-* Estimated word count checks added to support analyzing data for GenAI use cases
-* Some pattern validation checks renamed to use a unified naming convention "invalid" in the check names 
+# 1.8.0
+
+* Small layout fixes in the UI, mostly related to the colors of issue severity colors
+* Documentation for the data quality process and rule mining
+* Duplicate record detection check
+* Some column-level checks fixed to skip NULL values, instead of treating them as invalid (not passing checks)
+* Store data lineage definition (links to source tables)
+* Default observability check patterns renamed to data quality policies
+* Default quality policy named "default" deleted, and replaced with multiple smaller policies to allow more control 

distribution/python/dqops/client/api/default_column_check_patterns/copy_from_default_column_checks_pattern.py → distribution/python/dqops/client/api/column_quality_policies/copy_from_column_quality_policy.py

@@ -17,7 +17,7 @@ def _get_kwargs(
 
     return {
         "method": "post",
-        "url": "api/default/checks/column/{targetPatternName}/copyfrom/{sourcePatternName}".format(
+        "url": "api/policies/checks/column/{targetPatternName}/copyfrom/{sourcePatternName}".format(
             targetPatternName=target_pattern_name,
             sourcePatternName=source_pattern_name,
         ),
@@ -52,7 +52,7 @@ def sync_detailed(
     *,
     client: AuthenticatedClient,
 ) -> Response[Any]:
-    """copyFromDefaultColumnChecksPattern
+    """copyFromColumnQualityPolicy
 
      Creates (adds) a copy of an existing default column-level checks pattern configuration, under a new
     name.
@@ -87,7 +87,7 @@ async def asyncio_detailed(
     *,
     client: AuthenticatedClient,
 ) -> Response[Any]:
-    """copyFromDefaultColumnChecksPattern
+    """copyFromColumnQualityPolicy
 
      Creates (adds) a copy of an existing default column-level checks pattern configuration, under a new
     name.

distribution/python/dqops/client/api/default_column_check_patterns/create_default_column_checks_pattern.py → distribution/python/dqops/client/api/column_quality_policies/create_column_quality_policy.py

@@ -5,16 +5,14 @@
 
 from ... import errors
 from ...client import AuthenticatedClient, Client
-from ...models.default_column_checks_pattern_model import (
-    DefaultColumnChecksPatternModel,
-)
+from ...models.column_quality_policy_model import ColumnQualityPolicyModel
 from ...types import Response
 
 
 def _get_kwargs(
     pattern_name: str,
     *,
-    json_body: DefaultColumnChecksPatternModel,
+    json_body: ColumnQualityPolicyModel,
 ) -> Dict[str, Any]:
 
     pass
@@ -23,7 +21,7 @@ def _get_kwargs(
 
     return {
         "method": "post",
-        "url": "api/default/checks/column/{patternName}".format(
+        "url": "api/policies/checks/column/{patternName}".format(
             patternName=pattern_name,
         ),
         "json": json_json_body,
@@ -56,16 +54,17 @@ def sync_detailed(
     pattern_name: str,
     *,
     client: AuthenticatedClient,
-    json_body: DefaultColumnChecksPatternModel,
+    json_body: ColumnQualityPolicyModel,
 ) -> Response[Any]:
-    """createDefaultColumnChecksPattern
+    """createColumnQualityPolicy
 
      Creates (adds) a new default column-level checks pattern configuration by saving a full
     specification object.
 
     Args:
         pattern_name (str):
-        json_body (DefaultColumnChecksPatternModel): Default column-level checks pattern model
+        json_body (ColumnQualityPolicyModel): Default column-level checks pattern (data quality
+            policy) model
 
     Raises:
         errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
@@ -91,16 +90,17 @@ async def asyncio_detailed(
     pattern_name: str,
     *,
     client: AuthenticatedClient,
-    json_body: DefaultColumnChecksPatternModel,
+    json_body: ColumnQualityPolicyModel,
 ) -> Response[Any]:
-    """createDefaultColumnChecksPattern
+    """createColumnQualityPolicy
 
      Creates (adds) a new default column-level checks pattern configuration by saving a full
     specification object.
 
     Args:
         pattern_name (str):
-        json_body (DefaultColumnChecksPatternModel): Default column-level checks pattern model
+        json_body (ColumnQualityPolicyModel): Default column-level checks pattern (data quality
+            policy) model
 
     Raises:
         errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.

distribution/python/dqops/client/api/default_column_check_patterns/create_default_column_checks_pattern_target.py → distribution/python/dqops/client/api/column_quality_policies/create_column_quality_policy_target.py

@@ -5,16 +5,14 @@
 
 from ... import errors
 from ...client import AuthenticatedClient, Client
-from ...models.default_column_checks_pattern_list_model import (
-    DefaultColumnChecksPatternListModel,
-)
+from ...models.column_quality_policy_list_model import ColumnQualityPolicyListModel
 from ...types import Response
 
 
 def _get_kwargs(
     pattern_name: str,
     *,
-    json_body: DefaultColumnChecksPatternListModel,
+    json_body: ColumnQualityPolicyListModel,
 ) -> Dict[str, Any]:
 
     pass
@@ -23,7 +21,7 @@ def _get_kwargs(
 
     return {
         "method": "post",
-        "url": "api/default/checks/column/{patternName}/target".format(
+        "url": "api/policies/checks/column/{patternName}/target".format(
             patternName=pattern_name,
         ),
         "json": json_json_body,
@@ -56,16 +54,16 @@ def sync_detailed(
     pattern_name: str,
     *,
     client: AuthenticatedClient,
-    json_body: DefaultColumnChecksPatternListModel,
+    json_body: ColumnQualityPolicyListModel,
 ) -> Response[Any]:
-    """createDefaultColumnChecksPatternTarget
+    """createColumnQualityPolicyTarget
 
      Creates (adds) a new default column-level checks pattern configuration.
 
     Args:
         pattern_name (str):
-        json_body (DefaultColumnChecksPatternListModel): Default column-level checks pattern list
-            model
+        json_body (ColumnQualityPolicyListModel): Default column-level checks pattern (data
+            quality policy) list model
 
     Raises:
         errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
@@ -91,16 +89,16 @@ async def asyncio_detailed(
     pattern_name: str,
     *,
     client: AuthenticatedClient,
-    json_body: DefaultColumnChecksPatternListModel,
+    json_body: ColumnQualityPolicyListModel,
 ) -> Response[Any]:
-    """createDefaultColumnChecksPatternTarget
+    """createColumnQualityPolicyTarget
 
      Creates (adds) a new default column-level checks pattern configuration.
 
     Args:
         pattern_name (str):
-        json_body (DefaultColumnChecksPatternListModel): Default column-level checks pattern list
-            model
+        json_body (ColumnQualityPolicyListModel): Default column-level checks pattern (data
+            quality policy) list model
 
     Raises:
         errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.

distribution/python/dqops/client/api/default_column_check_patterns/delete_default_column_checks_pattern.py → distribution/python/dqops/client/api/column_quality_policies/delete_column_quality_policy.py

@@ -16,7 +16,7 @@ def _get_kwargs(
 
     return {
         "method": "delete",
-        "url": "api/default/checks/column/{patternName}".format(
+        "url": "api/policies/checks/column/{patternName}".format(
             patternName=pattern_name,
         ),
     }
@@ -49,7 +49,7 @@ def sync_detailed(
     *,
     client: AuthenticatedClient,
 ) -> Response[Any]:
-    """deleteDefaultColumnChecksPattern
+    """deleteColumnQualityPolicy
 
      Deletes a default column-level checks pattern
 
@@ -80,7 +80,7 @@ async def asyncio_detailed(
     *,
     client: AuthenticatedClient,
 ) -> Response[Any]:
-    """deleteDefaultColumnChecksPattern
+    """deleteColumnQualityPolicy
 
      Deletes a default column-level checks pattern
 

distribution/python/dqops/client/api/default_column_check_patterns/get_all_default_column_checks_patterns.py → distribution/python/dqops/client/api/column_quality_policies/get_column_quality_policies.py

@@ -5,9 +5,7 @@
 
 from ... import errors
 from ...client import AuthenticatedClient, Client
-from ...models.default_column_checks_pattern_list_model import (
-    DefaultColumnChecksPatternListModel,
-)
+from ...models.column_quality_policy_list_model import ColumnQualityPolicyListModel
 from ...types import Response
 
 
@@ -17,18 +15,18 @@ def _get_kwargs() -> Dict[str, Any]:
 
     return {
         "method": "get",
-        "url": "api/default/checks/column",
+        "url": "api/policies/checks/column",
     }
 
 
 def _parse_response(
     *, client: Union[AuthenticatedClient, Client], response: httpx.Response
-) -> Optional[List["DefaultColumnChecksPatternListModel"]]:
+) -> Optional[List["ColumnQualityPolicyListModel"]]:
     if response.status_code == HTTPStatus.OK:
         response_200 = []
         _response_200 = response.json()
         for response_200_item_data in _response_200:
-            response_200_item = DefaultColumnChecksPatternListModel.from_dict(
+            response_200_item = ColumnQualityPolicyListModel.from_dict(
                 response_200_item_data
             )
 
@@ -43,7 +41,7 @@ def _parse_response(
 
 def _build_response(
     *, client: Union[AuthenticatedClient, Client], response: httpx.Response
-) -> Response[List["DefaultColumnChecksPatternListModel"]]:
+) -> Response[List["ColumnQualityPolicyListModel"]]:
     return Response(
         status_code=HTTPStatus(response.status_code),
         content=response.content,
@@ -55,8 +53,8 @@ def _build_response(
 def sync_detailed(
     *,
     client: AuthenticatedClient,
-) -> Response[List["DefaultColumnChecksPatternListModel"]]:
-    """getAllDefaultColumnChecksPatterns
+) -> Response[List["ColumnQualityPolicyListModel"]]:
+    """getColumnQualityPolicies
 
      Returns a flat list of all column-level default check patterns configured for this instance. Default
     checks are applied on columns dynamically.
@@ -66,7 +64,7 @@ def sync_detailed(
         httpx.TimeoutException: If the request takes longer than Client.timeout.
 
     Returns:
-        Response[List['DefaultColumnChecksPatternListModel']]
+        Response[List['ColumnQualityPolicyListModel']]
     """
 
     kwargs = _get_kwargs()
@@ -81,8 +79,8 @@ def sync_detailed(
 def sync(
     *,
     client: AuthenticatedClient,
-) -> Optional[List["DefaultColumnChecksPatternListModel"]]:
-    """getAllDefaultColumnChecksPatterns
+) -> Optional[List["ColumnQualityPolicyListModel"]]:
+    """getColumnQualityPolicies
 
      Returns a flat list of all column-level default check patterns configured for this instance. Default
     checks are applied on columns dynamically.
@@ -92,7 +90,7 @@ def sync(
         httpx.TimeoutException: If the request takes longer than Client.timeout.
 
     Returns:
-        List['DefaultColumnChecksPatternListModel']
+        List['ColumnQualityPolicyListModel']
     """
 
     return sync_detailed(
@@ -103,8 +101,8 @@ def sync(
 async def asyncio_detailed(
     *,
     client: AuthenticatedClient,
-) -> Response[List["DefaultColumnChecksPatternListModel"]]:
-    """getAllDefaultColumnChecksPatterns
+) -> Response[List["ColumnQualityPolicyListModel"]]:
+    """getColumnQualityPolicies
 
      Returns a flat list of all column-level default check patterns configured for this instance. Default
     checks are applied on columns dynamically.
@@ -114,7 +112,7 @@ async def asyncio_detailed(
         httpx.TimeoutException: If the request takes longer than Client.timeout.
 
     Returns:
-        Response[List['DefaultColumnChecksPatternListModel']]
+        Response[List['ColumnQualityPolicyListModel']]
     """
 
     kwargs = _get_kwargs()
@@ -127,8 +125,8 @@ async def asyncio_detailed(
 async def asyncio(
     *,
     client: AuthenticatedClient,
-) -> Optional[List["DefaultColumnChecksPatternListModel"]]:
-    """getAllDefaultColumnChecksPatterns
+) -> Optional[List["ColumnQualityPolicyListModel"]]:
+    """getColumnQualityPolicies
 
      Returns a flat list of all column-level default check patterns configured for this instance. Default
     checks are applied on columns dynamically.
@@ -138,7 +136,7 @@ async def asyncio(
         httpx.TimeoutException: If the request takes longer than Client.timeout.
 
     Returns:
-        List['DefaultColumnChecksPatternListModel']
+        List['ColumnQualityPolicyListModel']
     """
 
     return (